feat(hub): v0.70.0 — a deleted customer actually disappears (residue leg + ghost cleanup)
Found validating v0.69.0 against the live hub. demo-vm-felhom was deleted on 07-18 and was still on the Customers list AND still raising offsite_stale (10 events, latest 07-21 17:34, operator email at 19:34) — because GetCustomers() is report-derived and no lifecycle tier ever deleted a report. New leg 3 (residue), before the record purge: reports, app_telemetry, app_log_tails, log_tail_requests, customer_notifications, plus the credential-bearing appliance_registrations and selfbind_tokens. Audit (events, notification_log) and F-14 provenance still survive. Ghost customers are now deletable: 404 means "nothing here", not "no config row". With no config row the offsite descriptor is unknowable, so the Hetzner and descriptor legs record skipped_no_config rather than a bare "skipped". Two more red-proofs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J55BQE1gE2V4ffud5jweGS
This commit is contained in:
@@ -13,16 +13,20 @@ package web
|
||||
// (ONLINE refuses; escrow is DEMOTED to retained custody, never destroyed)
|
||||
// leg 2 reset — the committed RESET sequence verbatim (Hetzner FIRST, PBS, claim, descriptor,
|
||||
// DB purge) via commitCustomerReset — with purgeEscrow=FALSE, see below
|
||||
// leg 3 purge — DeleteCustomerConfig: the customer record AND all escrow ciphertext
|
||||
// leg 3 residue — the report stream + credential-bearing bindings (v0.70.0): reports, app
|
||||
// telemetry/log tails, notification prefs, self-bind tokens, appliance
|
||||
// registrations. Without this leg a deleted customer stays on the Customers list
|
||||
// and keeps raising staleness alerts, because GetCustomers() is report-derived
|
||||
// leg 4 purge — DeleteCustomerConfig: the customer record AND all escrow ciphertext
|
||||
//
|
||||
// Nothing here is newly destructive: the cascade only SEQUENCES three operations that already exist,
|
||||
// each keeping its own safety rules. Two invariants are load-bearing:
|
||||
// Nothing here is newly destructive: the cascade only SEQUENCES operations that already exist (plus
|
||||
// the v0.70.0 residue sweep), each keeping its own safety rules. Two invariants are load-bearing:
|
||||
//
|
||||
// - Ruling 3 is preserved BY CONSTRUCTION: leg 2 can only run after leg 1, so the RESET sequence
|
||||
// never sees a host row. The standalone RESET handler's 409 gate is untouched.
|
||||
// - Custody is purged EXACTLY ONCE, in leg 3. Leg 1 demotes (host_escrow → host_escrow_superseded);
|
||||
// - Custody is purged EXACTLY ONCE, in leg 4. Leg 1 demotes (host_escrow → host_escrow_superseded);
|
||||
// leg 2 is called with purgeEscrow=false so PurgeCustomerResetDBState leaves the retained blobs
|
||||
// alone; leg 3's DeleteCustomerConfig is the one true purge point (v0.60.1).
|
||||
// alone; leg 4's DeleteCustomerConfig is the one true purge point (v0.60.1).
|
||||
//
|
||||
// A leg that fails leaves the journal row retained and the error names the leg. A re-run resumes:
|
||||
// leg 1 is a no-op once the hosts are gone, and every leg of the RESET sequence is idempotent.
|
||||
@@ -55,7 +59,8 @@ func readDeleteCascadeAcks(r *http.Request) deleteCascadeAcks {
|
||||
func (a deleteCascadeAcks) complete() bool { return a.Hosts && a.Reset && a.Purge }
|
||||
|
||||
// handleCustomerDeletePreview — GET /configs/{id}/delete. The read-only inventory the guided dialog
|
||||
// renders: the RESET inventory EXTENDED with the host list (ruling 4 applied to all three legs).
|
||||
// renders: the RESET inventory EXTENDED with the host list and the residue counts (ruling 4 applied
|
||||
// to every leg).
|
||||
// Counts, names and booleans only — never a secret, blob or key. Also surfaces an incomplete journal
|
||||
// row so the dialog can offer "Resume".
|
||||
func (s *Server) handleCustomerDeletePreview(w http.ResponseWriter, r *http.Request, customerID string) {
|
||||
@@ -65,10 +70,6 @@ func (s *Server) handleCustomerDeletePreview(w http.ResponseWriter, r *http.Requ
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if cfg == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
inv, err := s.store.CustomerResetInventory(customerID)
|
||||
if err != nil {
|
||||
s.logger.Printf("[ERROR] delete preview %s: inventory: %v", customerID, err)
|
||||
@@ -81,6 +82,18 @@ func (s *Server) handleCustomerDeletePreview(w http.ResponseWriter, r *http.Requ
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
residue, err := s.store.CustomerResidue(customerID)
|
||||
if err != nil {
|
||||
s.logger.Printf("[ERROR] delete preview %s: residue: %v", customerID, err)
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Ghost customers (config row already gone, residue alive) MUST preview — the dialog is the only
|
||||
// surface that can clear them. 404 only when there is genuinely nothing left.
|
||||
if cfg == nil && len(hosts) == 0 && residue.Total() == 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
hostRows := make([]map[string]any, 0, len(hosts))
|
||||
onlineBlocked := false
|
||||
for i := range hosts {
|
||||
@@ -94,7 +107,10 @@ func (s *Server) handleCustomerDeletePreview(w http.ResponseWriter, r *http.Requ
|
||||
"online": status == "ok",
|
||||
})
|
||||
}
|
||||
offsiteEnabled, offsiteType := offsiteChoice(cfg.ConfigJSON)
|
||||
offsiteEnabled, offsiteType := false, ""
|
||||
if cfg != nil {
|
||||
offsiteEnabled, offsiteType = offsiteChoice(cfg.ConfigJSON)
|
||||
}
|
||||
offsiteName := ""
|
||||
if offsiteEnabled && s.offsite != nil {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
|
||||
@@ -105,6 +121,10 @@ func (s *Server) handleCustomerDeletePreview(w http.ResponseWriter, r *http.Requ
|
||||
offsiteName = n
|
||||
}
|
||||
}
|
||||
customerName := ""
|
||||
if cfg != nil {
|
||||
customerName = cfg.CustomerName
|
||||
}
|
||||
// An incomplete journal row = a cascade that stopped mid-way; the dialog renders it + Resume.
|
||||
var pending map[string]any
|
||||
if cr, jerr := s.store.LatestCustomerReset(customerID); jerr != nil {
|
||||
@@ -119,7 +139,8 @@ func (s *Server) handleCustomerDeletePreview(w http.ResponseWriter, r *http.Requ
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"customer_id": customerID,
|
||||
"customer_name": cfg.CustomerName,
|
||||
"customer_name": customerName,
|
||||
"has_config": cfg != nil, // false = GHOST (config already gone, residue alive)
|
||||
"hosts": hostRows,
|
||||
"host_count": inv.HostCount,
|
||||
"online_host_present": onlineBlocked, // leg 1 refuses; decommission the agent first
|
||||
@@ -132,6 +153,16 @@ func (s *Server) handleCustomerDeletePreview(w http.ResponseWriter, r *http.Requ
|
||||
"offsite_identifier": offsiteName,
|
||||
"pbs_tenancy_configured": s.tenantsync != nil,
|
||||
"pending_journal": pending,
|
||||
"residue_total": residue.Total(),
|
||||
"residue": map[string]int{
|
||||
"reports": residue.Reports,
|
||||
"app_telemetry": residue.AppTelemetry,
|
||||
"app_log_tails": residue.AppLogTails,
|
||||
"log_tail_requests": residue.LogTailRequests,
|
||||
"notification_prefs": residue.NotificationPrefs,
|
||||
"selfbind_tokens": residue.SelfBindTokens,
|
||||
"appliance_registrations": residue.ApplianceRegistrations,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -145,16 +176,26 @@ func (s *Server) handleCustomerDelete(w http.ResponseWriter, r *http.Request, cu
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if cfg == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
hosts, err := s.store.ListHostsByCustomer(customerID)
|
||||
if err != nil {
|
||||
s.logger.Printf("[ERROR] delete %s: hosts: %v", customerID, err)
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
residue, err := s.store.CustomerResidue(customerID)
|
||||
if err != nil {
|
||||
s.logger.Printf("[ERROR] delete %s: residue: %v", customerID, err)
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// 404 means "there is nothing here", not "there is no config row". A GHOST customer — config
|
||||
// already deleted but the report stream (and therefore the Customers list, and therefore the
|
||||
// staleness alerts) still alive — MUST be deletable; before v0.70.0 it 404'd and no operator
|
||||
// surface could clear it.
|
||||
if cfg == nil && len(hosts) == 0 && residue.Total() == 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// ── Gates (all before any write) ────────────────────────────────────────────────────────────
|
||||
acks := readDeleteCascadeAcks(r)
|
||||
@@ -214,18 +255,36 @@ func (s *Server) handleCustomerDelete(w http.ResponseWriter, r *http.Request, cu
|
||||
_ = s.store.UpdateResetLeg(journalID, "hosts", "ok")
|
||||
|
||||
// ── Leg 2: the committed RESET sequence (external teardown FIRST, DB purge last) ─────────────
|
||||
// purgeEscrow=false — retained custody dies exactly once, in leg 3.
|
||||
if lerr := s.commitCustomerReset(ctx, cfg, journalID, false); lerr != nil {
|
||||
// purgeEscrow=false — retained custody dies exactly once, in leg 4.
|
||||
// cfg may be nil (ghost customer): the Hetzner + descriptor legs then record `skipped_no_config`.
|
||||
if lerr := s.commitCustomerReset(ctx, cfg, customerID, journalID, false); lerr != nil {
|
||||
s.logger.Printf("[ERROR] delete %s: cascade stopped at leg 2 (%s) — journal #%d retained", customerID, lerr.Leg, journalID)
|
||||
http.Error(w, "Delete incomplete at leg 2 (reset/"+lerr.Leg+"): "+lerr.Msg+" The host(s) are already deleted; re-run to resume.", lerr.Status)
|
||||
return
|
||||
}
|
||||
|
||||
// ── Leg 3: the one true purge point — customer record + ALL escrow ciphertext ────────────────
|
||||
// ── Leg 3: residue — the report stream + the credential-bearing bindings ─────────────────────
|
||||
// This is what actually makes the customer DISAPPEAR: `GetCustomers()` builds the Customers list
|
||||
// (and the staleness/offsite checkers' work list) purely from `reports`, so before v0.70.0 a
|
||||
// fully deleted customer stayed visible AND kept emailing the operator. It runs BEFORE the record
|
||||
// purge on purpose — the customer_configs row is the identifying descriptor and goes LAST, and a
|
||||
// crash between the two legs leaves a ghost the cascade can now clean up on a re-run.
|
||||
if perr := s.store.PurgeCustomerResidue(customerID); perr != nil {
|
||||
_ = s.store.UpdateResetLeg(journalID, "residue", "failed")
|
||||
s.logger.Printf("[ERROR] delete %s: residue purge FAILED (journal #%d retained; re-run to resume): %v", customerID, journalID, perr)
|
||||
http.Error(w, "Delete incomplete at leg 3 (residue): the report stream and bindings could not be purged — the customer record is NOT yet removed; re-run to resume. ("+perr.Error()+")", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = s.store.UpdateResetLeg(journalID, "residue", "ok")
|
||||
s.logger.Printf("[INFO] delete %s: residue purged (reports=%d app_telemetry=%d app_log_tails=%d log_tail_requests=%d notif_prefs=%d selfbind_tokens=%d appliance_registrations=%d)",
|
||||
customerID, residue.Reports, residue.AppTelemetry, residue.AppLogTails, residue.LogTailRequests,
|
||||
residue.NotificationPrefs, residue.SelfBindTokens, residue.ApplianceRegistrations)
|
||||
|
||||
// ── Leg 4: the one true purge point — customer record + ALL escrow ciphertext ────────────────
|
||||
if derr := s.store.DeleteCustomerConfig(customerID); derr != nil {
|
||||
_ = s.store.UpdateResetLeg(journalID, "customer_delete", "failed")
|
||||
s.logger.Printf("[ERROR] delete %s: final purge FAILED (journal #%d retained; re-run to resume): %v", customerID, journalID, derr)
|
||||
http.Error(w, "Delete incomplete at leg 3 (purge): the customer record could not be removed — re-run to resume. ("+derr.Error()+")", http.StatusInternalServerError)
|
||||
http.Error(w, "Delete incomplete at leg 4 (purge): the customer record could not be removed — re-run to resume. ("+derr.Error()+")", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_ = s.store.UpdateResetLeg(journalID, "customer_delete", "ok")
|
||||
|
||||
@@ -379,7 +379,7 @@ func TestCommitCustomerReset_PurgeEscrowFlagGovernsCustody(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("journal: %v", err)
|
||||
}
|
||||
if lerr := s.commitCustomerReset(context.Background(), cfg, id, false); lerr != nil {
|
||||
if lerr := s.commitCustomerReset(context.Background(), cfg, "acme", id, false); lerr != nil {
|
||||
t.Fatalf("commitCustomerReset: %v", lerr)
|
||||
}
|
||||
if n := superseded(t, st, "acme"); n != 2 {
|
||||
@@ -419,3 +419,169 @@ func TestDeleteCascadePreview_Inventory(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Residue (v0.70.0): the leg that actually makes a deleted customer DISAPPEAR ─────────────────
|
||||
//
|
||||
// The v0.69.0 cascade left the report stream behind, and `GetCustomers()` builds the Customers list
|
||||
// (and the staleness/offsite checkers' work list) purely from `reports` — so a fully deleted
|
||||
// customer stayed visible AND kept emailing the operator. Observed live on `demo-vm-felhom`:
|
||||
// deleted 2026-07-18, still raising `offsite_stale` on 2026-07-21.
|
||||
|
||||
// seedResidue adds the report-derived state + the credential-bearing bindings to a customer.
|
||||
func seedResidue(t *testing.T, st *store.Store, customerID string) {
|
||||
t.Helper()
|
||||
if err := st.SaveReport(customerID, []byte(`{"health":{"status":"ok"}}`)); err != nil {
|
||||
t.Fatalf("seed report: %v", err)
|
||||
}
|
||||
if err := st.SaveAppTelemetry(customerID, time.Now(), []store.AppTelemetryRecord{
|
||||
{AppName: "immich", DisplayName: "Immich", MemoryCurrentMB: 512},
|
||||
}); err != nil {
|
||||
t.Fatalf("seed telemetry: %v", err)
|
||||
}
|
||||
if err := st.SaveNotificationPrefs(customerID, "t@example.com", []string{"host_down"}, 6); err != nil {
|
||||
t.Fatalf("seed notif prefs: %v", err)
|
||||
}
|
||||
if err := st.MintSelfBindToken(customerID, "tokenhash-"+customerID, time.Hour); err != nil {
|
||||
t.Fatalf("seed selfbind token: %v", err)
|
||||
}
|
||||
// A DELIVERED appliance registration bound to the customer — credential-bearing (token_hash).
|
||||
if _, _, err := st.RegisterAppliance("uuid-"+customerID, "aa:bb", "ssh-ed25519 AAAA", "{}", "apphash-"+customerID, ""); err != nil {
|
||||
t.Fatalf("seed appliance: %v", err)
|
||||
}
|
||||
app, err := st.ApplianceByToken("apphash-" + customerID)
|
||||
if err != nil || app == nil {
|
||||
t.Fatalf("seed appliance lookup: %v (row=%v)", err, app != nil)
|
||||
}
|
||||
if err := st.BindAppliance(app.ID, customerID, "appliance", ""); err != nil {
|
||||
t.Fatalf("bind appliance: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func listedInCustomers(t *testing.T, st *store.Store, customerID string) bool {
|
||||
t.Helper()
|
||||
cs, err := st.GetCustomers()
|
||||
if err != nil {
|
||||
t.Fatalf("GetCustomers: %v", err)
|
||||
}
|
||||
for _, c := range cs {
|
||||
if c.CustomerID == customerID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The cascade purges the residue, so the customer leaves the Customers list — and with it the
|
||||
// staleness/offsite checkers' work list. RED-PROOF: drop the residue leg and this FAILS with the
|
||||
// customer still listed and 1 report row alive.
|
||||
func TestDeleteCascade_PurgesResidueAndUnlistsCustomer(t *testing.T) {
|
||||
s, st := newTestServer(t)
|
||||
seedDeletable(t, st, "acme")
|
||||
seedResidue(t, st, "acme")
|
||||
s.SetTenantSync(&orderTenancy{})
|
||||
|
||||
if !listedInCustomers(t, st, "acme") {
|
||||
t.Fatal("precondition: the customer must be listed before the cascade")
|
||||
}
|
||||
res, err := st.CustomerResidue("acme")
|
||||
if err != nil || res.Total() == 0 {
|
||||
t.Fatalf("precondition: residue must exist (err=%v, total=%d)", err, res.Total())
|
||||
}
|
||||
|
||||
if rr := postDelete(t, s, "acme", cascadeForm("acme", 1)); rr.Code != http.StatusSeeOther {
|
||||
t.Fatalf("status = %d, want 303: %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
|
||||
after, err := st.CustomerResidue("acme")
|
||||
if err != nil {
|
||||
t.Fatalf("residue: %v", err)
|
||||
}
|
||||
if after.Total() != 0 {
|
||||
t.Errorf("residue after cascade = %+v, want all zero", *after)
|
||||
}
|
||||
if listedInCustomers(t, st, "acme") {
|
||||
t.Error("the customer is STILL on the Customers list after a complete delete — the ghost that " +
|
||||
"kept raising offsite_stale alerts for demo-vm-felhom")
|
||||
}
|
||||
// The credential-bearing rows are gone by NAME, not just by count.
|
||||
if app, _ := st.ApplianceByToken("apphash-acme"); app != nil {
|
||||
t.Error("the appliance registration (token_hash, status=delivered) outlived its customer")
|
||||
}
|
||||
if n, _ := st.CountSelfBindTokens("acme"); n != 0 {
|
||||
t.Errorf("self-bind tokens = %d, want 0 — a live bind path to a deleted customer", n)
|
||||
}
|
||||
// Audit + provenance SURVIVE, exactly as in every other tier.
|
||||
if evs, _ := st.GetRecentEvents("acme", 10); len(evs) == 0 {
|
||||
t.Error("the audit event stream was purged — it must outlive every lifecycle tier")
|
||||
}
|
||||
if d, _ := st.LatestHostDeletion("acme"); d == nil {
|
||||
t.Error("F-14 host-deletion provenance was purged — it must survive")
|
||||
}
|
||||
cr, _ := st.LatestCustomerReset("acme")
|
||||
if cr == nil || cr.Legs["residue"] != "ok" || cr.Legs["customer_delete"] != "ok" {
|
||||
t.Errorf("journal legs = %v, want residue=ok customer_delete=ok", cr)
|
||||
}
|
||||
}
|
||||
|
||||
// A GHOST — config row already gone (a pre-v0.70.0 delete), residue alive. Before v0.70.0 this
|
||||
// 404'd and NO operator surface could clear it. This is the demo-vm-felhom shape exactly.
|
||||
func TestDeleteCascade_GhostCustomerIsDeletable(t *testing.T) {
|
||||
s, st := newTestServer(t)
|
||||
seedDeletable(t, st, "ghost")
|
||||
seedResidue(t, st, "ghost")
|
||||
s.SetTenantSync(&orderTenancy{})
|
||||
// Model the pre-v0.70.0 aftermath: hosts deleted, config row dropped, residue left behind.
|
||||
if err := st.DeleteHost("ghost-01", true); err != nil {
|
||||
t.Fatalf("delete host: %v", err)
|
||||
}
|
||||
if err := st.DeleteCustomerConfig("ghost"); err != nil {
|
||||
t.Fatalf("drop config: %v", err)
|
||||
}
|
||||
if cfg, _ := st.GetCustomerConfig("ghost"); cfg != nil {
|
||||
t.Fatal("precondition: the config row must be gone")
|
||||
}
|
||||
if !listedInCustomers(t, st, "ghost") {
|
||||
t.Fatal("precondition: the ghost must still be listed (that IS the defect)")
|
||||
}
|
||||
|
||||
// The preview must render it rather than 404 — it is the only surface that can clear a ghost.
|
||||
req := httptest.NewRequest("GET", "/configs/ghost/delete", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
s.handleCustomerDeletePreview(rr, req, "ghost")
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("ghost preview = %d, want 200: %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), `"has_config":false`) {
|
||||
t.Errorf("preview must mark the ghost (has_config=false): %s", rr.Body.String())
|
||||
}
|
||||
|
||||
if rr := postDelete(t, s, "ghost", cascadeForm("ghost", 0)); rr.Code != http.StatusSeeOther {
|
||||
t.Fatalf("ghost cascade = %d, want 303: %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if listedInCustomers(t, st, "ghost") {
|
||||
t.Error("the ghost survived its own cleanup")
|
||||
}
|
||||
if res, _ := st.CustomerResidue("ghost"); res.Total() != 0 {
|
||||
t.Errorf("ghost residue = %+v, want all zero", *res)
|
||||
}
|
||||
// With no config row the offsite descriptor is unknowable — the journal must SAY so, never
|
||||
// record a bare "skipped" that reads as "there was nothing to do".
|
||||
cr, _ := st.LatestCustomerReset("ghost")
|
||||
if cr == nil || cr.Legs["hetzner"] != "skipped_no_config" || cr.Legs["descriptor"] != "skipped_no_config" {
|
||||
t.Errorf("journal legs = %v, want hetzner/descriptor = skipped_no_config", cr)
|
||||
}
|
||||
}
|
||||
|
||||
// 404 still means "there is nothing here" — an id with no config, no host and no residue.
|
||||
func TestDeleteCascade_404WhenNothingRemains(t *testing.T) {
|
||||
s, _ := newTestServer(t)
|
||||
req := httptest.NewRequest("GET", "/configs/nobody/delete", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
s.handleCustomerDeletePreview(rr, req, "nobody")
|
||||
if rr.Code != http.StatusNotFound {
|
||||
t.Errorf("preview for an empty id = %d, want 404", rr.Code)
|
||||
}
|
||||
if rr2 := postDelete(t, s, "nobody", cascadeForm("nobody", 0)); rr2.Code != http.StatusNotFound {
|
||||
t.Errorf("cascade for an empty id = %d, want 404", rr2.Code)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,7 +137,7 @@ func (s *Server) handleCustomerReset(w http.ResponseWriter, r *http.Request, cus
|
||||
s.logger.Printf("[INFO] customer RESET started for %s (journal #%d, escrow_ack=%t)", customerID, resetID, escrowAck)
|
||||
|
||||
// Standalone RESET purges the retained custody itself, gated by the ack it just checked.
|
||||
if lerr := s.commitCustomerReset(ctx, cfg, resetID, escrowAck); lerr != nil {
|
||||
if lerr := s.commitCustomerReset(ctx, cfg, customerID, resetID, escrowAck); lerr != nil {
|
||||
http.Error(w, lerr.Msg, lerr.Status)
|
||||
return
|
||||
}
|
||||
@@ -193,12 +193,28 @@ func (e *resetLegError) Error() string {
|
||||
//
|
||||
// Behaviour for the standalone caller is byte-identical to v0.68.1 (same order, same leg names, same
|
||||
// messages, same status codes).
|
||||
func (s *Server) commitCustomerReset(ctx context.Context, cfg *store.CustomerConfig, resetID int64, purgeEscrow bool) *resetLegError {
|
||||
customerID := cfg.CustomerID
|
||||
//
|
||||
// GHOST CUSTOMERS (v0.70.0): cfg may be nil — the DELETE cascade also runs against a customer whose
|
||||
// config row is already gone but whose residue is not (a pre-v0.70.0 delete leaves the report stream
|
||||
// behind; see store/customer_delete.go). The standalone RESET handler 404s on a nil config before it
|
||||
// ever gets here, so this path is cascade-only. With no config row the offsite DESCRIPTOR is
|
||||
// unknowable, so the Hetzner leg is skipped and SAYS SO in the journal (`skipped_no_config`) rather
|
||||
// than silently reporting "skipped"; PBS is customer-id-keyed and idempotent, so it still runs.
|
||||
func (s *Server) commitCustomerReset(ctx context.Context, cfg *store.CustomerConfig, customerID string, resetID int64, purgeEscrow bool) *resetLegError {
|
||||
if cfg != nil {
|
||||
customerID = cfg.CustomerID
|
||||
}
|
||||
|
||||
// Leg: Hetzner offsite (repo DATA destroyed). Only when the customer chose an offsite tier.
|
||||
offsiteEnabled, offsiteType := offsiteChoice(cfg.ConfigJSON)
|
||||
if offsiteEnabled && s.offsite != nil {
|
||||
offsiteEnabled, offsiteType := false, ""
|
||||
if cfg != nil {
|
||||
offsiteEnabled, offsiteType = offsiteChoice(cfg.ConfigJSON)
|
||||
}
|
||||
if cfg == nil {
|
||||
// No descriptor to read — never guess a tier, and never let the journal imply "nothing to do".
|
||||
s.logger.Printf("[WARN] %s: no config row — the offsite (Hetzner) teardown CANNOT be determined and is SKIPPED; verify the Hetzner side by hand", customerID)
|
||||
_ = s.store.UpdateResetLeg(resetID, "hetzner", "skipped_no_config")
|
||||
} else if offsiteEnabled && s.offsite != nil {
|
||||
if derr := s.offsite.Deprovision(ctx, customerID, offsiteType); derr != nil {
|
||||
_ = s.store.UpdateResetLeg(resetID, "hetzner", "failed")
|
||||
s.logger.Printf("[ERROR] reset %s: hetzner deprovision FAILED (journal #%d retained; re-run to resume): %v", customerID, resetID, derr)
|
||||
@@ -228,7 +244,7 @@ func (s *Server) commitCustomerReset(ctx context.Context, cfg *store.CustomerCon
|
||||
|
||||
// All external legs are ok — now the DB side (publish-last, one leg at a time so the journal
|
||||
// records where a mid-purge crash stopped). Claim → unclaimed (fresh code next onboarding).
|
||||
if s.claimEngine != nil {
|
||||
if s.claimEngine != nil && cfg != nil {
|
||||
if cerr := s.claimEngine.ResetToUnclaimed(cfg); cerr != nil {
|
||||
_ = s.store.UpdateResetLeg(resetID, "claim", "failed")
|
||||
s.logger.Printf("[ERROR] reset %s: claim reset failed: %v", customerID, cerr)
|
||||
@@ -244,19 +260,25 @@ func (s *Server) commitCustomerReset(ctx context.Context, cfg *store.CustomerCon
|
||||
|
||||
// Clear the provisioned offsite descriptor (keep the tier CHOICE, drop provisioned host/user/repo/
|
||||
// fingerprint) and re-save → ConfigVersion bump. Identity + basic config survive intact.
|
||||
newConfigJSON, cerr := offsite.ClearProvisionedDescriptor(cfg.ConfigJSON)
|
||||
if cerr != nil {
|
||||
_ = s.store.UpdateResetLeg(resetID, "descriptor", "failed")
|
||||
s.logger.Printf("[ERROR] reset %s: clear offsite descriptor: %v", customerID, cerr)
|
||||
return &resetLegError{Leg: "descriptor", Status: http.StatusInternalServerError, Err: cerr, Msg: "Internal error"}
|
||||
// Ghost customers have no row to clear OR to re-save — re-saving here would RESURRECT the very
|
||||
// record the cascade is deleting, so the leg is skipped, not "made to work".
|
||||
if cfg == nil {
|
||||
_ = s.store.UpdateResetLeg(resetID, "descriptor", "skipped_no_config")
|
||||
} else {
|
||||
newConfigJSON, cerr := offsite.ClearProvisionedDescriptor(cfg.ConfigJSON)
|
||||
if cerr != nil {
|
||||
_ = s.store.UpdateResetLeg(resetID, "descriptor", "failed")
|
||||
s.logger.Printf("[ERROR] reset %s: clear offsite descriptor: %v", customerID, cerr)
|
||||
return &resetLegError{Leg: "descriptor", Status: http.StatusInternalServerError, Err: cerr, Msg: "Internal error"}
|
||||
}
|
||||
cfg.ConfigJSON = newConfigJSON
|
||||
if serr := s.store.SaveCustomerConfig(cfg); serr != nil {
|
||||
_ = s.store.UpdateResetLeg(resetID, "descriptor", "failed")
|
||||
s.logger.Printf("[ERROR] reset %s: save cleared config: %v", customerID, serr)
|
||||
return &resetLegError{Leg: "descriptor", Status: http.StatusInternalServerError, Err: serr, Msg: "Internal error"}
|
||||
}
|
||||
_ = s.store.UpdateResetLeg(resetID, "descriptor", "ok")
|
||||
}
|
||||
cfg.ConfigJSON = newConfigJSON
|
||||
if serr := s.store.SaveCustomerConfig(cfg); serr != nil {
|
||||
_ = s.store.UpdateResetLeg(resetID, "descriptor", "failed")
|
||||
s.logger.Printf("[ERROR] reset %s: save cleared config: %v", customerID, serr)
|
||||
return &resetLegError{Leg: "descriptor", Status: http.StatusInternalServerError, Err: serr, Msg: "Internal error"}
|
||||
}
|
||||
_ = s.store.UpdateResetLeg(resetID, "descriptor", "ok")
|
||||
|
||||
// DB purge LAST: retained escrow (ack-gated), one-time secret, DR recipe, log bundles.
|
||||
if perr := s.store.PurgeCustomerResetDBState(customerID, purgeEscrow); perr != nil {
|
||||
|
||||
@@ -893,13 +893,23 @@
|
||||
if (d.dr_recipe_present) dies.push('DR recipe');
|
||||
if (d.one_time_secret) dies.push('one-time password');
|
||||
if (d.claim_present) dies.push('claim state');
|
||||
if (d.residue && d.residue.reports) dies.push(d.residue.reports + ' report row(s)');
|
||||
if (d.residue && d.residue.app_telemetry) dies.push(d.residue.app_telemetry + ' app-telemetry row(s)');
|
||||
if (d.residue && d.residue.appliance_registrations) dies.push('appliance registration (token)');
|
||||
if (d.residue && d.residue.selfbind_tokens) dies.push('self-bind token(s)');
|
||||
if (d.residue && d.residue.notification_prefs) dies.push('notification preferences');
|
||||
dies.push('customer record');
|
||||
var custody = d.superseded_blobs > 0
|
||||
? d.superseded_blobs + ' retained escrow blob(s) + every current host escrow'
|
||||
: 'every current host escrow';
|
||||
inv.innerHTML = '<strong>Will be destroyed:</strong> ' + dies.join(', ') +
|
||||
'. <strong>Custody:</strong> ' + custody + ' (purged in the final leg). ' +
|
||||
'<strong>Survives:</strong> the audit event stream and the deletion provenance.';
|
||||
'<strong>Survives:</strong> the audit event stream, the notification log and the deletion provenance.';
|
||||
if (d.has_config === false) {
|
||||
inv.innerHTML = '<strong style="color: var(--warn)">Ghost customer:</strong> the configuration record ' +
|
||||
'is already gone, but ' + d.residue_total + ' row(s) of report/telemetry state keep it on the ' +
|
||||
'Customers list and keep it raising staleness alerts. This clears it.<br>' + inv.innerHTML;
|
||||
}
|
||||
if (d.online_host_present) {
|
||||
inv.innerHTML += '<br><strong style="color: var(--crit)">Refused:</strong> a host is ONLINE. ' +
|
||||
'Decommission the box first — the cascade never deletes a live host.';
|
||||
|
||||
Reference in New Issue
Block a user