package web // Customer DELETE cascade (v0.69.0, R-25b — operator ruling 2026-07-21). // // Before v0.69.0 the customer page carried two half-truths: RESET was the real teardown but REFUSED // while any host row existed, and DELETE quietly removed only the customer_configs row (plus escrow // custody) — leaving the Hetzner Storage-Box repo, the PBS namespace + credentials, the tunnel/zone // plumbing and the host rows themselves behind. The ruling makes DELETE what its name promises: ONE // guided flow that shows exactly what exists, takes THREE explicit acknowledgements plus the typed // customer-id, then runs the full teardown in the safe order: // // leg 1 hosts — every host row deleted through the SAME service path as a manual host delete // (ONLINE refuses; escrow is DEMOTED to retained custody, never destroyed) // leg 2 reset — the committed RESET sequence verbatim (Hetzner FIRST, PBS, claim, descriptor, // DB purge) via commitCustomerReset — with purgeEscrow=FALSE, see below // leg 3 residue — the report stream + credential-bearing bindings (v0.70.0): reports, app // telemetry/log tails, notification prefs, self-bind tokens, appliance // registrations. Without this leg a deleted customer stays on the Customers list // and keeps raising staleness alerts, because GetCustomers() is report-derived // leg 4 purge — DeleteCustomerConfig: the customer record AND all escrow ciphertext // // Nothing here is newly destructive: the cascade only SEQUENCES operations that already exist (plus // the v0.70.0 residue sweep), each keeping its own safety rules. Two invariants are load-bearing: // // - Ruling 3 is preserved BY CONSTRUCTION: leg 2 can only run after leg 1, so the RESET sequence // never sees a host row. The standalone RESET handler's 409 gate is untouched. // - Custody is purged EXACTLY ONCE, in leg 4. Leg 1 demotes (host_escrow → host_escrow_superseded); // leg 2 is called with purgeEscrow=false so PurgeCustomerResetDBState leaves the retained blobs // alone; leg 4's DeleteCustomerConfig is the one true purge point (v0.60.1). // // A leg that fails leaves the journal row retained and the error names the leg. A re-run resumes: // leg 1 is a no-op once the hosts are gone, and every leg of the RESET sequence is idempotent. import ( "context" "encoding/json" "net/http" "strconv" "strings" "time" ) // deleteCascadeAcks is the three-acknowledgement gate. Every one is REQUIRED — there is no force or // skip flag anywhere in this file (a missing ack is a refusal, never a downgrade to a partial run). type deleteCascadeAcks struct { Hosts bool // "N host(s) will be deleted — recovery-key custody is demoted, not destroyed" Reset bool // "the customer will be RESET — offsite repo DESTROYED, PBS revoked, tunnel/zone removed" Purge bool // "the customer record and ALL escrow ciphertext are PURGED — unrecoverable" } func readDeleteCascadeAcks(r *http.Request) deleteCascadeAcks { return deleteCascadeAcks{ Hosts: r.FormValue("ack_hosts") == "1", Reset: r.FormValue("ack_reset") == "1", Purge: r.FormValue("ack_purge") == "1", } } func (a deleteCascadeAcks) complete() bool { return a.Hosts && a.Reset && a.Purge } // handleCustomerDeletePreview — GET /configs/{id}/delete. The read-only inventory the guided dialog // renders: the RESET inventory EXTENDED with the host list and the residue counts (ruling 4 applied // to every leg). // Counts, names and booleans only — never a secret, blob or key. Also surfaces an incomplete journal // row so the dialog can offer "Resume". func (s *Server) handleCustomerDeletePreview(w http.ResponseWriter, r *http.Request, customerID string) { cfg, err := s.store.GetCustomerConfig(customerID) if err != nil { s.logger.Printf("[ERROR] delete preview %s: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) return } inv, err := s.store.CustomerResetInventory(customerID) if err != nil { s.logger.Printf("[ERROR] delete preview %s: inventory: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) return } hosts, err := s.store.ListHostsByCustomer(customerID) if err != nil { s.logger.Printf("[ERROR] delete preview %s: hosts: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) return } residue, err := s.store.CustomerResidue(customerID) if err != nil { s.logger.Printf("[ERROR] delete preview %s: residue: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) return } // Ghost customers (config row already gone, residue alive) MUST preview — the dialog is the only // surface that can clear them. 404 only when there is genuinely nothing left. if cfg == nil && len(hosts) == 0 && residue.Total() == 0 { http.NotFound(w, r) return } hostRows := make([]map[string]any, 0, len(hosts)) onlineBlocked := false for i := range hosts { status := s.hostStatus(hosts[i].LastReportAt) if status == "ok" { onlineBlocked = true } hostRows = append(hostRows, map[string]any{ "host_id": hosts[i].HostID, "status": status, "online": status == "ok", }) } offsiteEnabled, offsiteType := false, "" if cfg != nil { offsiteEnabled, offsiteType = offsiteChoice(cfg.ConfigJSON) } offsiteName := "" if offsiteEnabled && s.offsite != nil { ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second) defer cancel() if n, oerr := s.offsite.OffsiteIdentifier(ctx, customerID, offsiteType); oerr != nil { s.logger.Printf("[WARN] delete preview %s: offsite identifier lookup: %v", customerID, oerr) } else { offsiteName = n } } customerName := "" if cfg != nil { customerName = cfg.CustomerName } // An incomplete journal row = a cascade that stopped mid-way; the dialog renders it + Resume. var pending map[string]any if cr, jerr := s.store.LatestCustomerReset(customerID); jerr != nil { s.logger.Printf("[WARN] delete preview %s: journal read: %v", customerID, jerr) } else if cr != nil && cr.CompletedAt == nil { pending = map[string]any{ "id": cr.ID, "started_at": cr.StartedAt.UTC().Format(time.RFC3339), "legs": cr.Legs, } } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "customer_id": customerID, "customer_name": customerName, "has_config": cfg != nil, // false = GHOST (config already gone, residue alive) "hosts": hostRows, "host_count": inv.HostCount, "online_host_present": onlineBlocked, // leg 1 refuses; decommission the agent first "superseded_blobs": inv.SupersededBlobs, "dr_recipe_present": inv.DRRecipePresent, "one_time_secret": inv.OneTimeSecretPresent, "claim_present": inv.ClaimPresent, "offsite_enabled": offsiteEnabled, "offsite_type": offsiteType, "offsite_identifier": offsiteName, "pbs_tenancy_configured": s.tenantsync != nil, "pending_journal": pending, "residue_total": residue.Total(), "residue": map[string]int{ "reports": residue.Reports, "app_telemetry": residue.AppTelemetry, "app_log_tails": residue.AppLogTails, "log_tail_requests": residue.LogTailRequests, "notification_prefs": residue.NotificationPrefs, "selfbind_tokens": residue.SelfBindTokens, "appliance_registrations": residue.ApplianceRegistrations, }, }) } // handleCustomerDelete — POST /configs/{id}/delete. The guided full-teardown cascade. EVERY gate is // checked before ANY write or external call: a refused delete leaves ZERO side effects (no host // deleted, no journal row opened, no external call made, no config row touched). func (s *Server) handleCustomerDelete(w http.ResponseWriter, r *http.Request, customerID string) { cfg, err := s.store.GetCustomerConfig(customerID) if err != nil { s.logger.Printf("[ERROR] delete %s: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) return } hosts, err := s.store.ListHostsByCustomer(customerID) if err != nil { s.logger.Printf("[ERROR] delete %s: hosts: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) return } residue, err := s.store.CustomerResidue(customerID) if err != nil { s.logger.Printf("[ERROR] delete %s: residue: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) return } // 404 means "there is nothing here", not "there is no config row". A GHOST customer — config // already deleted but the report stream (and therefore the Customers list, and therefore the // staleness alerts) still alive — MUST be deletable; before v0.70.0 it 404'd and no operator // surface could clear it. if cfg == nil && len(hosts) == 0 && residue.Total() == 0 { http.NotFound(w, r) return } // ── Gates (all before any write) ──────────────────────────────────────────────────────────── acks := readDeleteCascadeAcks(r) if !acks.complete() { s.logger.Printf("[WARN] delete %s REFUSED: acknowledgements incomplete (hosts=%t reset=%t purge=%t)", customerID, acks.Hosts, acks.Reset, acks.Purge) http.Error(w, "Delete refused: all three acknowledgements are required — nothing was deleted.", http.StatusBadRequest) return } if strings.TrimSpace(r.FormValue("confirm_id")) != customerID { s.logger.Printf("[WARN] delete %s REFUSED: typed customer-id mismatch", customerID) http.Error(w, "Delete refused: the typed customer-id does not match — nothing was deleted.", http.StatusBadRequest) return } // Stale-preview gate: the operator acknowledged a specific host count. If the fleet changed // between opening the dialog and submitting, the acknowledgement no longer describes reality. if expect := strings.TrimSpace(r.FormValue("expect_hosts")); expect == "" || expect != strconv.Itoa(len(hosts)) { s.logger.Printf("[WARN] delete %s REFUSED: stale preview (acknowledged %q host(s), live %d)", customerID, expect, len(hosts)) http.Error(w, "Delete refused: the inventory changed since the dialog was opened — re-open it and confirm again. Nothing was deleted.", http.StatusConflict) return } // Leg 1 keeps host-delete's own safety rule: an ONLINE host is never deleted (a live agent would // receive 401s permanently). Checked for EVERY host up front, so the cascade never half-runs. for i := range hosts { if s.hostStatus(hosts[i].LastReportAt) == "ok" { s.logger.Printf("[WARN] delete %s REFUSED: host %s is ONLINE", customerID, hosts[i].HostID) http.Error(w, "Delete refused: host "+hosts[i].HostID+" is ONLINE. Decommission the box first — the cascade never deletes a live host.", http.StatusConflict) return } } // ── Committed. Detached ctx: once teardown starts it must run to a clean journal state ─────── ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 10*time.Minute) defer cancel() // escrow_acked=true on the journal row: ack #3 carries the ruling-1 custody-destruction // acknowledgement. The PURGE itself is leg 3's, not the RESET sequence's (see the file header). journalID, err := s.store.StartCustomerReset(customerID, true) if err != nil { s.logger.Printf("[ERROR] delete %s: open journal: %v", customerID, err) http.Error(w, "Internal error", http.StatusInternalServerError) return } s.logger.Printf("[INFO] customer DELETE cascade started for %s (journal #%d, %d host(s))", customerID, journalID, len(hosts)) // ── Leg 1: hosts (demotion, never destruction) ─────────────────────────────────────────────── for i := range hosts { hostID := hosts[i].HostID if derr := s.store.DeleteHost(hostID, true); derr != nil { _ = s.store.UpdateResetLeg(journalID, "hosts", "failed") s.logger.Printf("[ERROR] delete %s: host %s delete FAILED (journal #%d retained; re-run to resume): %v", customerID, hostID, journalID, derr) http.Error(w, "Delete incomplete at leg 1 (hosts): removing host "+hostID+" failed — nothing else was touched; re-run to resume. ("+derr.Error()+")", http.StatusInternalServerError) return } s.logger.Printf("[INFO] delete %s: host %s deleted (escrow DEMOTED to retained custody)", customerID, hostID) } _ = s.store.UpdateResetLeg(journalID, "hosts", "ok") // ── Leg 2: the committed RESET sequence (external teardown FIRST, DB purge last) ───────────── // purgeEscrow=false — retained custody dies exactly once, in leg 4. // cfg may be nil (ghost customer): the Hetzner + descriptor legs then record `skipped_no_config`. if lerr := s.commitCustomerReset(ctx, cfg, customerID, journalID, false); lerr != nil { s.logger.Printf("[ERROR] delete %s: cascade stopped at leg 2 (%s) — journal #%d retained", customerID, lerr.Leg, journalID) http.Error(w, "Delete incomplete at leg 2 (reset/"+lerr.Leg+"): "+lerr.Msg+" The host(s) are already deleted; re-run to resume.", lerr.Status) return } // ── Leg 3: residue — the report stream + the credential-bearing bindings ───────────────────── // This is what actually makes the customer DISAPPEAR: `GetCustomers()` builds the Customers list // (and the staleness/offsite checkers' work list) purely from `reports`, so before v0.70.0 a // fully deleted customer stayed visible AND kept emailing the operator. It runs BEFORE the record // purge on purpose — the customer_configs row is the identifying descriptor and goes LAST, and a // crash between the two legs leaves a ghost the cascade can now clean up on a re-run. if perr := s.store.PurgeCustomerResidue(customerID); perr != nil { _ = s.store.UpdateResetLeg(journalID, "residue", "failed") s.logger.Printf("[ERROR] delete %s: residue purge FAILED (journal #%d retained; re-run to resume): %v", customerID, journalID, perr) http.Error(w, "Delete incomplete at leg 3 (residue): the report stream and bindings could not be purged — the customer record is NOT yet removed; re-run to resume. ("+perr.Error()+")", http.StatusInternalServerError) return } _ = s.store.UpdateResetLeg(journalID, "residue", "ok") s.logger.Printf("[INFO] delete %s: residue purged (reports=%d app_telemetry=%d app_log_tails=%d log_tail_requests=%d notif_prefs=%d selfbind_tokens=%d appliance_registrations=%d)", customerID, residue.Reports, residue.AppTelemetry, residue.AppLogTails, residue.LogTailRequests, residue.NotificationPrefs, residue.SelfBindTokens, residue.ApplianceRegistrations) // ── Leg 4: the one true purge point — customer record + ALL escrow ciphertext ──────────────── if derr := s.store.DeleteCustomerConfig(customerID); derr != nil { _ = s.store.UpdateResetLeg(journalID, "customer_delete", "failed") s.logger.Printf("[ERROR] delete %s: final purge FAILED (journal #%d retained; re-run to resume): %v", customerID, journalID, derr) http.Error(w, "Delete incomplete at leg 4 (purge): the customer record could not be removed — re-run to resume. ("+derr.Error()+")", http.StatusInternalServerError) return } _ = s.store.UpdateResetLeg(journalID, "customer_delete", "ok") if ferr := s.store.FinishCustomerReset(journalID); ferr != nil { s.logger.Printf("[WARN] delete %s: journal finish stamp failed (state is complete): %v", customerID, ferr) } // Audit event SURVIVES the delete (events are never wiped — the audit trail outlives every // lifecycle tier, and the customer_configs row is gone by now, which is fine: events are keyed // by customer_id, not by a foreign key). msg := "Ügyfél TÖRLÉSE (teljes lebontás): host(ok) törölve, offsite tároló és PBS névtér megsemmisítve, majd az ügyfélrekord és a teljes helyreállítási-kulcs letét véglegesen törölve. Visszafordíthatatlan." if _, eerr := s.store.SaveEvent(customerID, "customer_deleted", "critical", msg, "", "hub"); eerr != nil { s.logger.Printf("[WARN] delete %s: save audit event: %v", customerID, eerr) } s.logger.Printf("[INFO] customer DELETE cascade COMPLETE for %s (journal #%d) — full teardown", customerID, journalID) s.bumpIntent(customerID) // Direction-2: wake any still-holding wait so it completes promptly http.Redirect(w, r, "/configs?flash=deleted", http.StatusSeeOther) }