Files
felhom.eu/hub/internal/web/customer_delete.go
T
admin 9b3381be0a feat(hub): v0.70.0 — a deleted customer actually disappears (residue leg + ghost cleanup)
Found validating v0.69.0 against the live hub. demo-vm-felhom was deleted
on 07-18 and was still on the Customers list AND still raising offsite_stale
(10 events, latest 07-21 17:34, operator email at 19:34) — because
GetCustomers() is report-derived and no lifecycle tier ever deleted a report.

New leg 3 (residue), before the record purge: reports, app_telemetry,
app_log_tails, log_tail_requests, customer_notifications, plus the
credential-bearing appliance_registrations and selfbind_tokens. Audit
(events, notification_log) and F-14 provenance still survive.

Ghost customers are now deletable: 404 means "nothing here", not "no config
row". With no config row the offsite descriptor is unknowable, so the Hetzner
and descriptor legs record skipped_no_config rather than a bare "skipped".

Two more red-proofs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J55BQE1gE2V4ffud5jweGS
2026-07-21 20:28:06 +02:00

307 lines
16 KiB
Go

package web
// Customer DELETE cascade (v0.69.0, R-25b — operator ruling 2026-07-21).
//
// Before v0.69.0 the customer page carried two half-truths: RESET was the real teardown but REFUSED
// while any host row existed, and DELETE quietly removed only the customer_configs row (plus escrow
// custody) — leaving the Hetzner Storage-Box repo, the PBS namespace + credentials, the tunnel/zone
// plumbing and the host rows themselves behind. The ruling makes DELETE what its name promises: ONE
// guided flow that shows exactly what exists, takes THREE explicit acknowledgements plus the typed
// customer-id, then runs the full teardown in the safe order:
//
// leg 1 hosts — every host row deleted through the SAME service path as a manual host delete
// (ONLINE refuses; escrow is DEMOTED to retained custody, never destroyed)
// leg 2 reset — the committed RESET sequence verbatim (Hetzner FIRST, PBS, claim, descriptor,
// DB purge) via commitCustomerReset — with purgeEscrow=FALSE, see below
// leg 3 residue — the report stream + credential-bearing bindings (v0.70.0): reports, app
// telemetry/log tails, notification prefs, self-bind tokens, appliance
// registrations. Without this leg a deleted customer stays on the Customers list
// and keeps raising staleness alerts, because GetCustomers() is report-derived
// leg 4 purge — DeleteCustomerConfig: the customer record AND all escrow ciphertext
//
// Nothing here is newly destructive: the cascade only SEQUENCES operations that already exist (plus
// the v0.70.0 residue sweep), each keeping its own safety rules. Two invariants are load-bearing:
//
// - Ruling 3 is preserved BY CONSTRUCTION: leg 2 can only run after leg 1, so the RESET sequence
// never sees a host row. The standalone RESET handler's 409 gate is untouched.
// - Custody is purged EXACTLY ONCE, in leg 4. Leg 1 demotes (host_escrow → host_escrow_superseded);
// leg 2 is called with purgeEscrow=false so PurgeCustomerResetDBState leaves the retained blobs
// alone; leg 4's DeleteCustomerConfig is the one true purge point (v0.60.1).
//
// A leg that fails leaves the journal row retained and the error names the leg. A re-run resumes:
// leg 1 is a no-op once the hosts are gone, and every leg of the RESET sequence is idempotent.
import (
"context"
"encoding/json"
"net/http"
"strconv"
"strings"
"time"
)
// deleteCascadeAcks is the three-acknowledgement gate. Every one is REQUIRED — there is no force or
// skip flag anywhere in this file (a missing ack is a refusal, never a downgrade to a partial run).
type deleteCascadeAcks struct {
Hosts bool // "N host(s) will be deleted — recovery-key custody is demoted, not destroyed"
Reset bool // "the customer will be RESET — offsite repo DESTROYED, PBS revoked, tunnel/zone removed"
Purge bool // "the customer record and ALL escrow ciphertext are PURGED — unrecoverable"
}
func readDeleteCascadeAcks(r *http.Request) deleteCascadeAcks {
return deleteCascadeAcks{
Hosts: r.FormValue("ack_hosts") == "1",
Reset: r.FormValue("ack_reset") == "1",
Purge: r.FormValue("ack_purge") == "1",
}
}
func (a deleteCascadeAcks) complete() bool { return a.Hosts && a.Reset && a.Purge }
// handleCustomerDeletePreview — GET /configs/{id}/delete. The read-only inventory the guided dialog
// renders: the RESET inventory EXTENDED with the host list and the residue counts (ruling 4 applied
// to every leg).
// Counts, names and booleans only — never a secret, blob or key. Also surfaces an incomplete journal
// row so the dialog can offer "Resume".
func (s *Server) handleCustomerDeletePreview(w http.ResponseWriter, r *http.Request, customerID string) {
cfg, err := s.store.GetCustomerConfig(customerID)
if err != nil {
s.logger.Printf("[ERROR] delete preview %s: %v", customerID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
inv, err := s.store.CustomerResetInventory(customerID)
if err != nil {
s.logger.Printf("[ERROR] delete preview %s: inventory: %v", customerID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
hosts, err := s.store.ListHostsByCustomer(customerID)
if err != nil {
s.logger.Printf("[ERROR] delete preview %s: hosts: %v", customerID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
residue, err := s.store.CustomerResidue(customerID)
if err != nil {
s.logger.Printf("[ERROR] delete preview %s: residue: %v", customerID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
// Ghost customers (config row already gone, residue alive) MUST preview — the dialog is the only
// surface that can clear them. 404 only when there is genuinely nothing left.
if cfg == nil && len(hosts) == 0 && residue.Total() == 0 {
http.NotFound(w, r)
return
}
hostRows := make([]map[string]any, 0, len(hosts))
onlineBlocked := false
for i := range hosts {
status := s.hostStatus(hosts[i].LastReportAt)
if status == "ok" {
onlineBlocked = true
}
hostRows = append(hostRows, map[string]any{
"host_id": hosts[i].HostID,
"status": status,
"online": status == "ok",
})
}
offsiteEnabled, offsiteType := false, ""
if cfg != nil {
offsiteEnabled, offsiteType = offsiteChoice(cfg.ConfigJSON)
}
offsiteName := ""
if offsiteEnabled && s.offsite != nil {
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
defer cancel()
if n, oerr := s.offsite.OffsiteIdentifier(ctx, customerID, offsiteType); oerr != nil {
s.logger.Printf("[WARN] delete preview %s: offsite identifier lookup: %v", customerID, oerr)
} else {
offsiteName = n
}
}
customerName := ""
if cfg != nil {
customerName = cfg.CustomerName
}
// An incomplete journal row = a cascade that stopped mid-way; the dialog renders it + Resume.
var pending map[string]any
if cr, jerr := s.store.LatestCustomerReset(customerID); jerr != nil {
s.logger.Printf("[WARN] delete preview %s: journal read: %v", customerID, jerr)
} else if cr != nil && cr.CompletedAt == nil {
pending = map[string]any{
"id": cr.ID,
"started_at": cr.StartedAt.UTC().Format(time.RFC3339),
"legs": cr.Legs,
}
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"customer_id": customerID,
"customer_name": customerName,
"has_config": cfg != nil, // false = GHOST (config already gone, residue alive)
"hosts": hostRows,
"host_count": inv.HostCount,
"online_host_present": onlineBlocked, // leg 1 refuses; decommission the agent first
"superseded_blobs": inv.SupersededBlobs,
"dr_recipe_present": inv.DRRecipePresent,
"one_time_secret": inv.OneTimeSecretPresent,
"claim_present": inv.ClaimPresent,
"offsite_enabled": offsiteEnabled,
"offsite_type": offsiteType,
"offsite_identifier": offsiteName,
"pbs_tenancy_configured": s.tenantsync != nil,
"pending_journal": pending,
"residue_total": residue.Total(),
"residue": map[string]int{
"reports": residue.Reports,
"app_telemetry": residue.AppTelemetry,
"app_log_tails": residue.AppLogTails,
"log_tail_requests": residue.LogTailRequests,
"notification_prefs": residue.NotificationPrefs,
"selfbind_tokens": residue.SelfBindTokens,
"appliance_registrations": residue.ApplianceRegistrations,
},
})
}
// handleCustomerDelete — POST /configs/{id}/delete. The guided full-teardown cascade. EVERY gate is
// checked before ANY write or external call: a refused delete leaves ZERO side effects (no host
// deleted, no journal row opened, no external call made, no config row touched).
func (s *Server) handleCustomerDelete(w http.ResponseWriter, r *http.Request, customerID string) {
cfg, err := s.store.GetCustomerConfig(customerID)
if err != nil {
s.logger.Printf("[ERROR] delete %s: %v", customerID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
hosts, err := s.store.ListHostsByCustomer(customerID)
if err != nil {
s.logger.Printf("[ERROR] delete %s: hosts: %v", customerID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
residue, err := s.store.CustomerResidue(customerID)
if err != nil {
s.logger.Printf("[ERROR] delete %s: residue: %v", customerID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
// 404 means "there is nothing here", not "there is no config row". A GHOST customer — config
// already deleted but the report stream (and therefore the Customers list, and therefore the
// staleness alerts) still alive — MUST be deletable; before v0.70.0 it 404'd and no operator
// surface could clear it.
if cfg == nil && len(hosts) == 0 && residue.Total() == 0 {
http.NotFound(w, r)
return
}
// ── Gates (all before any write) ────────────────────────────────────────────────────────────
acks := readDeleteCascadeAcks(r)
if !acks.complete() {
s.logger.Printf("[WARN] delete %s REFUSED: acknowledgements incomplete (hosts=%t reset=%t purge=%t)",
customerID, acks.Hosts, acks.Reset, acks.Purge)
http.Error(w, "Delete refused: all three acknowledgements are required — nothing was deleted.", http.StatusBadRequest)
return
}
if strings.TrimSpace(r.FormValue("confirm_id")) != customerID {
s.logger.Printf("[WARN] delete %s REFUSED: typed customer-id mismatch", customerID)
http.Error(w, "Delete refused: the typed customer-id does not match — nothing was deleted.", http.StatusBadRequest)
return
}
// Stale-preview gate: the operator acknowledged a specific host count. If the fleet changed
// between opening the dialog and submitting, the acknowledgement no longer describes reality.
if expect := strings.TrimSpace(r.FormValue("expect_hosts")); expect == "" || expect != strconv.Itoa(len(hosts)) {
s.logger.Printf("[WARN] delete %s REFUSED: stale preview (acknowledged %q host(s), live %d)", customerID, expect, len(hosts))
http.Error(w, "Delete refused: the inventory changed since the dialog was opened — re-open it and confirm again. Nothing was deleted.", http.StatusConflict)
return
}
// Leg 1 keeps host-delete's own safety rule: an ONLINE host is never deleted (a live agent would
// receive 401s permanently). Checked for EVERY host up front, so the cascade never half-runs.
for i := range hosts {
if s.hostStatus(hosts[i].LastReportAt) == "ok" {
s.logger.Printf("[WARN] delete %s REFUSED: host %s is ONLINE", customerID, hosts[i].HostID)
http.Error(w, "Delete refused: host "+hosts[i].HostID+" is ONLINE. Decommission the box first — the cascade never deletes a live host.", http.StatusConflict)
return
}
}
// ── Committed. Detached ctx: once teardown starts it must run to a clean journal state ───────
ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 10*time.Minute)
defer cancel()
// escrow_acked=true on the journal row: ack #3 carries the ruling-1 custody-destruction
// acknowledgement. The PURGE itself is leg 3's, not the RESET sequence's (see the file header).
journalID, err := s.store.StartCustomerReset(customerID, true)
if err != nil {
s.logger.Printf("[ERROR] delete %s: open journal: %v", customerID, err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
s.logger.Printf("[INFO] customer DELETE cascade started for %s (journal #%d, %d host(s))", customerID, journalID, len(hosts))
// ── Leg 1: hosts (demotion, never destruction) ───────────────────────────────────────────────
for i := range hosts {
hostID := hosts[i].HostID
if derr := s.store.DeleteHost(hostID, true); derr != nil {
_ = s.store.UpdateResetLeg(journalID, "hosts", "failed")
s.logger.Printf("[ERROR] delete %s: host %s delete FAILED (journal #%d retained; re-run to resume): %v", customerID, hostID, journalID, derr)
http.Error(w, "Delete incomplete at leg 1 (hosts): removing host "+hostID+" failed — nothing else was touched; re-run to resume. ("+derr.Error()+")", http.StatusInternalServerError)
return
}
s.logger.Printf("[INFO] delete %s: host %s deleted (escrow DEMOTED to retained custody)", customerID, hostID)
}
_ = s.store.UpdateResetLeg(journalID, "hosts", "ok")
// ── Leg 2: the committed RESET sequence (external teardown FIRST, DB purge last) ─────────────
// purgeEscrow=false — retained custody dies exactly once, in leg 4.
// cfg may be nil (ghost customer): the Hetzner + descriptor legs then record `skipped_no_config`.
if lerr := s.commitCustomerReset(ctx, cfg, customerID, journalID, false); lerr != nil {
s.logger.Printf("[ERROR] delete %s: cascade stopped at leg 2 (%s) — journal #%d retained", customerID, lerr.Leg, journalID)
http.Error(w, "Delete incomplete at leg 2 (reset/"+lerr.Leg+"): "+lerr.Msg+" The host(s) are already deleted; re-run to resume.", lerr.Status)
return
}
// ── Leg 3: residue — the report stream + the credential-bearing bindings ─────────────────────
// This is what actually makes the customer DISAPPEAR: `GetCustomers()` builds the Customers list
// (and the staleness/offsite checkers' work list) purely from `reports`, so before v0.70.0 a
// fully deleted customer stayed visible AND kept emailing the operator. It runs BEFORE the record
// purge on purpose — the customer_configs row is the identifying descriptor and goes LAST, and a
// crash between the two legs leaves a ghost the cascade can now clean up on a re-run.
if perr := s.store.PurgeCustomerResidue(customerID); perr != nil {
_ = s.store.UpdateResetLeg(journalID, "residue", "failed")
s.logger.Printf("[ERROR] delete %s: residue purge FAILED (journal #%d retained; re-run to resume): %v", customerID, journalID, perr)
http.Error(w, "Delete incomplete at leg 3 (residue): the report stream and bindings could not be purged — the customer record is NOT yet removed; re-run to resume. ("+perr.Error()+")", http.StatusInternalServerError)
return
}
_ = s.store.UpdateResetLeg(journalID, "residue", "ok")
s.logger.Printf("[INFO] delete %s: residue purged (reports=%d app_telemetry=%d app_log_tails=%d log_tail_requests=%d notif_prefs=%d selfbind_tokens=%d appliance_registrations=%d)",
customerID, residue.Reports, residue.AppTelemetry, residue.AppLogTails, residue.LogTailRequests,
residue.NotificationPrefs, residue.SelfBindTokens, residue.ApplianceRegistrations)
// ── Leg 4: the one true purge point — customer record + ALL escrow ciphertext ────────────────
if derr := s.store.DeleteCustomerConfig(customerID); derr != nil {
_ = s.store.UpdateResetLeg(journalID, "customer_delete", "failed")
s.logger.Printf("[ERROR] delete %s: final purge FAILED (journal #%d retained; re-run to resume): %v", customerID, journalID, derr)
http.Error(w, "Delete incomplete at leg 4 (purge): the customer record could not be removed — re-run to resume. ("+derr.Error()+")", http.StatusInternalServerError)
return
}
_ = s.store.UpdateResetLeg(journalID, "customer_delete", "ok")
if ferr := s.store.FinishCustomerReset(journalID); ferr != nil {
s.logger.Printf("[WARN] delete %s: journal finish stamp failed (state is complete): %v", customerID, ferr)
}
// Audit event SURVIVES the delete (events are never wiped — the audit trail outlives every
// lifecycle tier, and the customer_configs row is gone by now, which is fine: events are keyed
// by customer_id, not by a foreign key).
msg := "Ügyfél TÖRLÉSE (teljes lebontás): host(ok) törölve, offsite tároló és PBS névtér megsemmisítve, majd az ügyfélrekord és a teljes helyreállítási-kulcs letét véglegesen törölve. Visszafordíthatatlan."
if _, eerr := s.store.SaveEvent(customerID, "customer_deleted", "critical", msg, "", "hub"); eerr != nil {
s.logger.Printf("[WARN] delete %s: save audit event: %v", customerID, eerr)
}
s.logger.Printf("[INFO] customer DELETE cascade COMPLETE for %s (journal #%d) — full teardown", customerID, journalID)
s.bumpIntent(customerID) // Direction-2: wake any still-holding wait so it completes promptly
http.Redirect(w, r, "/configs?flash=deleted", http.StatusSeeOther)
}