feat(hub): v0.70.0 — a deleted customer actually disappears (residue leg + ghost cleanup)

Found validating v0.69.0 against the live hub. demo-vm-felhom was deleted
on 07-18 and was still on the Customers list AND still raising offsite_stale
(10 events, latest 07-21 17:34, operator email at 19:34) — because
GetCustomers() is report-derived and no lifecycle tier ever deleted a report.

New leg 3 (residue), before the record purge: reports, app_telemetry,
app_log_tails, log_tail_requests, customer_notifications, plus the
credential-bearing appliance_registrations and selfbind_tokens. Audit
(events, notification_log) and F-14 provenance still survive.

Ghost customers are now deletable: 404 means "nothing here", not "no config
row". With no config row the offsite descriptor is unknowable, so the Hetzner
and descriptor legs record skipped_no_config rather than a bare "skipped".

Two more red-proofs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J55BQE1gE2V4ffud5jweGS
This commit is contained in:
2026-07-21 20:28:06 +02:00
parent a1d503be98
commit 9b3381be0a
13 changed files with 521 additions and 45 deletions
+40 -18
View File
@@ -137,7 +137,7 @@ func (s *Server) handleCustomerReset(w http.ResponseWriter, r *http.Request, cus
s.logger.Printf("[INFO] customer RESET started for %s (journal #%d, escrow_ack=%t)", customerID, resetID, escrowAck)
// Standalone RESET purges the retained custody itself, gated by the ack it just checked.
if lerr := s.commitCustomerReset(ctx, cfg, resetID, escrowAck); lerr != nil {
if lerr := s.commitCustomerReset(ctx, cfg, customerID, resetID, escrowAck); lerr != nil {
http.Error(w, lerr.Msg, lerr.Status)
return
}
@@ -193,12 +193,28 @@ func (e *resetLegError) Error() string {
//
// Behaviour for the standalone caller is byte-identical to v0.68.1 (same order, same leg names, same
// messages, same status codes).
func (s *Server) commitCustomerReset(ctx context.Context, cfg *store.CustomerConfig, resetID int64, purgeEscrow bool) *resetLegError {
customerID := cfg.CustomerID
//
// GHOST CUSTOMERS (v0.70.0): cfg may be nil — the DELETE cascade also runs against a customer whose
// config row is already gone but whose residue is not (a pre-v0.70.0 delete leaves the report stream
// behind; see store/customer_delete.go). The standalone RESET handler 404s on a nil config before it
// ever gets here, so this path is cascade-only. With no config row the offsite DESCRIPTOR is
// unknowable, so the Hetzner leg is skipped and SAYS SO in the journal (`skipped_no_config`) rather
// than silently reporting "skipped"; PBS is customer-id-keyed and idempotent, so it still runs.
func (s *Server) commitCustomerReset(ctx context.Context, cfg *store.CustomerConfig, customerID string, resetID int64, purgeEscrow bool) *resetLegError {
if cfg != nil {
customerID = cfg.CustomerID
}
// Leg: Hetzner offsite (repo DATA destroyed). Only when the customer chose an offsite tier.
offsiteEnabled, offsiteType := offsiteChoice(cfg.ConfigJSON)
if offsiteEnabled && s.offsite != nil {
offsiteEnabled, offsiteType := false, ""
if cfg != nil {
offsiteEnabled, offsiteType = offsiteChoice(cfg.ConfigJSON)
}
if cfg == nil {
// No descriptor to read — never guess a tier, and never let the journal imply "nothing to do".
s.logger.Printf("[WARN] %s: no config row — the offsite (Hetzner) teardown CANNOT be determined and is SKIPPED; verify the Hetzner side by hand", customerID)
_ = s.store.UpdateResetLeg(resetID, "hetzner", "skipped_no_config")
} else if offsiteEnabled && s.offsite != nil {
if derr := s.offsite.Deprovision(ctx, customerID, offsiteType); derr != nil {
_ = s.store.UpdateResetLeg(resetID, "hetzner", "failed")
s.logger.Printf("[ERROR] reset %s: hetzner deprovision FAILED (journal #%d retained; re-run to resume): %v", customerID, resetID, derr)
@@ -228,7 +244,7 @@ func (s *Server) commitCustomerReset(ctx context.Context, cfg *store.CustomerCon
// All external legs are ok — now the DB side (publish-last, one leg at a time so the journal
// records where a mid-purge crash stopped). Claim → unclaimed (fresh code next onboarding).
if s.claimEngine != nil {
if s.claimEngine != nil && cfg != nil {
if cerr := s.claimEngine.ResetToUnclaimed(cfg); cerr != nil {
_ = s.store.UpdateResetLeg(resetID, "claim", "failed")
s.logger.Printf("[ERROR] reset %s: claim reset failed: %v", customerID, cerr)
@@ -244,19 +260,25 @@ func (s *Server) commitCustomerReset(ctx context.Context, cfg *store.CustomerCon
// Clear the provisioned offsite descriptor (keep the tier CHOICE, drop provisioned host/user/repo/
// fingerprint) and re-save → ConfigVersion bump. Identity + basic config survive intact.
newConfigJSON, cerr := offsite.ClearProvisionedDescriptor(cfg.ConfigJSON)
if cerr != nil {
_ = s.store.UpdateResetLeg(resetID, "descriptor", "failed")
s.logger.Printf("[ERROR] reset %s: clear offsite descriptor: %v", customerID, cerr)
return &resetLegError{Leg: "descriptor", Status: http.StatusInternalServerError, Err: cerr, Msg: "Internal error"}
// Ghost customers have no row to clear OR to re-save — re-saving here would RESURRECT the very
// record the cascade is deleting, so the leg is skipped, not "made to work".
if cfg == nil {
_ = s.store.UpdateResetLeg(resetID, "descriptor", "skipped_no_config")
} else {
newConfigJSON, cerr := offsite.ClearProvisionedDescriptor(cfg.ConfigJSON)
if cerr != nil {
_ = s.store.UpdateResetLeg(resetID, "descriptor", "failed")
s.logger.Printf("[ERROR] reset %s: clear offsite descriptor: %v", customerID, cerr)
return &resetLegError{Leg: "descriptor", Status: http.StatusInternalServerError, Err: cerr, Msg: "Internal error"}
}
cfg.ConfigJSON = newConfigJSON
if serr := s.store.SaveCustomerConfig(cfg); serr != nil {
_ = s.store.UpdateResetLeg(resetID, "descriptor", "failed")
s.logger.Printf("[ERROR] reset %s: save cleared config: %v", customerID, serr)
return &resetLegError{Leg: "descriptor", Status: http.StatusInternalServerError, Err: serr, Msg: "Internal error"}
}
_ = s.store.UpdateResetLeg(resetID, "descriptor", "ok")
}
cfg.ConfigJSON = newConfigJSON
if serr := s.store.SaveCustomerConfig(cfg); serr != nil {
_ = s.store.UpdateResetLeg(resetID, "descriptor", "failed")
s.logger.Printf("[ERROR] reset %s: save cleared config: %v", customerID, serr)
return &resetLegError{Leg: "descriptor", Status: http.StatusInternalServerError, Err: serr, Msg: "Internal error"}
}
_ = s.store.UpdateResetLeg(resetID, "descriptor", "ok")
// DB purge LAST: retained escrow (ack-gated), one-time secret, DR recipe, log bundles.
if perr := s.store.PurgeCustomerResetDBState(customerID, purgeEscrow); perr != nil {