feat(hub): v0.70.0 — a deleted customer actually disappears (residue leg + ghost cleanup)

Found validating v0.69.0 against the live hub. demo-vm-felhom was deleted on 07-18 and was still on the Customers list AND still raising offsite_stale (10 events, latest 07-21 17:34, operator email at 19:34) — because GetCustomers() is report-derived and no lifecycle tier ever deleted a report. New leg 3 (residue), before the record purge: reports, app_telemetry, app_log_tails, log_tail_requests, customer_notifications, plus the credential-bearing appliance_registrations and selfbind_tokens. Audit (events, notification_log) and F-14 provenance still survive. Ghost customers are now deletable: 404 means "nothing here", not "no config row". With no config row the offsite descriptor is unknowable, so the Hetzner and descriptor legs record skipped_no_config rather than a bare "skipped". Two more red-proofs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J55BQE1gE2V4ffud5jweGS
2026-07-21 20:28:06 +02:00
parent a1d503be98
commit 9b3381be0a
13 changed files with 521 additions and 45 deletions
@@ -379,7 +379,7 @@ func TestCommitCustomerReset_PurgeEscrowFlagGovernsCustody(t *testing.T) {
 	if err != nil {
 		t.Fatalf("journal: %v", err)
 	}
-	if lerr := s.commitCustomerReset(context.Background(), cfg, id, false); lerr != nil {
+	if lerr := s.commitCustomerReset(context.Background(), cfg, "acme", id, false); lerr != nil {
 		t.Fatalf("commitCustomerReset: %v", lerr)
 	}
 	if n := superseded(t, st, "acme"); n != 2 {
@@ -419,3 +419,169 @@ func TestDeleteCascadePreview_Inventory(t *testing.T) {
 		}
 	}
 }
+
+// ── Residue (v0.70.0): the leg that actually makes a deleted customer DISAPPEAR ─────────────────
+//
+// The v0.69.0 cascade left the report stream behind, and `GetCustomers()` builds the Customers list
+// (and the staleness/offsite checkers' work list) purely from `reports` — so a fully deleted
+// customer stayed visible AND kept emailing the operator. Observed live on `demo-vm-felhom`:
+// deleted 2026-07-18, still raising `offsite_stale` on 2026-07-21.
+
+// seedResidue adds the report-derived state + the credential-bearing bindings to a customer.
+func seedResidue(t *testing.T, st *store.Store, customerID string) {
+	t.Helper()
+	if err := st.SaveReport(customerID, []byte(`{"health":{"status":"ok"}}`)); err != nil {
+		t.Fatalf("seed report: %v", err)
+	}
+	if err := st.SaveAppTelemetry(customerID, time.Now(), []store.AppTelemetryRecord{
+		{AppName: "immich", DisplayName: "Immich", MemoryCurrentMB: 512},
+	}); err != nil {
+		t.Fatalf("seed telemetry: %v", err)
+	}
+	if err := st.SaveNotificationPrefs(customerID, "t@example.com", []string{"host_down"}, 6); err != nil {
+		t.Fatalf("seed notif prefs: %v", err)
+	}
+	if err := st.MintSelfBindToken(customerID, "tokenhash-"+customerID, time.Hour); err != nil {
+		t.Fatalf("seed selfbind token: %v", err)
+	}
+	// A DELIVERED appliance registration bound to the customer — credential-bearing (token_hash).
+	if _, _, err := st.RegisterAppliance("uuid-"+customerID, "aa:bb", "ssh-ed25519 AAAA", "{}", "apphash-"+customerID, ""); err != nil {
+		t.Fatalf("seed appliance: %v", err)
+	}
+	app, err := st.ApplianceByToken("apphash-" + customerID)
+	if err != nil || app == nil {
+		t.Fatalf("seed appliance lookup: %v (row=%v)", err, app != nil)
+	}
+	if err := st.BindAppliance(app.ID, customerID, "appliance", ""); err != nil {
+		t.Fatalf("bind appliance: %v", err)
+	}
+}
+
+func listedInCustomers(t *testing.T, st *store.Store, customerID string) bool {
+	t.Helper()
+	cs, err := st.GetCustomers()
+	if err != nil {
+		t.Fatalf("GetCustomers: %v", err)
+	}
+	for _, c := range cs {
+		if c.CustomerID == customerID {
+			return true
+		}
+	}
+	return false
+}
+
+// The cascade purges the residue, so the customer leaves the Customers list — and with it the
+// staleness/offsite checkers' work list. RED-PROOF: drop the residue leg and this FAILS with the
+// customer still listed and 1 report row alive.
+func TestDeleteCascade_PurgesResidueAndUnlistsCustomer(t *testing.T) {
+	s, st := newTestServer(t)
+	seedDeletable(t, st, "acme")
+	seedResidue(t, st, "acme")
+	s.SetTenantSync(&orderTenancy{})
+
+	if !listedInCustomers(t, st, "acme") {
+		t.Fatal("precondition: the customer must be listed before the cascade")
+	}
+	res, err := st.CustomerResidue("acme")
+	if err != nil || res.Total() == 0 {
+		t.Fatalf("precondition: residue must exist (err=%v, total=%d)", err, res.Total())
+	}
+
+	if rr := postDelete(t, s, "acme", cascadeForm("acme", 1)); rr.Code != http.StatusSeeOther {
+		t.Fatalf("status = %d, want 303: %s", rr.Code, rr.Body.String())
+	}
+
+	after, err := st.CustomerResidue("acme")
+	if err != nil {
+		t.Fatalf("residue: %v", err)
+	}
+	if after.Total() != 0 {
+		t.Errorf("residue after cascade = %+v, want all zero", *after)
+	}
+	if listedInCustomers(t, st, "acme") {
+		t.Error("the customer is STILL on the Customers list after a complete delete — the ghost that " +
+			"kept raising offsite_stale alerts for demo-vm-felhom")
+	}
+	// The credential-bearing rows are gone by NAME, not just by count.
+	if app, _ := st.ApplianceByToken("apphash-acme"); app != nil {
+		t.Error("the appliance registration (token_hash, status=delivered) outlived its customer")
+	}
+	if n, _ := st.CountSelfBindTokens("acme"); n != 0 {
+		t.Errorf("self-bind tokens = %d, want 0 — a live bind path to a deleted customer", n)
+	}
+	// Audit + provenance SURVIVE, exactly as in every other tier.
+	if evs, _ := st.GetRecentEvents("acme", 10); len(evs) == 0 {
+		t.Error("the audit event stream was purged — it must outlive every lifecycle tier")
+	}
+	if d, _ := st.LatestHostDeletion("acme"); d == nil {
+		t.Error("F-14 host-deletion provenance was purged — it must survive")
+	}
+	cr, _ := st.LatestCustomerReset("acme")
+	if cr == nil || cr.Legs["residue"] != "ok" || cr.Legs["customer_delete"] != "ok" {
+		t.Errorf("journal legs = %v, want residue=ok customer_delete=ok", cr)
+	}
+}
+
+// A GHOST — config row already gone (a pre-v0.70.0 delete), residue alive. Before v0.70.0 this
+// 404'd and NO operator surface could clear it. This is the demo-vm-felhom shape exactly.
+func TestDeleteCascade_GhostCustomerIsDeletable(t *testing.T) {
+	s, st := newTestServer(t)
+	seedDeletable(t, st, "ghost")
+	seedResidue(t, st, "ghost")
+	s.SetTenantSync(&orderTenancy{})
+	// Model the pre-v0.70.0 aftermath: hosts deleted, config row dropped, residue left behind.
+	if err := st.DeleteHost("ghost-01", true); err != nil {
+		t.Fatalf("delete host: %v", err)
+	}
+	if err := st.DeleteCustomerConfig("ghost"); err != nil {
+		t.Fatalf("drop config: %v", err)
+	}
+	if cfg, _ := st.GetCustomerConfig("ghost"); cfg != nil {
+		t.Fatal("precondition: the config row must be gone")
+	}
+	if !listedInCustomers(t, st, "ghost") {
+		t.Fatal("precondition: the ghost must still be listed (that IS the defect)")
+	}
+
+	// The preview must render it rather than 404 — it is the only surface that can clear a ghost.
+	req := httptest.NewRequest("GET", "/configs/ghost/delete", nil)
+	rr := httptest.NewRecorder()
+	s.handleCustomerDeletePreview(rr, req, "ghost")
+	if rr.Code != http.StatusOK {
+		t.Fatalf("ghost preview = %d, want 200: %s", rr.Code, rr.Body.String())
+	}
+	if !strings.Contains(rr.Body.String(), `"has_config":false`) {
+		t.Errorf("preview must mark the ghost (has_config=false): %s", rr.Body.String())
+	}
+
+	if rr := postDelete(t, s, "ghost", cascadeForm("ghost", 0)); rr.Code != http.StatusSeeOther {
+		t.Fatalf("ghost cascade = %d, want 303: %s", rr.Code, rr.Body.String())
+	}
+	if listedInCustomers(t, st, "ghost") {
+		t.Error("the ghost survived its own cleanup")
+	}
+	if res, _ := st.CustomerResidue("ghost"); res.Total() != 0 {
+		t.Errorf("ghost residue = %+v, want all zero", *res)
+	}
+	// With no config row the offsite descriptor is unknowable — the journal must SAY so, never
+	// record a bare "skipped" that reads as "there was nothing to do".
+	cr, _ := st.LatestCustomerReset("ghost")
+	if cr == nil || cr.Legs["hetzner"] != "skipped_no_config" || cr.Legs["descriptor"] != "skipped_no_config" {
+		t.Errorf("journal legs = %v, want hetzner/descriptor = skipped_no_config", cr)
+	}
+}
+
+// 404 still means "there is nothing here" — an id with no config, no host and no residue.
+func TestDeleteCascade_404WhenNothingRemains(t *testing.T) {
+	s, _ := newTestServer(t)
+	req := httptest.NewRequest("GET", "/configs/nobody/delete", nil)
+	rr := httptest.NewRecorder()
+	s.handleCustomerDeletePreview(rr, req, "nobody")
+	if rr.Code != http.StatusNotFound {
+		t.Errorf("preview for an empty id = %d, want 404", rr.Code)
+	}
+	if rr2 := postDelete(t, s, "nobody", cascadeForm("nobody", 0)); rr2.Code != http.StatusNotFound {
+		t.Errorf("cascade for an empty id = %d, want 404", rr2.Code)
+	}
+}