felhom.eu/hub/internal/web/customer_delete_test.go

package web

// Customer DELETE cascade (v0.69.0, R-25b) — the load-bearing contracts:
//
//	A) Happy cascade: the legs run in the ORDER hosts → reset → purge. Proven from inside leg 2 (the
//	   PBS deprovision callback): at that instant the host rows are ALREADY gone (leg 1 done) and the
//	   customer row is STILL there (leg 3 not started). Ruling 3 — RESET never runs while a host
//	   exists — is therefore preserved BY CONSTRUCTION and asserted, not merely commented.
//	B) Gates fail-closed: any missing ack / typed-id mismatch / stale host count / ONLINE host →
//	   4xx and ZERO mutations (no host deleted, no journal row, no external call, no config touched).
//	C) Resume: a leg-2 external failure retains the journal and names the leg; the hosts are already
//	   gone and the customer + custody SURVIVE; a re-run resumes and completes without re-demoting.
//	D) Standalone RESET is untouched (its own suite stays green; here: the cascade's purgeEscrow=false
//	   does not change what a standalone RESET purges).
//	E) Custody: leg 1 DEMOTES (never purges); the RESET leg with purgeEscrow=false leaves the retained
//	   blobs alone; the purge happens exactly once, in leg 3 (DeleteCustomerConfig).

import (
	"context"
	"errors"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strconv"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
	"gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync"
)

// errCascadeBoom is the injected external-teardown failure (scenario C).
var errCascadeBoom = errors.New("pbs unreachable")

// orderTenancy is a tenancyProvisioner that runs a callback at the exact moment leg 2's PBS
// deprovision fires — the observation point that proves the cascade's leg ORDER.
type orderTenancy struct {
	deprovisionCalls int
	err              error
	onDeprovision    func()
}

func (f *orderTenancy) Provision(ctx context.Context, customerID string) (*tenantsync.Result, error) {
	return nil, nil
}
func (f *orderTenancy) Reissue(ctx context.Context, customerID string) (*tenantsync.Result, error) {
	return nil, nil
}
func (f *orderTenancy) Deprovision(ctx context.Context, customerID string) (bool, error) {
	f.deprovisionCalls++
	if f.onDeprovision != nil {
		f.onDeprovision()
	}
	if f.err != nil {
		return false, f.err
	}
	return true, nil
}

// seedDeletable seeds a customer with a full footprint INCLUDING one OFFLINE host that carries a
// current escrow blob plus one already-retained (superseded) blob. Returns the host id.
func seedDeletable(t *testing.T, st *store.Store, customerID string) string {
	t.Helper()
	cfgJSON := `{"offsite":{"enabled":true,"type":"shared","host":"u1-sub3.your-storagebox.de","user":"u1-sub3","port":23,"repo_path":"/home/felhom","quota_gb":100,"host_fingerprint":"SHA256:abc"}}`
	if err := st.SaveCustomerConfig(&store.CustomerConfig{
		CustomerID: customerID, CustomerName: "Teszt", Domain: customerID + ".example",
		Email: "t@example.com", RetrievalPassword: "pw", APIKey: "capi-" + customerID, ConfigJSON: cfgJSON,
	}); err != nil {
		t.Fatalf("seed config: %v", err)
	}
	hostID := customerID + "-01"
	long := time.Now().Add(-72 * time.Hour) // far past the stale threshold → status "down", deletable
	if err := st.UpsertHost(&store.Host{HostID: hostID, CustomerID: customerID, APIKey: "hapi-" + hostID, LastReportAt: &long}); err != nil {
		t.Fatalf("seed host: %v", err)
	}
	if _, err := st.SaveHostEscrow(hostID, []byte("blobA"), "fpA", "posture", "2026-01-01T00:00:00Z", "shaA"); err != nil {
		t.Fatalf("seed escrow A: %v", err)
	}
	// A second save supersedes A → one RETAINED blob + one CURRENT blob before the cascade runs.
	if _, err := st.SaveHostEscrow(hostID, []byte("blobB"), "fpB", "posture", "2026-01-02T00:00:00Z", "shaB"); err != nil {
		t.Fatalf("seed escrow B: %v", err)
	}
	if err := st.SaveOneTimeSecret(customerID, "one-time-pw"); err != nil {
		t.Fatalf("seed one-time secret: %v", err)
	}
	if err := st.SaveDRRecipeHostHalf(customerID, hostID, 1, []byte("half")); err != nil {
		t.Fatalf("seed dr recipe: %v", err)
	}
	if _, err := st.RotateClaimCode(customerID, "$2a$10$hashhashhashhashhashha"); err != nil {
		t.Fatalf("seed claim: %v", err)
	}
	return hostID
}

func cascadeForm(customerID string, hostCount int) url.Values {
	return url.Values{
		"ack_hosts":    {"1"},
		"ack_reset":    {"1"},
		"ack_purge":    {"1"},
		"confirm_id":   {customerID},
		"expect_hosts": {strconv.Itoa(hostCount)},
	}
}

func postDelete(t *testing.T, s *Server, customerID string, form url.Values) *httptest.ResponseRecorder {
	t.Helper()
	req := httptest.NewRequest("POST", "/configs/"+customerID+"/delete", strings.NewReader(form.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	rr := httptest.NewRecorder()
	s.handleCustomerDelete(rr, req, customerID)
	return rr
}

func hostCount(t *testing.T, st *store.Store, customerID string) int {
	t.Helper()
	hosts, err := st.ListHostsByCustomer(customerID)
	if err != nil {
		t.Fatalf("list hosts: %v", err)
	}
	return len(hosts)
}

func configPresent(t *testing.T, st *store.Store, customerID string) bool {
	t.Helper()
	cfg, err := st.GetCustomerConfig(customerID)
	if err != nil {
		t.Fatalf("get config: %v", err)
	}
	return cfg != nil
}

// ── Scenario A: the happy cascade, and the leg ORDER it must run in ─────────────────────────────

func TestDeleteCascade_HappyPath_LegOrder(t *testing.T) {
	s, st := newTestServer(t)
	hostID := seedDeletable(t, st, "acme")

	var (
		hostsAtResetLeg   = -1
		configAtResetLeg  = false
		custodyAtResetLeg = -1
	)
	fake := &orderTenancy{onDeprovision: func() {
		hostsAtResetLeg = hostCount(t, st, "acme")
		configAtResetLeg = configPresent(t, st, "acme")
		custodyAtResetLeg = superseded(t, st, "acme")
	}}
	s.SetTenantSync(fake)

	rr := postDelete(t, s, "acme", cascadeForm("acme", 1))
	if rr.Code != http.StatusSeeOther {
		t.Fatalf("status = %d, want 303: %s", rr.Code, rr.Body.String())
	}
	if loc := rr.Header().Get("Location"); loc != "/configs?flash=deleted" {
		t.Errorf("Location = %q, want /configs?flash=deleted", loc)
	}

	// ORDER, observed from inside leg 2 — this is the assertion the whole cascade hangs on.
	if fake.deprovisionCalls != 1 {
		t.Fatalf("PBS deprovision calls = %d, want 1 (leg 2 must run)", fake.deprovisionCalls)
	}
	if hostsAtResetLeg != 0 {
		t.Errorf("at the RESET leg the customer still had %d host(s) — leg 1 must complete FIRST "+
			"(ruling 3: the RESET sequence never runs while a host row exists)", hostsAtResetLeg)
	}
	if !configAtResetLeg {
		t.Error("at the RESET leg the customer row was already gone — leg 3 must run LAST")
	}
	if custodyAtResetLeg == 0 {
		t.Error("at the RESET leg the retained custody was already gone — leg 1 DEMOTES, it must never purge")
	}

	// Final state: hosts gone, customer gone, ALL custody gone (leg 3 is the one true purge point).
	if n := hostCount(t, st, "acme"); n != 0 {
		t.Errorf("hosts after cascade = %d, want 0", n)
	}
	if configPresent(t, st, "acme") {
		t.Error("customer row survived the cascade")
	}
	if esc, err := st.GetHostEscrow(hostID); err != nil || esc != nil {
		t.Errorf("current escrow survived the cascade (err=%v, row=%v)", err, esc != nil)
	}
	if n := superseded(t, st, "acme"); n != 0 {
		t.Errorf("retained escrow blobs after cascade = %d, want 0 (leg 3 purges custody)", n)
	}

	// Journal: every leg stamped, completion stamped.
	cr, err := st.LatestCustomerReset("acme")
	if err != nil || cr == nil {
		t.Fatalf("journal: %v (row=%v)", err, cr != nil)
	}
	if cr.CompletedAt == nil {
		t.Error("journal not stamped complete")
	}
	for leg, want := range map[string]string{"hosts": "ok", "pbs": "ok", "db_purge": "ok", "customer_delete": "ok"} {
		if got := cr.Legs[leg]; got != want {
			t.Errorf("journal leg %q = %q, want %q (legs=%v)", leg, got, want, cr.Legs)
		}
	}

	// The audit event SURVIVES the customer row (events are keyed by id, never wiped).
	evs, err := st.GetRecentEvents("acme", 10)
	if err != nil {
		t.Fatalf("events: %v", err)
	}
	found := false
	for _, e := range evs {
		if e.EventType == "customer_deleted" {
			found = true
		}
	}
	if !found {
		t.Errorf("no customer_deleted audit event survived the cascade (events=%d)", len(evs))
	}
}

// ── Scenario B: every gate fails closed, with ZERO mutations ────────────────────────────────────

func TestDeleteCascade_GatesFailClosed(t *testing.T) {
	base := func() url.Values { return cascadeForm("acme", 1) }
	cases := []struct {
		name     string
		mutate   func(url.Values)
		online   bool
		wantCode int
	}{
		{"missing ack 1 (hosts)", func(f url.Values) { f.Del("ack_hosts") }, false, http.StatusBadRequest},
		{"missing ack 2 (reset)", func(f url.Values) { f.Del("ack_reset") }, false, http.StatusBadRequest},
		{"missing ack 3 (purge)", func(f url.Values) { f.Del("ack_purge") }, false, http.StatusBadRequest},
		{"ack sent as something other than 1", func(f url.Values) { f.Set("ack_purge", "yes") }, false, http.StatusBadRequest},
		{"typed id mismatch", func(f url.Values) { f.Set("confirm_id", "acm") }, false, http.StatusBadRequest},
		{"typed id absent", func(f url.Values) { f.Del("confirm_id") }, false, http.StatusBadRequest},
		{"stale preview (host count moved)", func(f url.Values) { f.Set("expect_hosts", "0") }, false, http.StatusConflict},
		{"stale preview (count absent)", func(f url.Values) { f.Del("expect_hosts") }, false, http.StatusConflict},
		{"ONLINE host", func(f url.Values) {}, true, http.StatusConflict},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			s, st := newTestServer(t)
			hostID := seedDeletable(t, st, "acme")
			if tc.online {
				// A just-saved report → status "ok" (online), the same way host-delete's own gate sees it.
				if err := st.SaveHostReport(hostID, "acme", []byte(`{}`), store.HostReportDenorm{}); err != nil {
					t.Fatalf("make host online: %v", err)
				}
			}
			fake := &orderTenancy{}
			s.SetTenantSync(fake)
			custodyBefore := superseded(t, st, "acme")

			f := base()
			tc.mutate(f)
			rr := postDelete(t, s, "acme", f)

			if rr.Code != tc.wantCode {
				t.Errorf("status = %d, want %d: %s", rr.Code, tc.wantCode, rr.Body.String())
			}
			// ZERO mutations — the whole point of a fail-closed gate.
			if n := hostCount(t, st, "acme"); n != 1 {
				t.Errorf("hosts = %d, want 1 (a refused delete deletes NOTHING)", n)
			}
			if !configPresent(t, st, "acme") {
				t.Error("customer row was deleted by a REFUSED delete")
			}
			if esc, _ := st.GetHostEscrow(hostID); esc == nil {
				t.Error("current escrow was touched by a REFUSED delete")
			}
			if n := superseded(t, st, "acme"); n != custodyBefore {
				t.Errorf("retained custody = %d, want %d (untouched)", n, custodyBefore)
			}
			if fake.deprovisionCalls != 0 {
				t.Errorf("PBS deprovision called %d time(s) on a REFUSED delete — no external call may fire", fake.deprovisionCalls)
			}
			if cr, _ := st.LatestCustomerReset("acme"); cr != nil {
				t.Errorf("a journal row was opened by a REFUSED delete (#%d) — gates run before any write", cr.ID)
			}
		})
	}
}

// ── Scenario C: a mid-cascade external failure is resumable ─────────────────────────────────────

func TestDeleteCascade_ResumesAfterExternalFailure(t *testing.T) {
	s, st := newTestServer(t)
	hostID := seedDeletable(t, st, "acme")
	fake := &orderTenancy{err: errCascadeBoom}
	s.SetTenantSync(fake)

	rr := postDelete(t, s, "acme", cascadeForm("acme", 1))
	if rr.Code != http.StatusBadGateway {
		t.Fatalf("status = %d, want 502: %s", rr.Code, rr.Body.String())
	}
	if body := rr.Body.String(); !strings.Contains(body, "leg 2") || !strings.Contains(body, "pbs") {
		t.Errorf("the error must NAME the failed leg, got %q", body)
	}
	// Leg 1 completed; legs 2-3 did not. The customer and ALL custody SURVIVE.
	if n := hostCount(t, st, "acme"); n != 0 {
		t.Errorf("hosts = %d, want 0 (leg 1 completed before the failure)", n)
	}
	if !configPresent(t, st, "acme") {
		t.Fatal("the customer row was purged despite a failed external leg — the purge must be withheld")
	}
	custodyAfterFailure := superseded(t, st, "acme")
	if custodyAfterFailure == 0 {
		t.Error("retained custody was destroyed by a FAILED cascade — leg 3 is the only purge point")
	}
	cr, err := st.LatestCustomerReset("acme")
	if err != nil || cr == nil {
		t.Fatalf("journal retained? err=%v row=%v", err, cr != nil)
	}
	if cr.CompletedAt != nil {
		t.Error("journal stamped complete despite a failed leg")
	}
	if cr.Legs["hosts"] != "ok" || cr.Legs["pbs"] != "failed" {
		t.Errorf("journal legs = %v, want hosts=ok pbs=failed", cr.Legs)
	}

	// ── Re-run: resumes at leg 2, does NOT re-delete/re-demote hosts, completes. ────────────────
	fake.err = nil
	rr2 := postDelete(t, s, "acme", cascadeForm("acme", 0)) // the live host count is now 0
	if rr2.Code != http.StatusSeeOther {
		t.Fatalf("resume status = %d, want 303: %s", rr2.Code, rr2.Body.String())
	}
	if fake.deprovisionCalls != 2 {
		t.Errorf("PBS deprovision calls = %d, want 2 (failed + resumed)", fake.deprovisionCalls)
	}
	if configPresent(t, st, "acme") {
		t.Error("customer row survived the resumed cascade")
	}
	if esc, _ := st.GetHostEscrow(hostID); esc != nil {
		t.Error("current escrow survived the resumed cascade")
	}
	if n := superseded(t, st, "acme"); n != 0 {
		t.Errorf("retained custody after resume = %d, want 0", n)
	}
	cr2, _ := st.LatestCustomerReset("acme")
	if cr2 == nil || cr2.CompletedAt == nil {
		t.Error("the resumed run did not stamp a completed journal")
	}
}

// A resumed run must still pass every gate — the acknowledgements are not cached across attempts.
func TestDeleteCascade_ResumeStillGated(t *testing.T) {
	s, st := newTestServer(t)
	seedDeletable(t, st, "acme")
	fake := &orderTenancy{err: errCascadeBoom}
	s.SetTenantSync(fake)
	if rr := postDelete(t, s, "acme", cascadeForm("acme", 1)); rr.Code != http.StatusBadGateway {
		t.Fatalf("first run status = %d, want 502", rr.Code)
	}
	fake.err = nil
	f := cascadeForm("acme", 0)
	f.Del("ack_purge")
	if rr := postDelete(t, s, "acme", f); rr.Code != http.StatusBadRequest {
		t.Fatalf("resume without ack #3 status = %d, want 400", rr.Code)
	}
	if !configPresent(t, st, "acme") {
		t.Error("an ungated resume purged the customer")
	}
}

// ── Scenario E: custody is purged exactly once, in leg 3 ────────────────────────────────────────

// The cascade calls commitCustomerReset with purgeEscrow=FALSE so the retained custody survives the
// RESET leg and dies only in DeleteCustomerConfig. RED-PROOF: pass true here and the first assertion
// fails with 0 retained blobs — i.e. the purge would have moved into leg 2.
func TestCommitCustomerReset_PurgeEscrowFlagGovernsCustody(t *testing.T) {
	s, st := newTestServer(t)
	hostID := seedDeletable(t, st, "acme")
	if err := st.DeleteHost(hostID, true); err != nil { // leg 1: DEMOTE
		t.Fatalf("demote: %v", err)
	}
	if n := superseded(t, st, "acme"); n != 2 {
		t.Fatalf("retained blobs after demotion = %d, want 2 (leg 1 demotes, never purges)", n)
	}
	cfg, _ := st.GetCustomerConfig("acme")
	id, err := st.StartCustomerReset("acme", true)
	if err != nil {
		t.Fatalf("journal: %v", err)
	}
	if lerr := s.commitCustomerReset(context.Background(), cfg, "acme", id, false); lerr != nil {
		t.Fatalf("commitCustomerReset: %v", lerr)
	}
	if n := superseded(t, st, "acme"); n != 2 {
		t.Errorf("retained blobs after the RESET leg = %d, want 2 — the cascade's RESET leg must NOT purge custody", n)
	}
	// Leg 3 is the one true purge point.
	if err := st.DeleteCustomerConfig("acme"); err != nil {
		t.Fatalf("leg 3: %v", err)
	}
	if n := superseded(t, st, "acme"); n != 0 {
		t.Errorf("retained blobs after leg 3 = %d, want 0", n)
	}
}

// ── Preview: the dialog's inventory names the real things (never a secret) ──────────────────────

func TestDeleteCascadePreview_Inventory(t *testing.T) {
	s, st := newTestServer(t)
	hostID := seedDeletable(t, st, "acme")
	s.SetTenantSync(&orderTenancy{})
	req := httptest.NewRequest("GET", "/configs/acme/delete", nil)
	rr := httptest.NewRecorder()
	s.handleCustomerDeletePreview(rr, req, "acme")
	if rr.Code != http.StatusOK {
		t.Fatalf("status = %d: %s", rr.Code, rr.Body.String())
	}
	body := rr.Body.String()
	for _, want := range []string{hostID, `"host_count":1`, `"online_host_present":false`,
		`"offsite_enabled":true`, `"pbs_tenancy_configured":true`, `"claim_present":true`, `"superseded_blobs":1`} {
		if !strings.Contains(body, want) {
			t.Errorf("preview missing %s\nbody: %s", want, body)
		}
	}
	for _, secret := range []string{"one-time-pw", "capi-acme", "hapi-", "blobA", "blobB"} {
		if strings.Contains(body, secret) {
			t.Errorf("preview leaked %q — counts and names only", secret)
		}
	}
}

// ── Residue (v0.70.0): the leg that actually makes a deleted customer DISAPPEAR ─────────────────
//
// The v0.69.0 cascade left the report stream behind, and `GetCustomers()` builds the Customers list
// (and the staleness/offsite checkers' work list) purely from `reports` — so a fully deleted
// customer stayed visible AND kept emailing the operator. Observed live on `demo-vm-felhom`:
// deleted 2026-07-18, still raising `offsite_stale` on 2026-07-21.

// seedResidue adds the report-derived state + the credential-bearing bindings to a customer.
func seedResidue(t *testing.T, st *store.Store, customerID string) {
	t.Helper()
	if err := st.SaveReport(customerID, []byte(`{"health":{"status":"ok"}}`)); err != nil {
		t.Fatalf("seed report: %v", err)
	}
	if err := st.SaveAppTelemetry(customerID, time.Now(), []store.AppTelemetryRecord{
		{AppName: "immich", DisplayName: "Immich", MemoryCurrentMB: 512},
	}); err != nil {
		t.Fatalf("seed telemetry: %v", err)
	}
	if err := st.SaveNotificationPrefs(customerID, "t@example.com", []string{"host_down"}, 6); err != nil {
		t.Fatalf("seed notif prefs: %v", err)
	}
	if err := st.MintSelfBindToken(customerID, "tokenhash-"+customerID, time.Hour); err != nil {
		t.Fatalf("seed selfbind token: %v", err)
	}
	// A DELIVERED appliance registration bound to the customer — credential-bearing (token_hash).
	if _, _, err := st.RegisterAppliance("uuid-"+customerID, "aa:bb", "ssh-ed25519 AAAA", "{}", "apphash-"+customerID, ""); err != nil {
		t.Fatalf("seed appliance: %v", err)
	}
	app, err := st.ApplianceByToken("apphash-" + customerID)
	if err != nil || app == nil {
		t.Fatalf("seed appliance lookup: %v (row=%v)", err, app != nil)
	}
	if err := st.BindAppliance(app.ID, customerID, "appliance", ""); err != nil {
		t.Fatalf("bind appliance: %v", err)
	}
}

func listedInCustomers(t *testing.T, st *store.Store, customerID string) bool {
	t.Helper()
	cs, err := st.GetCustomers()
	if err != nil {
		t.Fatalf("GetCustomers: %v", err)
	}
	for _, c := range cs {
		if c.CustomerID == customerID {
			return true
		}
	}
	return false
}

// The cascade purges the residue, so the customer leaves the Customers list — and with it the
// staleness/offsite checkers' work list. RED-PROOF: drop the residue leg and this FAILS with the
// customer still listed and 1 report row alive.
func TestDeleteCascade_PurgesResidueAndUnlistsCustomer(t *testing.T) {
	s, st := newTestServer(t)
	seedDeletable(t, st, "acme")
	seedResidue(t, st, "acme")
	s.SetTenantSync(&orderTenancy{})

	if !listedInCustomers(t, st, "acme") {
		t.Fatal("precondition: the customer must be listed before the cascade")
	}
	res, err := st.CustomerResidue("acme")
	if err != nil || res.Total() == 0 {
		t.Fatalf("precondition: residue must exist (err=%v, total=%d)", err, res.Total())
	}

	if rr := postDelete(t, s, "acme", cascadeForm("acme", 1)); rr.Code != http.StatusSeeOther {
		t.Fatalf("status = %d, want 303: %s", rr.Code, rr.Body.String())
	}

	after, err := st.CustomerResidue("acme")
	if err != nil {
		t.Fatalf("residue: %v", err)
	}
	if after.Total() != 0 {
		t.Errorf("residue after cascade = %+v, want all zero", *after)
	}
	if listedInCustomers(t, st, "acme") {
		t.Error("the customer is STILL on the Customers list after a complete delete — the ghost that " +
			"kept raising offsite_stale alerts for demo-vm-felhom")
	}
	// The credential-bearing rows are gone by NAME, not just by count.
	if app, _ := st.ApplianceByToken("apphash-acme"); app != nil {
		t.Error("the appliance registration (token_hash, status=delivered) outlived its customer")
	}
	if n, _ := st.CountSelfBindTokens("acme"); n != 0 {
		t.Errorf("self-bind tokens = %d, want 0 — a live bind path to a deleted customer", n)
	}
	// Audit + provenance SURVIVE, exactly as in every other tier.
	if evs, _ := st.GetRecentEvents("acme", 10); len(evs) == 0 {
		t.Error("the audit event stream was purged — it must outlive every lifecycle tier")
	}
	if d, _ := st.LatestHostDeletion("acme"); d == nil {
		t.Error("F-14 host-deletion provenance was purged — it must survive")
	}
	cr, _ := st.LatestCustomerReset("acme")
	if cr == nil || cr.Legs["residue"] != "ok" || cr.Legs["customer_delete"] != "ok" {
		t.Errorf("journal legs = %v, want residue=ok customer_delete=ok", cr)
	}
}

// A GHOST — config row already gone (a pre-v0.70.0 delete), residue alive. Before v0.70.0 this
// 404'd and NO operator surface could clear it. This is the demo-vm-felhom shape exactly.
func TestDeleteCascade_GhostCustomerIsDeletable(t *testing.T) {
	s, st := newTestServer(t)
	seedDeletable(t, st, "ghost")
	seedResidue(t, st, "ghost")
	s.SetTenantSync(&orderTenancy{})
	// Model the pre-v0.70.0 aftermath: hosts deleted, config row dropped, residue left behind.
	if err := st.DeleteHost("ghost-01", true); err != nil {
		t.Fatalf("delete host: %v", err)
	}
	if err := st.DeleteCustomerConfig("ghost"); err != nil {
		t.Fatalf("drop config: %v", err)
	}
	if cfg, _ := st.GetCustomerConfig("ghost"); cfg != nil {
		t.Fatal("precondition: the config row must be gone")
	}
	if !listedInCustomers(t, st, "ghost") {
		t.Fatal("precondition: the ghost must still be listed (that IS the defect)")
	}

	// The preview must render it rather than 404 — it is the only surface that can clear a ghost.
	req := httptest.NewRequest("GET", "/configs/ghost/delete", nil)
	rr := httptest.NewRecorder()
	s.handleCustomerDeletePreview(rr, req, "ghost")
	if rr.Code != http.StatusOK {
		t.Fatalf("ghost preview = %d, want 200: %s", rr.Code, rr.Body.String())
	}
	if !strings.Contains(rr.Body.String(), `"has_config":false`) {
		t.Errorf("preview must mark the ghost (has_config=false): %s", rr.Body.String())
	}

	if rr := postDelete(t, s, "ghost", cascadeForm("ghost", 0)); rr.Code != http.StatusSeeOther {
		t.Fatalf("ghost cascade = %d, want 303: %s", rr.Code, rr.Body.String())
	}
	if listedInCustomers(t, st, "ghost") {
		t.Error("the ghost survived its own cleanup")
	}
	if res, _ := st.CustomerResidue("ghost"); res.Total() != 0 {
		t.Errorf("ghost residue = %+v, want all zero", *res)
	}
	// With no config row the offsite descriptor is unknowable — the journal must SAY so, never
	// record a bare "skipped" that reads as "there was nothing to do".
	cr, _ := st.LatestCustomerReset("ghost")
	if cr == nil || cr.Legs["hetzner"] != "skipped_no_config" || cr.Legs["descriptor"] != "skipped_no_config" {
		t.Errorf("journal legs = %v, want hetzner/descriptor = skipped_no_config", cr)
	}
}

// 404 still means "there is nothing here" — an id with no config, no host and no residue.
func TestDeleteCascade_404WhenNothingRemains(t *testing.T) {
	s, _ := newTestServer(t)
	req := httptest.NewRequest("GET", "/configs/nobody/delete", nil)
	rr := httptest.NewRecorder()
	s.handleCustomerDeletePreview(rr, req, "nobody")
	if rr.Code != http.StatusNotFound {
		t.Errorf("preview for an empty id = %d, want 404", rr.Code)
	}
	if rr2 := postDelete(t, s, "nobody", cascadeForm("nobody", 0)); rr2.Code != http.StatusNotFound {
		t.Errorf("cascade for an empty id = %d, want 404", rr2.Code)
	}
}