package web // Customer DELETE cascade (v0.69.0, R-25b) — the load-bearing contracts: // // A) Happy cascade: the legs run in the ORDER hosts → reset → purge. Proven from inside leg 2 (the // PBS deprovision callback): at that instant the host rows are ALREADY gone (leg 1 done) and the // customer row is STILL there (leg 3 not started). Ruling 3 — RESET never runs while a host // exists — is therefore preserved BY CONSTRUCTION and asserted, not merely commented. // B) Gates fail-closed: any missing ack / typed-id mismatch / stale host count / ONLINE host → // 4xx and ZERO mutations (no host deleted, no journal row, no external call, no config touched). // C) Resume: a leg-2 external failure retains the journal and names the leg; the hosts are already // gone and the customer + custody SURVIVE; a re-run resumes and completes without re-demoting. // D) Standalone RESET is untouched (its own suite stays green; here: the cascade's purgeEscrow=false // does not change what a standalone RESET purges). // E) Custody: leg 1 DEMOTES (never purges); the RESET leg with purgeEscrow=false leaves the retained // blobs alone; the purge happens exactly once, in leg 3 (DeleteCustomerConfig). import ( "context" "errors" "net/http" "net/http/httptest" "net/url" "strconv" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" "gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync" ) // errCascadeBoom is the injected external-teardown failure (scenario C). var errCascadeBoom = errors.New("pbs unreachable") // orderTenancy is a tenancyProvisioner that runs a callback at the exact moment leg 2's PBS // deprovision fires — the observation point that proves the cascade's leg ORDER. type orderTenancy struct { deprovisionCalls int err error onDeprovision func() } func (f *orderTenancy) Provision(ctx context.Context, customerID string) (*tenantsync.Result, error) { return nil, nil } func (f *orderTenancy) Reissue(ctx context.Context, customerID string) (*tenantsync.Result, error) { return nil, nil } func (f *orderTenancy) Deprovision(ctx context.Context, customerID string) (bool, error) { f.deprovisionCalls++ if f.onDeprovision != nil { f.onDeprovision() } if f.err != nil { return false, f.err } return true, nil } // seedDeletable seeds a customer with a full footprint INCLUDING one OFFLINE host that carries a // current escrow blob plus one already-retained (superseded) blob. Returns the host id. func seedDeletable(t *testing.T, st *store.Store, customerID string) string { t.Helper() cfgJSON := `{"offsite":{"enabled":true,"type":"shared","host":"u1-sub3.your-storagebox.de","user":"u1-sub3","port":23,"repo_path":"/home/felhom","quota_gb":100,"host_fingerprint":"SHA256:abc"}}` if err := st.SaveCustomerConfig(&store.CustomerConfig{ CustomerID: customerID, CustomerName: "Teszt", Domain: customerID + ".example", Email: "t@example.com", RetrievalPassword: "pw", APIKey: "capi-" + customerID, ConfigJSON: cfgJSON, }); err != nil { t.Fatalf("seed config: %v", err) } hostID := customerID + "-01" long := time.Now().Add(-72 * time.Hour) // far past the stale threshold → status "down", deletable if err := st.UpsertHost(&store.Host{HostID: hostID, CustomerID: customerID, APIKey: "hapi-" + hostID, LastReportAt: &long}); err != nil { t.Fatalf("seed host: %v", err) } if _, err := st.SaveHostEscrow(hostID, []byte("blobA"), "fpA", "posture", "2026-01-01T00:00:00Z", "shaA"); err != nil { t.Fatalf("seed escrow A: %v", err) } // A second save supersedes A → one RETAINED blob + one CURRENT blob before the cascade runs. if _, err := st.SaveHostEscrow(hostID, []byte("blobB"), "fpB", "posture", "2026-01-02T00:00:00Z", "shaB"); err != nil { t.Fatalf("seed escrow B: %v", err) } if err := st.SaveOneTimeSecret(customerID, "one-time-pw"); err != nil { t.Fatalf("seed one-time secret: %v", err) } if err := st.SaveDRRecipeHostHalf(customerID, hostID, 1, []byte("half")); err != nil { t.Fatalf("seed dr recipe: %v", err) } if _, err := st.RotateClaimCode(customerID, "$2a$10$hashhashhashhashhashha"); err != nil { t.Fatalf("seed claim: %v", err) } return hostID } func cascadeForm(customerID string, hostCount int) url.Values { return url.Values{ "ack_hosts": {"1"}, "ack_reset": {"1"}, "ack_purge": {"1"}, "confirm_id": {customerID}, "expect_hosts": {strconv.Itoa(hostCount)}, } } func postDelete(t *testing.T, s *Server, customerID string, form url.Values) *httptest.ResponseRecorder { t.Helper() req := httptest.NewRequest("POST", "/configs/"+customerID+"/delete", strings.NewReader(form.Encode())) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") rr := httptest.NewRecorder() s.handleCustomerDelete(rr, req, customerID) return rr } func hostCount(t *testing.T, st *store.Store, customerID string) int { t.Helper() hosts, err := st.ListHostsByCustomer(customerID) if err != nil { t.Fatalf("list hosts: %v", err) } return len(hosts) } func configPresent(t *testing.T, st *store.Store, customerID string) bool { t.Helper() cfg, err := st.GetCustomerConfig(customerID) if err != nil { t.Fatalf("get config: %v", err) } return cfg != nil } // ── Scenario A: the happy cascade, and the leg ORDER it must run in ───────────────────────────── func TestDeleteCascade_HappyPath_LegOrder(t *testing.T) { s, st := newTestServer(t) hostID := seedDeletable(t, st, "acme") var ( hostsAtResetLeg = -1 configAtResetLeg = false custodyAtResetLeg = -1 ) fake := &orderTenancy{onDeprovision: func() { hostsAtResetLeg = hostCount(t, st, "acme") configAtResetLeg = configPresent(t, st, "acme") custodyAtResetLeg = superseded(t, st, "acme") }} s.SetTenantSync(fake) rr := postDelete(t, s, "acme", cascadeForm("acme", 1)) if rr.Code != http.StatusSeeOther { t.Fatalf("status = %d, want 303: %s", rr.Code, rr.Body.String()) } if loc := rr.Header().Get("Location"); loc != "/configs?flash=deleted" { t.Errorf("Location = %q, want /configs?flash=deleted", loc) } // ORDER, observed from inside leg 2 — this is the assertion the whole cascade hangs on. if fake.deprovisionCalls != 1 { t.Fatalf("PBS deprovision calls = %d, want 1 (leg 2 must run)", fake.deprovisionCalls) } if hostsAtResetLeg != 0 { t.Errorf("at the RESET leg the customer still had %d host(s) — leg 1 must complete FIRST "+ "(ruling 3: the RESET sequence never runs while a host row exists)", hostsAtResetLeg) } if !configAtResetLeg { t.Error("at the RESET leg the customer row was already gone — leg 3 must run LAST") } if custodyAtResetLeg == 0 { t.Error("at the RESET leg the retained custody was already gone — leg 1 DEMOTES, it must never purge") } // Final state: hosts gone, customer gone, ALL custody gone (leg 3 is the one true purge point). if n := hostCount(t, st, "acme"); n != 0 { t.Errorf("hosts after cascade = %d, want 0", n) } if configPresent(t, st, "acme") { t.Error("customer row survived the cascade") } if esc, err := st.GetHostEscrow(hostID); err != nil || esc != nil { t.Errorf("current escrow survived the cascade (err=%v, row=%v)", err, esc != nil) } if n := superseded(t, st, "acme"); n != 0 { t.Errorf("retained escrow blobs after cascade = %d, want 0 (leg 3 purges custody)", n) } // Journal: every leg stamped, completion stamped. cr, err := st.LatestCustomerReset("acme") if err != nil || cr == nil { t.Fatalf("journal: %v (row=%v)", err, cr != nil) } if cr.CompletedAt == nil { t.Error("journal not stamped complete") } for leg, want := range map[string]string{"hosts": "ok", "pbs": "ok", "db_purge": "ok", "customer_delete": "ok"} { if got := cr.Legs[leg]; got != want { t.Errorf("journal leg %q = %q, want %q (legs=%v)", leg, got, want, cr.Legs) } } // The audit event SURVIVES the customer row (events are keyed by id, never wiped). evs, err := st.GetRecentEvents("acme", 10) if err != nil { t.Fatalf("events: %v", err) } found := false for _, e := range evs { if e.EventType == "customer_deleted" { found = true } } if !found { t.Errorf("no customer_deleted audit event survived the cascade (events=%d)", len(evs)) } } // ── Scenario B: every gate fails closed, with ZERO mutations ──────────────────────────────────── func TestDeleteCascade_GatesFailClosed(t *testing.T) { base := func() url.Values { return cascadeForm("acme", 1) } cases := []struct { name string mutate func(url.Values) online bool wantCode int }{ {"missing ack 1 (hosts)", func(f url.Values) { f.Del("ack_hosts") }, false, http.StatusBadRequest}, {"missing ack 2 (reset)", func(f url.Values) { f.Del("ack_reset") }, false, http.StatusBadRequest}, {"missing ack 3 (purge)", func(f url.Values) { f.Del("ack_purge") }, false, http.StatusBadRequest}, {"ack sent as something other than 1", func(f url.Values) { f.Set("ack_purge", "yes") }, false, http.StatusBadRequest}, {"typed id mismatch", func(f url.Values) { f.Set("confirm_id", "acm") }, false, http.StatusBadRequest}, {"typed id absent", func(f url.Values) { f.Del("confirm_id") }, false, http.StatusBadRequest}, {"stale preview (host count moved)", func(f url.Values) { f.Set("expect_hosts", "0") }, false, http.StatusConflict}, {"stale preview (count absent)", func(f url.Values) { f.Del("expect_hosts") }, false, http.StatusConflict}, {"ONLINE host", func(f url.Values) {}, true, http.StatusConflict}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s, st := newTestServer(t) hostID := seedDeletable(t, st, "acme") if tc.online { // A just-saved report → status "ok" (online), the same way host-delete's own gate sees it. if err := st.SaveHostReport(hostID, "acme", []byte(`{}`), store.HostReportDenorm{}); err != nil { t.Fatalf("make host online: %v", err) } } fake := &orderTenancy{} s.SetTenantSync(fake) custodyBefore := superseded(t, st, "acme") f := base() tc.mutate(f) rr := postDelete(t, s, "acme", f) if rr.Code != tc.wantCode { t.Errorf("status = %d, want %d: %s", rr.Code, tc.wantCode, rr.Body.String()) } // ZERO mutations — the whole point of a fail-closed gate. if n := hostCount(t, st, "acme"); n != 1 { t.Errorf("hosts = %d, want 1 (a refused delete deletes NOTHING)", n) } if !configPresent(t, st, "acme") { t.Error("customer row was deleted by a REFUSED delete") } if esc, _ := st.GetHostEscrow(hostID); esc == nil { t.Error("current escrow was touched by a REFUSED delete") } if n := superseded(t, st, "acme"); n != custodyBefore { t.Errorf("retained custody = %d, want %d (untouched)", n, custodyBefore) } if fake.deprovisionCalls != 0 { t.Errorf("PBS deprovision called %d time(s) on a REFUSED delete — no external call may fire", fake.deprovisionCalls) } if cr, _ := st.LatestCustomerReset("acme"); cr != nil { t.Errorf("a journal row was opened by a REFUSED delete (#%d) — gates run before any write", cr.ID) } }) } } // ── Scenario C: a mid-cascade external failure is resumable ───────────────────────────────────── func TestDeleteCascade_ResumesAfterExternalFailure(t *testing.T) { s, st := newTestServer(t) hostID := seedDeletable(t, st, "acme") fake := &orderTenancy{err: errCascadeBoom} s.SetTenantSync(fake) rr := postDelete(t, s, "acme", cascadeForm("acme", 1)) if rr.Code != http.StatusBadGateway { t.Fatalf("status = %d, want 502: %s", rr.Code, rr.Body.String()) } if body := rr.Body.String(); !strings.Contains(body, "leg 2") || !strings.Contains(body, "pbs") { t.Errorf("the error must NAME the failed leg, got %q", body) } // Leg 1 completed; legs 2-3 did not. The customer and ALL custody SURVIVE. if n := hostCount(t, st, "acme"); n != 0 { t.Errorf("hosts = %d, want 0 (leg 1 completed before the failure)", n) } if !configPresent(t, st, "acme") { t.Fatal("the customer row was purged despite a failed external leg — the purge must be withheld") } custodyAfterFailure := superseded(t, st, "acme") if custodyAfterFailure == 0 { t.Error("retained custody was destroyed by a FAILED cascade — leg 3 is the only purge point") } cr, err := st.LatestCustomerReset("acme") if err != nil || cr == nil { t.Fatalf("journal retained? err=%v row=%v", err, cr != nil) } if cr.CompletedAt != nil { t.Error("journal stamped complete despite a failed leg") } if cr.Legs["hosts"] != "ok" || cr.Legs["pbs"] != "failed" { t.Errorf("journal legs = %v, want hosts=ok pbs=failed", cr.Legs) } // ── Re-run: resumes at leg 2, does NOT re-delete/re-demote hosts, completes. ──────────────── fake.err = nil rr2 := postDelete(t, s, "acme", cascadeForm("acme", 0)) // the live host count is now 0 if rr2.Code != http.StatusSeeOther { t.Fatalf("resume status = %d, want 303: %s", rr2.Code, rr2.Body.String()) } if fake.deprovisionCalls != 2 { t.Errorf("PBS deprovision calls = %d, want 2 (failed + resumed)", fake.deprovisionCalls) } if configPresent(t, st, "acme") { t.Error("customer row survived the resumed cascade") } if esc, _ := st.GetHostEscrow(hostID); esc != nil { t.Error("current escrow survived the resumed cascade") } if n := superseded(t, st, "acme"); n != 0 { t.Errorf("retained custody after resume = %d, want 0", n) } cr2, _ := st.LatestCustomerReset("acme") if cr2 == nil || cr2.CompletedAt == nil { t.Error("the resumed run did not stamp a completed journal") } } // A resumed run must still pass every gate — the acknowledgements are not cached across attempts. func TestDeleteCascade_ResumeStillGated(t *testing.T) { s, st := newTestServer(t) seedDeletable(t, st, "acme") fake := &orderTenancy{err: errCascadeBoom} s.SetTenantSync(fake) if rr := postDelete(t, s, "acme", cascadeForm("acme", 1)); rr.Code != http.StatusBadGateway { t.Fatalf("first run status = %d, want 502", rr.Code) } fake.err = nil f := cascadeForm("acme", 0) f.Del("ack_purge") if rr := postDelete(t, s, "acme", f); rr.Code != http.StatusBadRequest { t.Fatalf("resume without ack #3 status = %d, want 400", rr.Code) } if !configPresent(t, st, "acme") { t.Error("an ungated resume purged the customer") } } // ── Scenario E: custody is purged exactly once, in leg 3 ──────────────────────────────────────── // The cascade calls commitCustomerReset with purgeEscrow=FALSE so the retained custody survives the // RESET leg and dies only in DeleteCustomerConfig. RED-PROOF: pass true here and the first assertion // fails with 0 retained blobs — i.e. the purge would have moved into leg 2. func TestCommitCustomerReset_PurgeEscrowFlagGovernsCustody(t *testing.T) { s, st := newTestServer(t) hostID := seedDeletable(t, st, "acme") if err := st.DeleteHost(hostID, true); err != nil { // leg 1: DEMOTE t.Fatalf("demote: %v", err) } if n := superseded(t, st, "acme"); n != 2 { t.Fatalf("retained blobs after demotion = %d, want 2 (leg 1 demotes, never purges)", n) } cfg, _ := st.GetCustomerConfig("acme") id, err := st.StartCustomerReset("acme", true) if err != nil { t.Fatalf("journal: %v", err) } if lerr := s.commitCustomerReset(context.Background(), cfg, id, false); lerr != nil { t.Fatalf("commitCustomerReset: %v", lerr) } if n := superseded(t, st, "acme"); n != 2 { t.Errorf("retained blobs after the RESET leg = %d, want 2 — the cascade's RESET leg must NOT purge custody", n) } // Leg 3 is the one true purge point. if err := st.DeleteCustomerConfig("acme"); err != nil { t.Fatalf("leg 3: %v", err) } if n := superseded(t, st, "acme"); n != 0 { t.Errorf("retained blobs after leg 3 = %d, want 0", n) } } // ── Preview: the dialog's inventory names the real things (never a secret) ────────────────────── func TestDeleteCascadePreview_Inventory(t *testing.T) { s, st := newTestServer(t) hostID := seedDeletable(t, st, "acme") s.SetTenantSync(&orderTenancy{}) req := httptest.NewRequest("GET", "/configs/acme/delete", nil) rr := httptest.NewRecorder() s.handleCustomerDeletePreview(rr, req, "acme") if rr.Code != http.StatusOK { t.Fatalf("status = %d: %s", rr.Code, rr.Body.String()) } body := rr.Body.String() for _, want := range []string{hostID, `"host_count":1`, `"online_host_present":false`, `"offsite_enabled":true`, `"pbs_tenancy_configured":true`, `"claim_present":true`, `"superseded_blobs":1`} { if !strings.Contains(body, want) { t.Errorf("preview missing %s\nbody: %s", want, body) } } for _, secret := range []string{"one-time-pw", "capi-acme", "hapi-", "blobA", "blobB"} { if strings.Contains(body, secret) { t.Errorf("preview leaked %q — counts and names only", secret) } } }