91cabdde1b
gates / gates (push) Successful in 7s
R-198 — host_escrow_superseded shipped with `blob` (the K-escrow / PBS datastore key) and
identity_blob was added to host_escrow LATER, never here. The offsite restic REPOSITORY
password lives in identity_blob. So demoteCurrentEscrowTx -- whose own comment calls it "THE
ONE escrow row-copy routine" -- retained the whole-guest key and silently dropped the off-site
data key, which is the secret the retention was built to preserve. And because the copy happens
as the new blob overwrites the old, the destroying act was the ESCROW CEREMONY: the exact thing
a rebuilt box tells its customer to run, on a card promising in Hungarian that the old backups
stay recoverable. Both demo boxes crossed that line on 2026-08-04.
- identity_blob added to the table (CREATE + additive ALTER) and carried in the shared copy
routine, so BOTH callers are fixed at once: re-escrow and host-delete demotion.
- ListSupersededEscrow reads it back; store.HostEscrow gains IdentityBlob.
- CountCurrentEscrowWithIdentity is the census of who the fix protects.
- Nothing is backfillable: pre-v0.93.0 retained rows have no blob and their sources are gone.
- Tests assert the CONSEQUENCE (a retained row can still yield a repo password), which is why
the pre-existing retention test stayed green for two months asserting the mechanism.
R-197 — SaveHostEscrow returns the hash it replaced; the escrow PUT raises
offsite_repo_key_changed (warning, operator-only, edge-triggered) when both hashes are known and
differ. No hash value travels. Severity chosen for the world v0.93.0 creates: with the identity
blob retained, a changed key is "this history now depends on an older recovery code", not a loss.
R-192 (half) — the stuck alert now reports the two shapes it actually covers, burned and
regressed, each stating its own measurement; the regressed text withdraws the Re-issue
recommendation. Every self-heal refusal leaves a notification_log row with its reason. The
guard's logic is unchanged; its 500-oldest-reports scoping stays OPEN and the window is named in
the alert text so the limitation travels with the number. offsite_delivery_stuck and
offsite_credential_restaged are added to operatorOnlyEvents -- neither was registered and neither
has a customerMessages entry, which is not a block.
R-196 — five comments (not the three the spec expected) claimed ReissueCredentials rotates the
restic repo password. It resets the PROVIDER password and cannot touch the repo password, which
is generated on the box. All five corrected; the staleness mark documented as precautionary. The
BEHAVIOUR stays open.
Not in this release: R-199, R-200, R-201 remain open -- the chain that hands the key back is
still unassembled. Part 5 hit its gate; the orphan card is untouched (R-202).
223 lines
9.3 KiB
Go
223 lines
9.3 KiB
Go
package web
|
|
|
|
// Customer RESET (v0.61.0) orchestration red-proofs. The load-bearing contracts:
|
|
// - Scenario A: RESET refuses while ANY host row exists — 409, ZERO side effects (no journal row).
|
|
// - Ruling 1: destroying retained escrow custody (M>0) requires the SEPARATE ack — missing → 400,
|
|
// nothing purged.
|
|
// - Typed-id gate: a wrong confirm_id → 400, nothing purged.
|
|
// - Happy path: external legs FIRST, DB purge LAST; identity + basic config SURVIVE; provenance +
|
|
// events SURVIVE; journal completes.
|
|
// - Partial failure (external FIRST): a failing external leg leaves the DB UNPURGED and the journal
|
|
// retained (resumable) — a re-run converges.
|
|
|
|
import (
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// seedResettable seeds a customer with a full operational footprint but NO host (the reset
|
|
// precondition). Returns the store. offsiteJSON is the config_json (with an offsite descriptor).
|
|
func seedResettable(t *testing.T, st *store.Store, customerID string) {
|
|
t.Helper()
|
|
cfgJSON := `{"offsite":{"enabled":true,"type":"shared","host":"u1-sub3.your-storagebox.de","user":"u1-sub3","port":23,"repo_path":"/home/felhom","quota_gb":100,"host_fingerprint":"SHA256:abc"}}`
|
|
if err := st.SaveCustomerConfig(&store.CustomerConfig{
|
|
CustomerID: customerID, CustomerName: "Teszt", Domain: customerID + ".example",
|
|
Email: "t@example.com", RetrievalPassword: "pw", APIKey: "capi", ConfigJSON: cfgJSON,
|
|
}); err != nil {
|
|
t.Fatalf("seed config: %v", err)
|
|
}
|
|
// A superseded escrow blob retained via F-14 provenance (host deleted, blob kept).
|
|
hostID := customerID + "-01"
|
|
if err := st.UpsertHost(&store.Host{HostID: hostID, CustomerID: customerID, APIKey: "hapi"}); err != nil {
|
|
t.Fatalf("seed host: %v", err)
|
|
}
|
|
if _, _, err := st.SaveHostEscrow(hostID, []byte("blobA"), "fpA", "posture", "2026-01-01T00:00:00Z", "shaA"); err != nil {
|
|
t.Fatalf("seed escrow A: %v", err)
|
|
}
|
|
if _, _, err := st.SaveHostEscrow(hostID, []byte("blobB"), "fpB", "posture", "2026-01-02T00:00:00Z", "shaB"); err != nil {
|
|
t.Fatalf("seed escrow B: %v", err)
|
|
}
|
|
if err := st.DeleteHost(hostID, true); err != nil { // demotes current → retained; records host_deletions
|
|
t.Fatalf("delete host (demote): %v", err)
|
|
}
|
|
if err := st.SaveOneTimeSecret(customerID, "one-time-pw"); err != nil {
|
|
t.Fatalf("seed one-time secret: %v", err)
|
|
}
|
|
if err := st.SaveDRRecipeHostHalf(customerID, hostID, 1, []byte("half")); err != nil {
|
|
t.Fatalf("seed dr recipe: %v", err)
|
|
}
|
|
if _, err := st.RotateClaimCode(customerID, "$2a$10$hashhashhashhashhashha"); err != nil {
|
|
t.Fatalf("seed claim: %v", err)
|
|
}
|
|
}
|
|
|
|
func postReset(t *testing.T, s *Server, customerID string, form url.Values) *httptest.ResponseRecorder {
|
|
t.Helper()
|
|
req := httptest.NewRequest("POST", "/configs/"+customerID+"/reset", strings.NewReader(form.Encode()))
|
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
rr := httptest.NewRecorder()
|
|
s.handleCustomerReset(rr, req, customerID)
|
|
return rr
|
|
}
|
|
|
|
func superseded(t *testing.T, st *store.Store, customerID string) int {
|
|
t.Helper()
|
|
inv, err := st.CustomerResetInventory(customerID)
|
|
if err != nil {
|
|
t.Fatalf("inventory: %v", err)
|
|
}
|
|
return inv.SupersededBlobs
|
|
}
|
|
|
|
func TestCustomerReset_RefusesWhileHostsExist(t *testing.T) {
|
|
s, st := newTestServer(t)
|
|
s.SetTenantSync(&fakeTenancy{})
|
|
seedResettable(t, st, "acme")
|
|
// Re-add a live host: RESET must refuse (ruling 3).
|
|
if err := st.UpsertHost(&store.Host{HostID: "acme-live", CustomerID: "acme", APIKey: "h"}); err != nil {
|
|
t.Fatalf("re-add host: %v", err)
|
|
}
|
|
|
|
rr := postReset(t, s, "acme", url.Values{"confirm_id": {"acme"}, "escrow_ack": {"1"}})
|
|
if rr.Code != http.StatusConflict {
|
|
t.Fatalf("reset-with-host = %d, want 409", rr.Code)
|
|
}
|
|
// ZERO side effects: no journal row, secret intact, blobs intact, claim intact.
|
|
if cr, _ := st.LatestCustomerReset("acme"); cr != nil {
|
|
t.Errorf("a journal row was opened despite the refusal: %+v", cr)
|
|
}
|
|
if inv, _ := st.CustomerResetInventory("acme"); !inv.OneTimeSecretPresent || !inv.ClaimPresent || inv.SupersededBlobs == 0 {
|
|
t.Errorf("refused reset still mutated state: %+v", inv)
|
|
}
|
|
}
|
|
|
|
// Red-proof (a): the escrow-custody ack gate. M>0 and no ack → 400, nothing purged. Dropping the
|
|
// `!escrowAck` guard would let the reset proceed and destroy the retained blobs → this FAILS.
|
|
func TestCustomerReset_EscrowAckRequired(t *testing.T) {
|
|
s, st := newTestServer(t)
|
|
s.SetTenantSync(&fakeTenancy{})
|
|
seedResettable(t, st, "acme")
|
|
if superseded(t, st, "acme") == 0 {
|
|
t.Fatal("precondition: expected retained blobs to gate on")
|
|
}
|
|
|
|
rr := postReset(t, s, "acme", url.Values{"confirm_id": {"acme"}}) // no escrow_ack
|
|
if rr.Code != http.StatusBadRequest {
|
|
t.Fatalf("reset without escrow ack = %d, want 400", rr.Code)
|
|
}
|
|
if cr, _ := st.LatestCustomerReset("acme"); cr != nil {
|
|
t.Errorf("journal opened despite the ack refusal: %+v", cr)
|
|
}
|
|
if superseded(t, st, "acme") == 0 {
|
|
t.Error("retained blobs were destroyed despite the missing ack")
|
|
}
|
|
}
|
|
|
|
func TestCustomerReset_TypedIDMustMatch(t *testing.T) {
|
|
s, st := newTestServer(t)
|
|
s.SetTenantSync(&fakeTenancy{})
|
|
seedResettable(t, st, "acme")
|
|
|
|
rr := postReset(t, s, "acme", url.Values{"confirm_id": {"acmee"}, "escrow_ack": {"1"}})
|
|
if rr.Code != http.StatusBadRequest {
|
|
t.Fatalf("reset with wrong confirm_id = %d, want 400", rr.Code)
|
|
}
|
|
if inv, _ := st.CustomerResetInventory("acme"); !inv.OneTimeSecretPresent {
|
|
t.Error("mismatched-id reset still purged state")
|
|
}
|
|
}
|
|
|
|
func TestCustomerReset_HappyPath(t *testing.T) {
|
|
s, st := newTestServer(t)
|
|
fake := &fakeTenancy{deprovisionExisted: true}
|
|
s.SetTenantSync(fake)
|
|
seedResettable(t, st, "acme")
|
|
|
|
rr := postReset(t, s, "acme", url.Values{"confirm_id": {"acme"}, "escrow_ack": {"1"}})
|
|
if rr.Code != http.StatusSeeOther {
|
|
t.Fatalf("happy reset = %d (%s), want 303", rr.Code, rr.Body.String())
|
|
}
|
|
// External leg ran.
|
|
if fake.deprovisionCalls != 1 {
|
|
t.Errorf("pbs deprovision calls = %d, want 1", fake.deprovisionCalls)
|
|
}
|
|
// Operational state DIED.
|
|
inv, _ := st.CustomerResetInventory("acme")
|
|
if inv.OneTimeSecretPresent || inv.DRRecipePresent || inv.ClaimPresent || inv.SupersededBlobs != 0 {
|
|
t.Errorf("operational state survived the reset: %+v", inv)
|
|
}
|
|
// Identity + basic config SURVIVE; the offsite tier CHOICE is kept, provisioned fields cleared.
|
|
cfg, _ := st.GetCustomerConfig("acme")
|
|
if cfg == nil {
|
|
t.Fatal("customer_config was destroyed — identity must survive a RESET")
|
|
}
|
|
if cfg.CustomerName != "Teszt" || cfg.Email != "t@example.com" {
|
|
t.Errorf("identity mutated: %+v", cfg)
|
|
}
|
|
if !strings.Contains(cfg.ConfigJSON, `"enabled":true`) || !strings.Contains(cfg.ConfigJSON, `"type":"shared"`) {
|
|
t.Errorf("offsite tier choice was lost: %s", cfg.ConfigJSON)
|
|
}
|
|
if strings.Contains(cfg.ConfigJSON, "your-storagebox.de") || strings.Contains(cfg.ConfigJSON, "u1-sub3") ||
|
|
strings.Contains(cfg.ConfigJSON, "repo_path") || strings.Contains(cfg.ConfigJSON, "host_fingerprint") {
|
|
t.Errorf("provisioned offsite fields survived the reset: %s", cfg.ConfigJSON)
|
|
}
|
|
// Journal completed with every leg recorded ok; provenance + audit event SURVIVE.
|
|
cr, _ := st.LatestCustomerReset("acme")
|
|
if cr == nil || cr.CompletedAt == nil {
|
|
t.Fatalf("journal not finished: %+v", cr)
|
|
}
|
|
for _, leg := range []string{"pbs", "claim", "descriptor", "db_purge"} {
|
|
if cr.Legs[leg] != "ok" {
|
|
t.Errorf("leg %q = %q, want ok (legs=%v)", leg, cr.Legs[leg], cr.Legs)
|
|
}
|
|
}
|
|
if ev, _ := st.GetLatestEventByType("acme", "customer_reset"); ev == nil {
|
|
t.Error("no customer_reset audit event was recorded")
|
|
}
|
|
}
|
|
|
|
// Red-proof (b): partial failure. A failing external leg (PBS) must leave the DB UNPURGED and the
|
|
// journal retained — the reset is resumable. Purging before the external legs succeed would erase the
|
|
// descriptors that tell a re-run what still needs tearing down → this FAILS.
|
|
func TestCustomerReset_PartialFailureIsResumable(t *testing.T) {
|
|
s, st := newTestServer(t)
|
|
fake := &fakeTenancy{err: errors.New("pbs boom")}
|
|
s.SetTenantSync(fake)
|
|
seedResettable(t, st, "acme")
|
|
|
|
rr := postReset(t, s, "acme", url.Values{"confirm_id": {"acme"}, "escrow_ack": {"1"}})
|
|
if rr.Code != http.StatusBadGateway {
|
|
t.Fatalf("partial reset = %d, want 502", rr.Code)
|
|
}
|
|
// Nothing purged — the external leg failed FIRST, before any DB mutation.
|
|
inv, _ := st.CustomerResetInventory("acme")
|
|
if !inv.OneTimeSecretPresent || !inv.DRRecipePresent || !inv.ClaimPresent || inv.SupersededBlobs == 0 {
|
|
t.Errorf("DB was purged despite the external leg failing: %+v", inv)
|
|
}
|
|
cfg, _ := st.GetCustomerConfig("acme")
|
|
if !strings.Contains(cfg.ConfigJSON, "your-storagebox.de") {
|
|
t.Errorf("descriptor was cleared despite the failure: %s", cfg.ConfigJSON)
|
|
}
|
|
cr, _ := st.LatestCustomerReset("acme")
|
|
if cr == nil || cr.CompletedAt != nil || cr.Legs["pbs"] != "failed" {
|
|
t.Fatalf("journal should be open with pbs=failed: %+v", cr)
|
|
}
|
|
|
|
// Resume: the external leg now succeeds; a re-run converges (idempotent from the top).
|
|
fake.err = nil
|
|
rr2 := postReset(t, s, "acme", url.Values{"confirm_id": {"acme"}, "escrow_ack": {"1"}})
|
|
if rr2.Code != http.StatusSeeOther {
|
|
t.Fatalf("resumed reset = %d (%s), want 303", rr2.Code, rr2.Body.String())
|
|
}
|
|
inv2, _ := st.CustomerResetInventory("acme")
|
|
if inv2.OneTimeSecretPresent || inv2.DRRecipePresent || inv2.ClaimPresent || inv2.SupersededBlobs != 0 {
|
|
t.Errorf("resume did not converge: %+v", inv2)
|
|
}
|
|
}
|