package offsite import ( "context" "database/sql" "encoding/json" "errors" "io" "log" "path/filepath" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) func newTestProvisioner(t *testing.T) (*Provisioner, *hetznerapi.Fake, *store.Store) { t.Helper() st, err := store.New(filepath.Join(t.TempDir(), "off.db"), log.New(io.Discard, "", 0)) if err != nil { t.Fatal(err) } t.Cleanup(func() { st.Close() }) fake := hetznerapi.NewFake() // ScanBackoff: empty (non-nil) → single scan attempt, no retries — tests that want the F2 retry set // their own schedule (nil would select the ~60s production default and stall the suite). return &Provisioner{API: fake, Store: st, Scanner: &fakeScanner{fp: "SHA256:testfp"}, PoolBoxID: 611421, Location: "fsn1", Logger: log.New(io.Discard, "", 0), ScanBackoff: []time.Duration{}}, fake, st } // flakyScanner fails the first `failures` calls (fresh-resource DNS lag), then succeeds. type flakyScanner struct { failures int fp string calls int } func (f *flakyScanner) Fingerprint(_ context.Context, _ string, _ int) (string, error) { f.calls++ if f.calls <= f.failures { return "", errors.New("dial tcp: lookup fresh.your-storagebox.de: no such host") } return f.fp, nil } // fakeScanner returns a fixed fingerprint (or an error) — no live SSH in tests. type fakeScanner struct { fp string err error } func (f *fakeScanner) Fingerprint(_ context.Context, _ string, _ int) (string, error) { return f.fp, f.err } // Scenario A — enable shared → sub-account provisioned, descriptor built, one-time password stored (NOT in // ConfigJSON), password absent from the merged config. func TestProvision_Shared(t *testing.T) { p, fake, st := newTestProvisioner(t) d, err := p.ProvisionOffsite(context.Background(), "cust-a", Input{Enabled: true, Type: "shared", QuotaGB: 50}) if err != nil { t.Fatalf("provision: %v", err) } if fake.CreatedSubaccounts != 1 { t.Fatalf("want 1 subaccount created, got %d", fake.CreatedSubaccounts) } if d.Type != "shared" || d.Port != 23 || d.RepoPath != "/home/felhom-repo" || d.QuotaGB != 50 || d.User == "" || d.Host == "" { t.Fatalf("descriptor wrong: %+v", d) } // one-time password stored + is NOT the descriptor / config pw, err := st.ConsumeOneTimeSecret("cust-a") if err != nil || pw == "" { t.Fatalf("one-time password not stored: %v", err) } merged, err := MergeDescriptor(`{"git":{"token":"x"}}`, d) if err != nil { t.Fatal(err) } if strings.Contains(merged, pw) { t.Fatal("the transient password LEAKED into ConfigJSON") } if !strings.Contains(merged, `"offsite"`) || !strings.Contains(merged, `"git"`) { t.Fatalf("merge lost keys: %s", merged) } // descriptor struct has no password field at all db, _ := json.Marshal(d) if strings.Contains(strings.ToLower(string(db)), "password") { t.Fatalf("descriptor carries a password field: %s", db) } } // Part 0 — the descriptor carries the box host-key fingerprint (captured at provision). func TestProvision_HostFingerprint(t *testing.T) { p, _, _ := newTestProvisioner(t) d, err := p.ProvisionOffsite(context.Background(), "cust-fp", Input{Enabled: true, Type: "shared", QuotaGB: 10}) if err != nil { t.Fatal(err) } if d.HostFingerprint != "SHA256:testfp" { t.Fatalf("descriptor must carry the host fingerprint, got %q", d.HostFingerprint) } } // Part 0 — a host-key scan failure is fail-closed (no descriptor served). func TestProvision_ScanFailClosed(t *testing.T) { p, _, st := newTestProvisioner(t) p.Scanner = &fakeScanner{err: errors.New("keyscan timeout")} d, err := p.ProvisionOffsite(context.Background(), "cust-sf", Input{Enabled: true, Type: "shared", QuotaGB: 10}) if err == nil || d != nil { t.Fatalf("a keyscan failure must fail-closed, got d=%+v err=%v", d, err) } // the resource may have been created + password stored, but no verifiable descriptor is served _ = st } // Scenario D (F2) — a fresh sub-account's DNS lags creation: the scan is retried and the FIRST save // serves the descriptor. func TestProvision_ScanRetriesThroughDNSLag(t *testing.T) { p, _, _ := newTestProvisioner(t) p.ScanBackoff = []time.Duration{0, 0, 0} // instant retries in tests sc := &flakyScanner{failures: 2, fp: "SHA256:late"} p.Scanner = sc d, err := p.ProvisionOffsite(context.Background(), "cust-dns", Input{Enabled: true, Type: "shared", QuotaGB: 10}) if err != nil { t.Fatalf("the first save must survive DNS lag via scan retries, got: %v", err) } if d.HostFingerprint != "SHA256:late" || sc.calls != 3 { t.Fatalf("want fingerprint after 2 failed + 1 good scan, got fp=%q calls=%d", d.HostFingerprint, sc.calls) } } // Scenario D (F2) — a scan that keeps failing past the budget still fails CLOSED (no descriptor). func TestProvision_ScanExhaustedFailsClosed(t *testing.T) { p, _, _ := newTestProvisioner(t) p.ScanBackoff = []time.Duration{0, 0} sc := &flakyScanner{failures: 99, fp: "SHA256:never"} p.Scanner = sc d, err := p.ProvisionOffsite(context.Background(), "cust-dns2", Input{Enabled: true, Type: "shared", QuotaGB: 10}) if err == nil || d != nil { t.Fatalf("an exhausted scan budget must fail closed, got d=%+v err=%v", d, err) } if sc.calls != 3 { // 1 initial + 2 retries t.Fatalf("want 3 attempts (1 + 2 retries), got %d", sc.calls) } } // Scenario A (F4) — re-issue resets the labelled sub-account's password and stores a FRESH one-time // secret (the consumed-password dead-end recovery). func TestReissue_SharedFreshSecret(t *testing.T) { p, fake, st := newTestProvisioner(t) if _, err := p.ProvisionOffsite(context.Background(), "cust-r", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil { t.Fatal(err) } pw1, err := st.ConsumeOneTimeSecret("cust-r") // the original is spent (the dead-end premise) if err != nil || pw1 == "" { t.Fatal("harness: no initial secret") } if err := p.ReissueCredentials(context.Background(), "cust-r", "shared"); err != nil { t.Fatalf("reissue: %v", err) } if fake.ResetCalls != 1 { t.Fatalf("want exactly 1 sub-account password reset, got %d", fake.ResetCalls) } pw2, err := st.ConsumeOneTimeSecret("cust-r") if err != nil || pw2 == "" { t.Fatal("a FRESH one-time secret must be stored after reissue") } if pw2 == pw1 { t.Fatal("the re-issued password must differ from the spent one") } } // Scenario A (F4) — the dedicated path resets the labelled box's password. func TestReissue_DedicatedFreshSecret(t *testing.T) { p, fake, st := newTestProvisioner(t) if _, err := p.ProvisionOffsite(context.Background(), "cust-rd", Input{Enabled: true, Type: "dedicated", BoxType: "bx11"}); err != nil { t.Fatal(err) } _, _ = st.ConsumeOneTimeSecret("cust-rd") if err := p.ReissueCredentials(context.Background(), "cust-rd", "dedicated"); err != nil { t.Fatalf("reissue dedicated: %v", err) } if fake.BoxResetCalls != 1 { t.Fatalf("want exactly 1 box password reset, got %d", fake.BoxResetCalls) } if pw, err := st.ConsumeOneTimeSecret("cust-rd"); err != nil || pw == "" { t.Fatal("fresh secret must be stored after a dedicated reissue") } } // Scenario A WRONG-guard (F4) — an ambiguous label lookup (≠1 resource) must REFUSE: no reset, no secret. func TestReissue_RefusesAmbiguousLookup(t *testing.T) { p, fake, st := newTestProvisioner(t) // two sub-accounts labelled for the same customer (should never happen — refuse rather than guess) fake.Subaccounts[1] = hetznerapi.Subaccount{ID: 1, StorageBox: 611421, Username: "u-sub1", Labels: map[string]string{"felhom-customer": "cust-amb"}} fake.Subaccounts[2] = hetznerapi.Subaccount{ID: 2, StorageBox: 611421, Username: "u-sub2", Labels: map[string]string{"felhom-customer": "cust-amb"}} err := p.ReissueCredentials(context.Background(), "cust-amb", "shared") if err == nil || !strings.Contains(err.Error(), "exactly 1") { t.Fatalf("ambiguous lookup must refuse, got %v", err) } if fake.ResetCalls != 0 { t.Fatal("NO reset may run on an ambiguous lookup") } if _, cerr := st.ConsumeOneTimeSecret("cust-amb"); cerr == nil { t.Fatal("NO secret may be stored on a refused reissue") } // zero resources → also refuse (nothing provisioned) if err := p.ReissueCredentials(context.Background(), "cust-none", "shared"); err == nil { t.Fatal("reissue with nothing provisioned must refuse") } } // SCENARIO C (R-196 / R-204 item 2, hub v0.95.0) — re-issuing the offsite PROVIDER credential must // LEAVE A HEALTHY ESCROW ALONE. // // THIS TEST REPLACES `TestReissue_InvalidatesEscrow`, which asserted the opposite and was wrong on a // false premise: a re-issue resets the storage-provider account password and cannot touch the restic // REPOSITORY password, which is generated on the box and never leaves it except sealed under R. The // old assertion looked right because the EFFECT it checked (a stale escrow) was real — nobody checked // the CAUSE. Kept as one test rather than deleted so the inversion is visible in `git log -L`. // // WHAT IT ASSERTS IS THE CONSEQUENCE, not the mechanism: not "MarkEscrowStale was not called" but // "the ACK still carries the hash", because the hash is what the controller's auto-confirm needs and // its absence is what blocked every off-site backup during the 2026-08-04 drill. // // RED-PROOF: restore the marking — put back // // if host, _ := p.Store.GetHostByCustomer(customerID); host != nil { // if esc, _ := p.Store.GetHostEscrow(host.HostID); esc != nil { p.Store.MarkEscrowStale(host.HostID) } // } // // in ReissueCredentials → es.Stale becomes true, the hash is withheld, and this test fails on both. func TestReissue_DoesNotMarkAHealthyEscrowStale(t *testing.T) { p, _, st := newTestProvisioner(t) const cust = "cust-esc" if _, err := p.ProvisionOffsite(context.Background(), cust, Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil { t.Fatal(err) } // A host + a key-escrow blob whose sealed repo-password hash the hub serves for auto-confirm. if err := st.UpsertHost(&store.Host{HostID: cust + "-01", CustomerID: cust, APIKey: "k"}); err != nil { t.Fatal(err) } if _, _, err := st.SaveHostEscrow(cust+"-01", []byte("opaque-blob"), "SHA256:fp", "zero_knowledge", "2026-07-16T00:00:00Z", "OLDHASH"); err != nil { t.Fatal(err) } es, err := st.GetEscrowStatusForCustomer(cust) if err != nil || es == nil { t.Fatalf("escrow status (before): %v", err) } if es.Stale || es.ResticPwSHA256 != "OLDHASH" { t.Fatalf("pre-reissue escrow must be current: stale=%v hash=%q", es.Stale, es.ResticPwSHA256) } // Re-issue the offsite PROVIDER credential. The box still holds its repo_password file, so the // repository password does NOT change — this is the ordinary re-issue shape. if err := p.ReissueCredentials(context.Background(), cust, "shared"); err != nil { t.Fatalf("reissue: %v", err) } es, err = st.GetEscrowStatusForCustomer(cust) if err != nil || es == nil { t.Fatalf("escrow status (after): %v", err) } if es.Stale { t.Fatal("a re-issue marked a HEALTHY escrow stale — this blocks every off-site backup and asks the customer for a ceremony that would supersede a good key (R-196)") } if es.ResticPwSHA256 != "OLDHASH" { t.Fatalf("the sealed hash must keep flowing to the ACK so the controller can auto-confirm; got %q", es.ResticPwSHA256) } // The customer is still TOLD the credential moved — removing the false alarm must not remove the // true notice. `offsite_reissued` is untouched by this change. ev, eerr := st.GetLatestEventByType(cust, "offsite_reissued") if eerr != nil || ev == nil { t.Fatalf("offsite_reissued must still fire on every re-issue (err=%v ev=%+v)", eerr, ev) } // …and the false alarm is gone. if bad, _ := st.GetLatestEventByType(cust, "escrow_stale"); bad != nil { t.Fatalf("a re-issue must not raise escrow_stale on a healthy escrow: %+v", bad) } } // The other half of Scenario C: the stale MECHANISM is intact and still does its job when something // legitimately sets it. Without this, "nothing marks stale any more" could decay into "stale no longer // works", and the next evidential caller would ship inert (the seam-built-but-never-wired shape). func TestEscrowStaleMechanism_StillWithholdsAndClears(t *testing.T) { _, _, st := newTestProvisioner(t) const cust = "cust-stale-mech" if err := st.UpsertHost(&store.Host{HostID: cust + "-01", CustomerID: cust, APIKey: "k"}); err != nil { t.Fatal(err) } if _, _, err := st.SaveHostEscrow(cust+"-01", []byte("blob"), "SHA256:fp", "zero_knowledge", "2026-07-16T00:00:00Z", "OLDHASH"); err != nil { t.Fatal(err) } if err := st.MarkEscrowStale(cust + "-01"); err != nil { t.Fatalf("MarkEscrowStale: %v", err) } es, _ := st.GetEscrowStatusForCustomer(cust) if es == nil || !es.Stale || es.ResticPwSHA256 != "" { t.Fatalf("a marked escrow must read stale AND withhold the hash: %+v", es) } // A fresh ceremony (new blob sealing the new password) clears stale + serves the new hash. if _, _, err := st.SaveHostEscrow(cust+"-01", []byte("blob-2"), "SHA256:fp", "zero_knowledge", "2026-07-16T01:00:00Z", "NEWHASH"); err != nil { t.Fatal(err) } es, _ = st.GetEscrowStatusForCustomer(cust) if es == nil || es.Stale || es.ResticPwSHA256 != "NEWHASH" { t.Fatalf("a fresh ceremony must clear stale + serve the new hash: %+v", es) } } // Scenario E (SLICE 4) — the freeze lever flips ONLY readonly on the exactly-1 labelled sub-account // (SSH stays on — a freeze must not cut access, just writes); ambiguity refuses; unfreeze reverses. func TestFreeze_SharedTogglesReadonlyOnly(t *testing.T) { p, fake, _ := newTestProvisioner(t) if _, err := p.ProvisionOffsite(context.Background(), "cust-fz", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil { t.Fatal(err) } if err := p.SetOffsiteFrozen(context.Background(), "cust-fz", true); err != nil { t.Fatalf("freeze: %v", err) } subs, _ := fake.ListSubaccounts(context.Background(), 611421, "felhom-customer=cust-fz") if len(subs) != 1 || !subs[0].AccessSettings.Readonly { t.Fatalf("freeze must set readonly on the labelled sub-account: %+v", subs) } if !subs[0].AccessSettings.SSHEnabled { t.Fatal("freeze must NOT disable SSH — only readonly flips") } if err := p.SetOffsiteFrozen(context.Background(), "cust-fz", false); err != nil { t.Fatalf("unfreeze: %v", err) } subs, _ = fake.ListSubaccounts(context.Background(), 611421, "felhom-customer=cust-fz") if subs[0].AccessSettings.Readonly { t.Fatal("unfreeze must clear readonly") } // nothing provisioned → refuse if err := p.SetOffsiteFrozen(context.Background(), "cust-none", true); err == nil { t.Fatal("freeze with no labelled sub-account must refuse") } // ambiguous → refuse fake.Subaccounts[71] = hetznerapi.Subaccount{ID: 71, StorageBox: 611421, Labels: map[string]string{"felhom-customer": "cust-amb2"}} fake.Subaccounts[72] = hetznerapi.Subaccount{ID: 72, StorageBox: 611421, Labels: map[string]string{"felhom-customer": "cust-amb2"}} if err := p.SetOffsiteFrozen(context.Background(), "cust-amb2", true); err == nil || !strings.Contains(err.Error(), "exactly 1") { t.Fatalf("ambiguous freeze must refuse, got %v", err) } } // Scenario B — enable dedicated → box provisioned. func TestProvision_Dedicated(t *testing.T) { p, fake, st := newTestProvisioner(t) d, err := p.ProvisionOffsite(context.Background(), "cust-b", Input{Enabled: true, Type: "dedicated", BoxType: "bx11"}) if err != nil { t.Fatalf("provision: %v", err) } if fake.CreatedBoxes != 1 { t.Fatalf("want 1 box created, got %d", fake.CreatedBoxes) } if d.Type != "dedicated" || d.BoxType != "bx11" || d.User == "" || d.Host == "" || d.RepoPath != "/home/felhom-repo" { t.Fatalf("descriptor wrong: %+v", d) } if pw, err := st.ConsumeOneTimeSecret("cust-b"); err != nil || pw == "" { t.Fatalf("one-time password not stored: %v", err) } } // Scenario C — idempotent re-save does NOT create a second resource. func TestProvision_Idempotent(t *testing.T) { p, fake, _ := newTestProvisioner(t) if _, err := p.ProvisionOffsite(context.Background(), "cust-c", Input{Enabled: true, Type: "shared", QuotaGB: 20}); err != nil { t.Fatal(err) } d2, err := p.ProvisionOffsite(context.Background(), "cust-c", Input{Enabled: true, Type: "shared", QuotaGB: 20}) if err != nil { t.Fatal(err) } if fake.CreatedSubaccounts != 1 { t.Fatalf("re-provision created a SECOND resource (%d) — not idempotent", fake.CreatedSubaccounts) } if d2.User == "" || d2.Type != "shared" { t.Fatalf("idempotent descriptor wrong: %+v", d2) } } // Scenario D — a provisioning error surfaces; nothing is recorded (fail-closed). Companion: the caller must // not mark offsite provisioned — modelled here by asserting no descriptor + no stored password on error. func TestProvision_FailClosed(t *testing.T) { p, fake, st := newTestProvisioner(t) fake.FailCreate = errors.New("hetzner 500") d, err := p.ProvisionOffsite(context.Background(), "cust-d", Input{Enabled: true, Type: "shared", QuotaGB: 10}) if err == nil { t.Fatal("a create failure must return an error (fail-closed)") } if d != nil { t.Fatalf("no descriptor may be returned on error, got %+v", d) } if _, cerr := st.ConsumeOneTimeSecret("cust-d"); cerr != sql.ErrNoRows { t.Fatal("no one-time password may be stored on a failed provision") } // action-failure path (create ok, action errors) is also fail-closed fake.FailCreate = nil fake.FailAction = true if _, err := p.ProvisionOffsite(context.Background(), "cust-d2", Input{Enabled: true, Type: "dedicated", BoxType: "bx11"}); err == nil { t.Fatal("a failed create-action must return an error") } } // Scenario E — the one-time password is consumable exactly once. func TestOneTimeSecret_ConsumedOnce(t *testing.T) { _, _, st := newTestProvisioner(t) if err := st.SaveOneTimeSecret("cust-e", "secretpw"); err != nil { t.Fatal(err) } got, err := st.ConsumeOneTimeSecret("cust-e") if err != nil || got != "secretpw" { t.Fatalf("first consume: got %q err %v", got, err) } if _, err := st.ConsumeOneTimeSecret("cust-e"); err != sql.ErrNoRows { t.Fatalf("second consume must be ErrNoRows, got %v", err) } if _, err := st.ConsumeOneTimeSecret("never-provisioned"); err != sql.ErrNoRows { t.Fatalf("absent consume must be ErrNoRows, got %v", err) } } // Disable → {Enabled:false}, no deprovision (no API delete). func TestProvision_DisableNoDeprovision(t *testing.T) { p, fake, _ := newTestProvisioner(t) d, err := p.ProvisionOffsite(context.Background(), "cust-f", Input{Enabled: false}) if err != nil || d == nil || d.Enabled { t.Fatalf("disable must return {enabled:false}, got %+v err %v", d, err) } if fake.DeletedSubaccounts != 0 || fake.DeletedBoxes != 0 { t.Fatal("disable must NOT deprovision (data-loss guard)") } } // Customer RESET (v0.61.0) — Deprovision DESTROYS the labelled shared sub-account, and is idempotent // (a second call finds nothing and succeeds). This is the deliberate teardown the disable-guard above // deliberately does NOT do. func TestDeprovision_SharedIdempotent(t *testing.T) { p, fake, _ := newTestProvisioner(t) if _, err := p.ProvisionOffsite(context.Background(), "cust-d", Input{Enabled: true, Type: "shared", QuotaGB: 10}); err != nil { t.Fatalf("provision: %v", err) } if fake.CreatedSubaccounts != 1 { t.Fatalf("precondition: want 1 subaccount, got %d", fake.CreatedSubaccounts) } if err := p.Deprovision(context.Background(), "cust-d", "shared"); err != nil { t.Fatalf("deprovision: %v", err) } if fake.DeletedSubaccounts != 1 { t.Fatalf("want 1 subaccount deleted, got %d", fake.DeletedSubaccounts) } // Idempotent: nothing labelled now → success, no extra delete. if err := p.Deprovision(context.Background(), "cust-d", "shared"); err != nil { t.Fatalf("second deprovision (idempotent) errored: %v", err) } if fake.DeletedSubaccounts != 1 { t.Fatalf("idempotent re-run deleted again: %d", fake.DeletedSubaccounts) } } // ClearProvisionedDescriptor keeps the tier CHOICE (enabled/type/quota/box_type) and drops every // PROVISIONED field — the pre-first-install shape a RESET returns the customer to. func TestClearProvisionedDescriptor(t *testing.T) { in := `{"git":{"token":"x"},"offsite":{"enabled":true,"type":"shared","host":"u1-sub3.your-storagebox.de","user":"u1-sub3","port":23,"repo_path":"/home/felhom","quota_gb":100,"box_type":"","host_fingerprint":"SHA256:abc"}}` out, err := ClearProvisionedDescriptor(in) if err != nil { t.Fatal(err) } if !strings.Contains(out, `"enabled":true`) || !strings.Contains(out, `"type":"shared"`) || !strings.Contains(out, `"quota_gb":100`) { t.Errorf("tier choice lost: %s", out) } for _, gone := range []string{"your-storagebox.de", "u1-sub3", "repo_path", "host_fingerprint", `"port"`} { if strings.Contains(out, gone) { t.Errorf("provisioned field %q survived: %s", gone, out) } } if !strings.Contains(out, `"git"`) { t.Errorf("unrelated config keys dropped: %s", out) } // No-op-safe: absent offsite block returns input unchanged. if got, _ := ClearProvisionedDescriptor(`{"git":{"token":"x"}}`); got != `{"git":{"token":"x"}}` { t.Errorf("no-offsite clear mutated config: %s", got) } }