package offsiteheal // Non-hollow reconciler tests mapping 1:1 to the task's integration scenarios A–F. A REAL store // (t.TempDir sqlite) supplies the work set, the customer configs and the reports; a FAKE Actions seam // counts restage/reissue calls without touching a storage provider. ReconcileOnce is driven directly // for determinism — the debounce is exercised by feeding DISTINCT REPORTS, never by sleeping, because // the debounce counts fresh evidence rather than elapsed time. import ( "context" "encoding/json" "fmt" "io" "log" "path/filepath" "sync" "testing" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) type fakeActions struct { mu sync.Mutex restaged []string reissued []string restageResult bool // does a stored secret row exist? restageErr error reissueErr error } func (f *fakeActions) Restage(customerID string) (bool, error) { f.mu.Lock() defer f.mu.Unlock() f.restaged = append(f.restaged, customerID) return f.restageResult, f.restageErr } func (f *fakeActions) Reissue(_ context.Context, customerID string) error { f.mu.Lock() defer f.mu.Unlock() f.reissued = append(f.reissued, customerID) return f.reissueErr } func (f *fakeActions) restages() int { f.mu.Lock(); defer f.mu.Unlock(); return len(f.restaged) } func (f *fakeActions) reissues() int { f.mu.Lock(); defer f.mu.Unlock(); return len(f.reissued) } func newHealStore(t *testing.T) *store.Store { t.Helper() st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0)) if err != nil { t.Fatalf("store.New: %v", err) } t.Cleanup(func() { st.Close() }) return st } // seedCustomer writes an ACTIVE customer whose offsite descriptor is enabled (or not). func seedCustomer(t *testing.T, st *store.Store, customerID string, offsiteEnabled bool) { t.Helper() cfgJSON := `{"offsite":{"enabled":false}}` if offsiteEnabled { cfgJSON = `{"offsite":{"enabled":true,"type":"shared","host":"x.your-storagebox.de","user":"u","port":23,"repo_path":"/home/felhom-repo"}}` } if err := st.SaveCustomerConfig(&store.CustomerConfig{ CustomerID: customerID, CustomerName: customerID, Domain: customerID + ".hu", APIKey: "k-" + customerID, ConfigJSON: cfgJSON, }); err != nil { t.Fatalf("SaveCustomerConfig %s: %v", customerID, err) } } // pushReport writes ONE controller report. state=="" → a healthy configured box (enabled offsite // object); otherwise the declaration shape controller v0.199.0 emits. Each call is a DISTINCT report, // which is the debounce's currency. func pushReport(t *testing.T, st *store.Store, customerID, state string, seq int) { t.Helper() var off map[string]any if state == "" { off = map[string]any{"enabled": true, "escrow_state": "escrowed", "last_status": "ok"} } else { off = map[string]any{"enabled": false, "state": state} } b, err := json.Marshal(map[string]any{ "controller_version": fmt.Sprintf("0.199.0-%d", seq), // makes each row distinguishable "offsite": off, }) if err != nil { t.Fatal(err) } if err := st.SaveReport(customerID, b); err != nil { t.Fatalf("SaveReport %s: %v", customerID, err) } } // pushReportNoOffsite writes a report with NO offsite object at all — the pre-v0.199.0 shape a // stranded box used to send, and the shape a box that never had off-site backups still sends. func pushReportNoOffsite(t *testing.T, st *store.Store, customerID string, seq int) { t.Helper() b, _ := json.Marshal(map[string]any{"controller_version": fmt.Sprintf("0.198.0-%d", seq)}) if err := st.SaveReport(customerID, b); err != nil { t.Fatalf("SaveReport %s: %v", customerID, err) } } func newRec(st *store.Store, act Actions) *Reconciler { return NewReconciler(st, act, log.New(io.Discard, "", 0)) } func countEvents(t *testing.T, st *store.Store, customerID, eventType string) int { t.Helper() ev, err := st.GetLatestEventByType(customerID, eventType) if err != nil { t.Fatalf("GetLatestEventByType: %v", err) } if ev == nil { return 0 } return 1 } // SCENARIO A + D — a declaring box is served, and the STORED secret is re-armed rather than a fresh // credential minted. // // RED-PROOF (D): invert the order in heal() so Reissue is called before Restage — the assertion // "no mint" fails, and the test names the cost: external churn for a credential the hub already has. func TestScenarioAD_DeclaringBoxIsRestagedNotMinted(t *testing.T) { st := newHealStore(t) act := &fakeActions{restageResult: true} // a stored secret exists r := newRec(st, act) seedCustomer(t, st, "c1", true) pushReport(t, st, "c1", StateNeedsCredential, 1) r.ReconcileOnce(context.Background()) if act.restages() != 0 || act.reissues() != 0 { t.Fatalf("acted on a SINGLE declaration — the debounce did not hold (restages=%d reissues=%d)", act.restages(), act.reissues()) } pushReport(t, st, "c1", StateNeedsCredential, 2) // a second DISTINCT report r.ReconcileOnce(context.Background()) if act.restages() != 1 { t.Fatalf("a sustained declaration was not served: restages=%d", act.restages()) } if act.reissues() != 0 { t.Fatalf("a fresh credential was minted although a stored one could be re-armed — external churn for nothing (reissues=%d)", act.reissues()) } if countEvents(t, st, "c1", eventRestaged) != 1 { t.Error("no offsite_selfheal_restaged audit event — an automatic remediation must be visible") } if countEvents(t, st, "c1", eventReissued) != 0 { t.Error("a reissue event was recorded for a restage") } } // SCENARIO B — a box that never had off-site backups says nothing, so nothing happens. (The // declaration itself is the controller's guard; here we assert the hub side is a no-op on silence.) func TestScenarioB_SilentBoxIsNeverActedOn(t *testing.T) { st := newHealStore(t) act := &fakeActions{restageResult: true} r := newRec(st, act) seedCustomer(t, st, "c1", true) for i := 1; i <= 4; i++ { pushReportNoOffsite(t, st, "c1", i) r.ReconcileOnce(context.Background()) } if act.restages() != 0 || act.reissues() != 0 { t.Fatalf("a silent box was acted on: restages=%d reissues=%d", act.restages(), act.reissues()) } } // SCENARIO C — a healthy box is a PURE no-op, repeatedly. No restage, no mint, no event. func TestScenarioC_HealthyBoxIsAPureNoOp(t *testing.T) { st := newHealStore(t) act := &fakeActions{restageResult: true} r := newRec(st, act) seedCustomer(t, st, "c1", true) for i := 1; i <= 5; i++ { pushReport(t, st, "c1", "", i) // healthy: enabled offsite object, no declaration r.ReconcileOnce(context.Background()) } if act.restages() != 0 || act.reissues() != 0 { t.Fatalf("a healthy box was acted on: restages=%d reissues=%d", act.restages(), act.reissues()) } if countEvents(t, st, "c1", eventRestaged)+countEvents(t, st, "c1", eventReissued) != 0 { t.Error("a healthy box produced a self-heal event") } } // SCENARIO E — nothing to re-arm escalates to a mint, ONCE, and does not repeat while the staged // secret sits unconsumed (a fresh report must re-confirm before acting again). func TestScenarioE_NothingToRestageEscalatesOnce(t *testing.T) { st := newHealStore(t) act := &fakeActions{restageResult: false} // NO stored secret r := newRec(st, act) seedCustomer(t, st, "c1", true) pushReport(t, st, "c1", StateNeedsCredential, 1) r.ReconcileOnce(context.Background()) pushReport(t, st, "c1", StateNeedsCredential, 2) r.ReconcileOnce(context.Background()) if act.reissues() != 1 { t.Fatalf("escalation did not happen exactly once: reissues=%d", act.reissues()) } if countEvents(t, st, "c1", eventReissued) != 1 { t.Error("no offsite_selfheal_reissued audit event") } // Repeated ticks on the SAME report must not mint again — this is the "a mint on every tick" // failure the sibling's resetAfterHeal exists to prevent. for i := 0; i < 5; i++ { r.ReconcileOnce(context.Background()) } if act.reissues() != 1 { t.Fatalf("minted repeatedly on an unchanged report: reissues=%d", act.reissues()) } } // SCENARIO F — a BLIP is absorbed: one declaring report followed by a healthy one triggers nothing. // // RED-PROOF: set debounceReports to 1 (i.e. remove the debounce) — the blip acts, and a credential is // churned by what was only a restart. func TestScenarioF_BlipIsAbsorbedByTheDebounce(t *testing.T) { st := newHealStore(t) act := &fakeActions{restageResult: true} r := newRec(st, act) seedCustomer(t, st, "c1", true) pushReport(t, st, "c1", StateNeedsCredential, 1) // the blip r.ReconcileOnce(context.Background()) pushReport(t, st, "c1", "", 2) // healthy again r.ReconcileOnce(context.Background()) if act.restages() != 0 || act.reissues() != 0 { t.Fatalf("a one-report blip triggered a credential action: restages=%d reissues=%d", act.restages(), act.reissues()) } // And the streak must have been FORGOTTEN, not merely paused: a single later declaration must // still not act. pushReport(t, st, "c1", StateNeedsCredential, 3) r.ReconcileOnce(context.Background()) if act.restages() != 0 { t.Fatal("the debounce streak survived a healthy report — a flapping box would be healed on every other cycle") } } // A DISABLED offsite descriptor is the operator's own choice and must never be re-credentialed, // even if the box declares (e.g. an old declaration left in the newest report). func TestDisabledDescriptorIsNeverHealed(t *testing.T) { st := newHealStore(t) act := &fakeActions{restageResult: true} r := newRec(st, act) seedCustomer(t, st, "c1", false) // offsite disabled in the config for i := 1; i <= 4; i++ { pushReport(t, st, "c1", StateNeedsCredential, i) r.ReconcileOnce(context.Background()) } if act.restages() != 0 || act.reissues() != 0 { t.Fatalf("a customer whose offsite is DISABLED was re-credentialed: restages=%d reissues=%d", act.restages(), act.reissues()) } } // A blocked customer is never healed. func TestBlockedCustomerIsNeverHealed(t *testing.T) { st := newHealStore(t) act := &fakeActions{restageResult: true} r := newRec(st, act) seedCustomer(t, st, "c1", true) // SaveCustomerConfig always writes status 'active' (its INSERT does not carry the column), so the // block must go through the real setter — asserted below, because a test that silently failed to // block would pass for the wrong reason. if err := st.SetCustomerConfigStatus("c1", "blocked"); err != nil { t.Fatalf("SetCustomerConfigStatus: %v", err) } if !st.IsCustomerBlocked("c1") { t.Fatal("precondition: the customer is not actually blocked — this test would pass vacuously") } for i := 1; i <= 4; i++ { pushReport(t, st, "c1", StateNeedsCredential, i) r.ReconcileOnce(context.Background()) } if act.restages() != 0 || act.reissues() != 0 { t.Fatalf("a BLOCKED customer was healed: restages=%d reissues=%d", act.restages(), act.reissues()) } } // A restage ERROR must not escalate to a mint — the reconciler does nothing and retries. Acting on a // failed read is the "never act on a partial view" rule. func TestRestageErrorDoesNotEscalate(t *testing.T) { st := newHealStore(t) act := &fakeActions{restageResult: true, restageErr: fmt.Errorf("db is busy")} r := newRec(st, act) seedCustomer(t, st, "c1", true) pushReport(t, st, "c1", StateNeedsCredential, 1) r.ReconcileOnce(context.Background()) pushReport(t, st, "c1", StateNeedsCredential, 2) r.ReconcileOnce(context.Background()) if act.reissues() != 0 { t.Fatalf("a failed re-stage escalated to an external mint: reissues=%d", act.reissues()) } if countEvents(t, st, "c1", eventRestaged) != 0 { t.Error("a failed re-stage recorded a success event") } }