package pbsdrheal // Non-hollow reconciler tests mapping 1:1 to the TASK's integration scenarios A–F. A REAL store // (t.TempDir sqlite) supplies the work-set + reports; a FAKE Actions seam counts restage/reissue // calls without any SSH/ep0. reconcileOnce is driven directly (same package) for determinism. Audit // events are asserted against the real store. Each test is the durable guard for its §10 red-proof. import ( "context" "io" "log" "path/filepath" "sync" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) type fakeActions struct { mu sync.Mutex restageHosts []string reissueCusts []string restageResult bool // what Restage returns (does a stored row exist?) restageErr error reissueErr error } func (f *fakeActions) Restage(hostID string) (bool, error) { f.mu.Lock() defer f.mu.Unlock() f.restageHosts = append(f.restageHosts, hostID) return f.restageResult, f.restageErr } func (f *fakeActions) Reissue(ctx context.Context, customerID string) error { f.mu.Lock() defer f.mu.Unlock() f.reissueCusts = append(f.reissueCusts, customerID) return f.reissueErr } func (f *fakeActions) restages() int { f.mu.Lock(); defer f.mu.Unlock(); return len(f.restageHosts) } func (f *fakeActions) reissues() int { f.mu.Lock(); defer f.mu.Unlock(); return len(f.reissueCusts) } func newHealStore(t *testing.T) *store.Store { t.Helper() st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0)) if err != nil { t.Fatalf("store.New: %v", err) } t.Cleanup(func() { st.Close() }) return st } // seedHost writes a host with a pbs_dr descriptor of the requested enable/provision shape. func seedHost(t *testing.T, st *store.Store, hostID, customerID string, enabled, provisioned bool) { t.Helper() if err := st.UpsertHost(&store.Host{HostID: hostID, CustomerID: customerID, APIKey: "k-" + hostID}); err != nil { t.Fatalf("UpsertHost %s: %v", hostID, err) } ns := "" if provisioned { ns = customerID } desc := `{"pbs_dr":{"enabled":` + boolStr(enabled) + `,"namespace":"` + ns + `","storage_id":"felhom-pbs"}}` if _, err := st.SetHostDesired(hostID, []byte(desc)); err != nil { t.Fatalf("SetHostDesired %s: %v", hostID, err) } } func seedReport(t *testing.T, st *store.Store, hostID, customerID, state string) { t.Helper() if err := st.SaveHostReport(hostID, customerID, []byte(`{"pbs_dr":{"state":"`+state+`"}}`), store.HostReportDenorm{AgentVersion: "0.88.0"}); err != nil { t.Fatalf("SaveHostReport %s: %v", hostID, err) } } // ageSecret back-dates a staged secret's created_at so grace-window behaviour is testable without // sleeping. Writes the house SQLite datetime format. func ageSecret(t *testing.T, st *store.Store, hostID string, by time.Duration) { t.Helper() when := time.Now().UTC().Add(-by).Format("2006-01-02 15:04:05") if err := st.SetHostPBSSecretCreatedAtForTest(hostID, when); err != nil { t.Fatalf("age secret %s: %v", hostID, err) } } func boolStr(b bool) string { if b { return "true" } return "false" } func newRec(st *store.Store, act Actions) *Reconciler { r := NewReconciler(st, act, log.New(io.Discard, "", 0)) r.debounceReports = 1 // fire on the first stuck report unless a test overrides return r } func genOf(t *testing.T, st *store.Store, hostID string) int64 { t.Helper() h, err := st.GetHost(hostID) if err != nil || h == nil { t.Fatalf("GetHost %s: (%v, %v)", hostID, h, err) } return h.DesiredGeneration } func eventTypes(t *testing.T, st *store.Store, customerID string) []string { t.Helper() evs, err := st.GetRecentEvents(customerID, 50) if err != nil { t.Fatalf("GetRecentEvents: %v", err) } var out []string for _, e := range evs { out = append(out, e.EventType) } return out } // A — the headline: a re-installed box (descriptor enabled+provisioned, stored CONSUMED secret, // waiting_secret) is re-staged — NOT reissued, and NO generation bump. func TestScenarioA_WaitingSecretRestages(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hA", "cA", true, true) seedReport(t, st, "hA", "cA", stateWaitingSecret) genBefore := genOf(t, st, "hA") fake := &fakeActions{restageResult: true} // a stored secret exists r := newRec(st, fake) r.reconcileOnce(context.Background()) if fake.restages() != 1 || fake.reissues() != 0 { t.Fatalf("restages=%d reissues=%d, want 1/0 (re-stage, not re-issue)", fake.restages(), fake.reissues()) } if fake.restageHosts[0] != "hA" { t.Errorf("restaged host = %s, want hA", fake.restageHosts[0]) } if g := genOf(t, st, "hA"); g != genBefore { t.Errorf("generation bumped by a re-stage heal: %d -> %d", genBefore, g) } if got := eventTypes(t, st, "cA"); len(got) != 1 || got[0] != eventRestaged { t.Errorf("events = %v, want [%s]", got, eventRestaged) } } // B — no stored secret → escalate to Re-issue exactly once. func TestScenarioB_NoStoredSecretReissues(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hB", "cB", true, true) seedReport(t, st, "hB", "cB", stateWaitingSecret) fake := &fakeActions{restageResult: false} // no row to re-stage r := newRec(st, fake) r.reconcileOnce(context.Background()) if fake.restages() != 1 || fake.reissues() != 1 { t.Fatalf("restages=%d reissues=%d, want 1/1 (tried re-stage, then escalated)", fake.restages(), fake.reissues()) } if fake.reissueCusts[0] != "cB" { t.Errorf("reissued customer = %s, want cB", fake.reissueCusts[0]) } if got := eventTypes(t, st, "cB"); len(got) != 1 || got[0] != eventReissued { t.Errorf("events = %v, want [%s]", got, eventReissued) } } // C — a converged/healthy host is a pure no-op (idempotency; the gen-thrash guard). func TestScenarioC_ConvergedHostNoOp(t *testing.T) { for _, state := range []string{"applied", "adopted", "disabled", "verify_failed"} { t.Run(state, func(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hC", "cC", true, true) seedReport(t, st, "hC", "cC", state) genBefore := genOf(t, st, "hC") fake := &fakeActions{restageResult: true} r := newRec(st, fake) r.reconcileOnce(context.Background()) if fake.restages() != 0 || fake.reissues() != 0 { t.Fatalf("state %s: restages=%d reissues=%d, want 0/0 (no-op)", state, fake.restages(), fake.reissues()) } if g := genOf(t, st, "hC"); g != genBefore { t.Errorf("state %s: generation changed on a healthy host", state) } if got := eventTypes(t, st, "cC"); len(got) != 0 { t.Errorf("state %s: events = %v, want none", state, got) } }) } } // D — debounce: a single waiting_secret report does NOT heal; a second DISTINCT one does. func TestScenarioD_DebounceRequiresTwoDistinctReports(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hD", "cD", true, true) fake := &fakeActions{restageResult: true} r := NewReconciler(st, fake, log.New(io.Discard, "", 0)) // default debounceReports = 2 seedReport(t, st, "hD", "cD", stateWaitingSecret) // report #1 r.reconcileOnce(context.Background()) if fake.restages() != 0 { t.Fatalf("healed on the FIRST waiting_secret report (restages=%d) — debounce broken", fake.restages()) } // Re-observing the SAME report must also not advance the debounce. r.reconcileOnce(context.Background()) if fake.restages() != 0 { t.Fatalf("healed on a RE-OBSERVED same report (restages=%d) — debounce must count distinct reports", fake.restages()) } seedReport(t, st, "hD", "cD", stateWaitingSecret) // report #2 (distinct) r.reconcileOnce(context.Background()) if fake.restages() != 1 { t.Fatalf("did not heal after 2 distinct waiting_secret reports (restages=%d)", fake.restages()) } } // E — consumed_failed is escalated (Re-issue), never re-staged (a re-stage re-feeds a burned secret). func TestScenarioE_ConsumedFailedEscalates(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hE", "cE", true, true) seedReport(t, st, "hE", "cE", stateConsumedFailed) fake := &fakeActions{restageResult: true} // even if a secret were re-stageable, must NOT re-stage r := newRec(st, fake) r.reconcileOnce(context.Background()) if fake.restages() != 0 || fake.reissues() != 1 { t.Fatalf("restages=%d reissues=%d, want 0/1 (consumed_failed → re-issue only)", fake.restages(), fake.reissues()) } if got := eventTypes(t, st, "cE"); len(got) != 1 || got[0] != eventConsumedFailed { t.Errorf("events = %v, want [%s] (distinct consumed_failed signal)", got, eventConsumedFailed) } } // F — DR-OFF (disabled descriptor) and enabled-but-unprovisioned hosts are never in the work set. func TestScenarioF_OutOfScopeHostsUntouched(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hOff", "cOff", false, true) // descriptor disabled seedReport(t, st, "hOff", "cOff", stateWaitingSecret) seedHost(t, st, "hUnprov", "cUnprov", true, false) // enabled but never provisioned seedReport(t, st, "hUnprov", "cUnprov", stateWaitingSecret) fake := &fakeActions{restageResult: true} r := newRec(st, fake) r.reconcileOnce(context.Background()) if fake.restages() != 0 || fake.reissues() != 0 { t.Fatalf("out-of-scope hosts acted on: restages=%d reissues=%d, want 0/0", fake.restages(), fake.reissues()) } } // RestrictToHost scopes the work set: a stuck host OTHER than the allowed one is untouched. func TestRestrictToHost_ScopesWorkSet(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hAllowed", "cAllowed", true, true) seedReport(t, st, "hAllowed", "cAllowed", stateWaitingSecret) seedHost(t, st, "hOther", "cOther", true, true) seedReport(t, st, "hOther", "cOther", stateWaitingSecret) fake := &fakeActions{restageResult: true} r := newRec(st, fake) r.RestrictToHost("hAllowed") r.reconcileOnce(context.Background()) if fake.restages() != 1 || fake.restageHosts[0] != "hAllowed" { t.Fatalf("restaged hosts = %v, want exactly [hAllowed] (scope restriction)", fake.restageHosts) } } // After a heal, the SAME stuck report must not re-heal on the next tick (only a fresh report does). func TestNoReHealOnSameReport(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hR", "cR", true, true) seedReport(t, st, "hR", "cR", stateWaitingSecret) fake := &fakeActions{restageResult: true} r := newRec(st, fake) r.reconcileOnce(context.Background()) r.reconcileOnce(context.Background()) // same report, no new evidence if fake.restages() != 1 { t.Fatalf("re-healed the same stuck report: restages=%d, want 1", fake.restages()) } } // R-39 — auth_failed is escalated to a fresh mint, never re-staged. // // This is the leg that closes the loop end to end: the agent (>=0.91.0) now PROVES the credential is // rejected instead of silently skipping, and the hub re-keys, which advances the secret generation, // which moves the descriptor hash, which finally re-arms the agent. A re-stage would re-feed the very // secret PBS just rejected. // // COMPANION RED-PROOF (run + recorded): delete the `case stateAuthFailed` arm → the state falls to // the default no-op branch and this test FAILS with reissues=0. func TestR39_AuthFailedEscalatesToReissue(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hAF", "cAF", true, true) seedReport(t, st, "hAF", "cAF", stateAuthFailed) fake := &fakeActions{restageResult: true} // a re-stageable secret exists — it must STILL not be used r := newRec(st, fake) r.reconcileOnce(context.Background()) if fake.restages() != 0 { t.Errorf("restages=%d, want 0 — re-staging re-feeds the credential PBS just rejected", fake.restages()) } if fake.reissues() != 1 { t.Fatalf("reissues=%d, want 1 — an applied-and-401 tier must be re-keyed, not left green", fake.reissues()) } if got := eventTypes(t, st, "cAF"); len(got) != 1 || got[0] != eventAuthFailed { t.Errorf("events = %v, want [%s] (distinct auth_failed signal, not borrowed from consumed_failed)", got, eventAuthFailed) } } // The damper is the burn protection: a 401 flap must not become a secret-minting chain. auth_failed // uses the SAME confirm() debounce as the other triggers — one report is not enough. func TestR39_AuthFailedIsDebounced(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hAF2", "cAF2", true, true) fake := &fakeActions{restageResult: true} r := newRec(st, fake) r.debounceReports = 2 // require two DISTINCT stuck reports seedReport(t, st, "hAF2", "cAF2", stateAuthFailed) r.reconcileOnce(context.Background()) if fake.reissues() != 0 { t.Fatalf("re-issued on the FIRST auth_failed report (reissues=%d) — the damper is bypassed", fake.reissues()) } seedReport(t, st, "hAF2", "cAF2", stateAuthFailed) // a second, distinct report r.reconcileOnce(context.Background()) if fake.reissues() != 1 { t.Fatalf("reissues=%d after two distinct auth_failed reports, want 1", fake.reissues()) } // And the same confirmed report must not re-heal on the next tick. r.reconcileOnce(context.Background()) if fake.reissues() != 1 { t.Errorf("re-healed on a stale report: reissues=%d, want 1", fake.reissues()) } } // A host that recovers (auth_failed → applied) is a pure no-op, and its streak is forgotten. func TestR39_AuthFailedRecoveryIsNoOp(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hAF3", "cAF3", true, true) fake := &fakeActions{restageResult: true} r := newRec(st, fake) r.debounceReports = 2 // so a single stuck report is not yet actionable seedReport(t, st, "hAF3", "cAF3", stateAuthFailed) r.reconcileOnce(context.Background()) // streak 1 — below the damper, nothing done seedReport(t, st, "hAF3", "cAF3", "applied") // the box healed on its own r.reconcileOnce(context.Background()) if fake.reissues() != 0 || fake.restages() != 0 { t.Errorf("a recovered host was acted on: restages=%d reissues=%d, want 0/0", fake.restages(), fake.reissues()) } // The streak must be FORGOTTEN, not merely paused: a later single auth_failed must start over // rather than instantly tripping the damper it had half-filled before recovering. seedReport(t, st, "hAF3", "cAF3", stateAuthFailed) r.reconcileOnce(context.Background()) if fake.reissues() != 0 { t.Errorf("a stale pre-recovery streak survived and fired early: reissues=%d, want 0", fake.reissues()) } } // R-39 Scenario F — the consumed_at HONESTY gauge. // // The July-18 fingerprint is a disagreement no single tier can see: the hub staged a credential, the // box never took it, and the box nonetheless reports `applied`. This surfaces it. It deliberately // does NOT auto-heal — minting on top of an unconsumed secret is the mint/consume race R-39(a) // already recorded. // // COMPANION RED-PROOF (run + recorded): delete the r.checkUnconsumed(row) call → no event is written // and TestR39_UnconsumedSecretUnderAppliedIsSurfaced FAILS. func TestR39_UnconsumedSecretUnderAppliedIsSurfaced(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hU", "cU", true, true) seedReport(t, st, "hU", "cU", "applied") // the box claims converged if _, err := st.SaveHostPBSSecret("hU", "never-taken"); err != nil { t.Fatalf("mint: %v", err) } // Age the staged secret past the grace window. ageSecret(t, st, "hU", 3*time.Hour) fake := &fakeActions{restageResult: true} r := newRec(st, fake) r.reconcileOnce(context.Background()) if got := eventTypes(t, st, "cU"); len(got) != 1 || got[0] != eventUnconsumed { t.Fatalf("events = %v, want [%s]", got, eventUnconsumed) } // SURFACE, not heal: an applied box is not in any heal arm, so nothing was minted or re-staged. if fake.reissues() != 0 || fake.restages() != 0 { t.Errorf("the honesty gauge must not mutate: restages=%d reissues=%d, want 0/0", fake.restages(), fake.reissues()) } // One event per DISTINCT report — a sustained disagreement must not spam the audit log. r.reconcileOnce(context.Background()) if got := eventTypes(t, st, "cU"); len(got) != 1 { t.Errorf("re-surfaced on the same report: %d events, want 1", len(got)) } } // A freshly staged secret (or a just-restaged one) is NORMAL inside the grace window and must not // alarm — RestageHostPBSSecret deliberately sets consumed_at back to NULL. func TestR39_FreshlyStagedSecretDoesNotAlarm(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hF", "cF", true, true) seedReport(t, st, "hF", "cF", "applied") if _, err := st.SaveHostPBSSecret("hF", "just-minted"); err != nil { t.Fatalf("mint: %v", err) } // A deliberate re-stage clears consumed_at — the Scenario-F edge case. if _, err := st.ConsumeHostPBSSecret("hF"); err != nil { t.Fatalf("consume: %v", err) } if _, err := st.RestageHostPBSSecret("hF"); err != nil { t.Fatalf("restage: %v", err) } r := newRec(st, &fakeActions{restageResult: true}) r.reconcileOnce(context.Background()) if got := eventTypes(t, st, "cF"); len(got) != 0 { t.Errorf("a just-staged secret alarmed inside the grace window: %v", got) } } // A CONSUMED secret never alarms, however old. func TestR39_ConsumedSecretNeverAlarms(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hC", "cC", true, true) seedReport(t, st, "hC", "cC", "applied") if _, err := st.SaveHostPBSSecret("hC", "taken"); err != nil { t.Fatalf("mint: %v", err) } if _, err := st.ConsumeHostPBSSecret("hC"); err != nil { t.Fatalf("consume: %v", err) } ageSecret(t, st, "hC", 30*24*time.Hour) r := newRec(st, &fakeActions{restageResult: true}) r.reconcileOnce(context.Background()) if got := eventTypes(t, st, "cC"); len(got) != 0 { t.Errorf("a consumed secret alarmed: %v", got) } } // A box that HONESTLY reports a stuck state is already being healed; its unconsumed secret is the // symptom, not a contradiction, so the gauge stays quiet and only the heal event is written. func TestR39_HonestStuckStateDoesNotDoubleReport(t *testing.T) { st := newHealStore(t) seedHost(t, st, "hH", "cH", true, true) seedReport(t, st, "hH", "cH", stateWaitingSecret) if _, err := st.SaveHostPBSSecret("hH", "staged"); err != nil { t.Fatalf("mint: %v", err) } ageSecret(t, st, "hH", 3*time.Hour) r := newRec(st, &fakeActions{restageResult: true}) r.reconcileOnce(context.Background()) for _, e := range eventTypes(t, st, "cH") { if e == eventUnconsumed { t.Error("an honestly-stuck box must not also raise the honesty gauge — it is the symptom being healed") } } }