hub v0.68.0 — auth_failed self-heal, consumed_at honesty gauge, wrapper drift (R-39 + R-50b(a))

Completes the hub half of R-39's fleet fix on top of the generation core (c484aa2). pbsdrheal gains an auth_failed TRIGGER — a new trigger in the existing machine, not a new machine. A box whose credential PBS rejects escalates to a fresh mint, never a re-stage (which would re-feed the secret PBS just rejected), through the EXISTING damper: a 401 flap must not become a secret-minting chain. With the generation stamp this closes the loop end to end — agent proves the 401, hub re-keys, generation advances, descriptor hash moves, agent re-consumes. consumed_at honesty gauge: a staged secret still unconsumed past a 15-minute grace while the box reports `applied` is surfaced with its own event. That is the exact 2026-07-18 fingerprint and a disagreement no single tier can see alone. Deliberately a SURFACE, not a heal — auto-re-issuing on it would mint a second secret on top of an unconsumed one, which is the mint/consume race R-39(a) already recorded. One event per distinct report, and an honestly-stuck box does not double-report (its unconsumed secret is the symptom being healed, not a contradiction). R-50b(a): ArtifactManifest.WrapperSHA256 + operator field + host-page drift surface. The PBS wrapper is root-owned 0755 and the pinned sudoers vector, yet installed unversioned from raw/branch/main and absent from every manifest. Agents >=0.91.0 report the installed hash; a mismatch is surfaced. An unknown on EITHER side reads as quiet, never as drift — lighting every host amber on rollout day is how a warning becomes background noise. The delivery channel itself stays R-50b(b)/(c). Compatibility unchanged: safe for 0.90.0 agents (unknown JSON key dropped); the re-arm and auth-honesty guarantees need agent >=0.91.0, so MinAgent moves only after the fleet has self-updated. Tests: auth_failed escalate/debounce/recovery-forgets-streak; honesty gauge incl. grace window, the restage edge (consumed_at deliberately NULLed), consumed-never-alarms, and honest-stuck-no-double-report; wrapper drift incl. both unknown directions. Red-proof run at the assertion level: removing the auth_failed arm fails the escalation tests with reissues=0.
2026-07-21 10:01:35 +02:00
parent c484aa204e
commit 107f74ea3c
10 changed files with 561 additions and 38 deletions
@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"

 	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
 )
@@ -75,6 +76,16 @@ func seedReport(t *testing.T, st *store.Store, hostID, customerID, state string)
 	}
 }

+// ageSecret back-dates a staged secret's created_at so grace-window behaviour is testable without
+// sleeping. Writes the house SQLite datetime format.
+func ageSecret(t *testing.T, st *store.Store, hostID string, by time.Duration) {
+	t.Helper()
+	when := time.Now().UTC().Add(-by).Format("2006-01-02 15:04:05")
+	if err := st.SetHostPBSSecretCreatedAtForTest(hostID, when); err != nil {
+		t.Fatalf("age secret %s: %v", hostID, err)
+	}
+}
+
 func boolStr(b bool) string {
 	if b {
 		return "true"
@@ -228,7 +239,7 @@ func TestScenarioE_ConsumedFailedEscalates(t *testing.T) {
 // F — DR-OFF (disabled descriptor) and enabled-but-unprovisioned hosts are never in the work set.
 func TestScenarioF_OutOfScopeHostsUntouched(t *testing.T) {
 	st := newHealStore(t)
-	seedHost(t, st, "hOff", "cOff", false, true)   // descriptor disabled
+	seedHost(t, st, "hOff", "cOff", false, true) // descriptor disabled
 	seedReport(t, st, "hOff", "cOff", stateWaitingSecret)
 	seedHost(t, st, "hUnprov", "cUnprov", true, false) // enabled but never provisioned
 	seedReport(t, st, "hUnprov", "cUnprov", stateWaitingSecret)
@@ -274,3 +285,190 @@ func TestNoReHealOnSameReport(t *testing.T) {
 		t.Fatalf("re-healed the same stuck report: restages=%d, want 1", fake.restages())
 	}
 }
+
+// R-39 — auth_failed is escalated to a fresh mint, never re-staged.
+//
+// This is the leg that closes the loop end to end: the agent (>=0.91.0) now PROVES the credential is
+// rejected instead of silently skipping, and the hub re-keys, which advances the secret generation,
+// which moves the descriptor hash, which finally re-arms the agent. A re-stage would re-feed the very
+// secret PBS just rejected.
+//
+// COMPANION RED-PROOF (run + recorded): delete the `case stateAuthFailed` arm → the state falls to
+// the default no-op branch and this test FAILS with reissues=0.
+func TestR39_AuthFailedEscalatesToReissue(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hAF", "cAF", true, true)
+	seedReport(t, st, "hAF", "cAF", stateAuthFailed)
+	fake := &fakeActions{restageResult: true} // a re-stageable secret exists — it must STILL not be used
+	r := newRec(st, fake)
+
+	r.reconcileOnce(context.Background())
+
+	if fake.restages() != 0 {
+		t.Errorf("restages=%d, want 0 — re-staging re-feeds the credential PBS just rejected", fake.restages())
+	}
+	if fake.reissues() != 1 {
+		t.Fatalf("reissues=%d, want 1 — an applied-and-401 tier must be re-keyed, not left green", fake.reissues())
+	}
+	if got := eventTypes(t, st, "cAF"); len(got) != 1 || got[0] != eventAuthFailed {
+		t.Errorf("events = %v, want [%s] (distinct auth_failed signal, not borrowed from consumed_failed)", got, eventAuthFailed)
+	}
+}
+
+// The damper is the burn protection: a 401 flap must not become a secret-minting chain. auth_failed
+// uses the SAME confirm() debounce as the other triggers — one report is not enough.
+func TestR39_AuthFailedIsDebounced(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hAF2", "cAF2", true, true)
+	fake := &fakeActions{restageResult: true}
+	r := newRec(st, fake)
+	r.debounceReports = 2 // require two DISTINCT stuck reports
+
+	seedReport(t, st, "hAF2", "cAF2", stateAuthFailed)
+	r.reconcileOnce(context.Background())
+	if fake.reissues() != 0 {
+		t.Fatalf("re-issued on the FIRST auth_failed report (reissues=%d) — the damper is bypassed", fake.reissues())
+	}
+
+	seedReport(t, st, "hAF2", "cAF2", stateAuthFailed) // a second, distinct report
+	r.reconcileOnce(context.Background())
+	if fake.reissues() != 1 {
+		t.Fatalf("reissues=%d after two distinct auth_failed reports, want 1", fake.reissues())
+	}
+
+	// And the same confirmed report must not re-heal on the next tick.
+	r.reconcileOnce(context.Background())
+	if fake.reissues() != 1 {
+		t.Errorf("re-healed on a stale report: reissues=%d, want 1", fake.reissues())
+	}
+}
+
+// A host that recovers (auth_failed → applied) is a pure no-op, and its streak is forgotten.
+func TestR39_AuthFailedRecoveryIsNoOp(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hAF3", "cAF3", true, true)
+	fake := &fakeActions{restageResult: true}
+	r := newRec(st, fake)
+	r.debounceReports = 2 // so a single stuck report is not yet actionable
+
+	seedReport(t, st, "hAF3", "cAF3", stateAuthFailed)
+	r.reconcileOnce(context.Background())        // streak 1 — below the damper, nothing done
+	seedReport(t, st, "hAF3", "cAF3", "applied") // the box healed on its own
+	r.reconcileOnce(context.Background())
+
+	if fake.reissues() != 0 || fake.restages() != 0 {
+		t.Errorf("a recovered host was acted on: restages=%d reissues=%d, want 0/0", fake.restages(), fake.reissues())
+	}
+
+	// The streak must be FORGOTTEN, not merely paused: a later single auth_failed must start over
+	// rather than instantly tripping the damper it had half-filled before recovering.
+	seedReport(t, st, "hAF3", "cAF3", stateAuthFailed)
+	r.reconcileOnce(context.Background())
+	if fake.reissues() != 0 {
+		t.Errorf("a stale pre-recovery streak survived and fired early: reissues=%d, want 0", fake.reissues())
+	}
+}
+
+// R-39 Scenario F — the consumed_at HONESTY gauge.
+//
+// The July-18 fingerprint is a disagreement no single tier can see: the hub staged a credential, the
+// box never took it, and the box nonetheless reports `applied`. This surfaces it. It deliberately
+// does NOT auto-heal — minting on top of an unconsumed secret is the mint/consume race R-39(a)
+// already recorded.
+//
+// COMPANION RED-PROOF (run + recorded): delete the r.checkUnconsumed(row) call → no event is written
+// and TestR39_UnconsumedSecretUnderAppliedIsSurfaced FAILS.
+func TestR39_UnconsumedSecretUnderAppliedIsSurfaced(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hU", "cU", true, true)
+	seedReport(t, st, "hU", "cU", "applied") // the box claims converged
+	if _, err := st.SaveHostPBSSecret("hU", "never-taken"); err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	// Age the staged secret past the grace window.
+	ageSecret(t, st, "hU", 3*time.Hour)
+
+	fake := &fakeActions{restageResult: true}
+	r := newRec(st, fake)
+	r.reconcileOnce(context.Background())
+
+	if got := eventTypes(t, st, "cU"); len(got) != 1 || got[0] != eventUnconsumed {
+		t.Fatalf("events = %v, want [%s]", got, eventUnconsumed)
+	}
+	// SURFACE, not heal: an applied box is not in any heal arm, so nothing was minted or re-staged.
+	if fake.reissues() != 0 || fake.restages() != 0 {
+		t.Errorf("the honesty gauge must not mutate: restages=%d reissues=%d, want 0/0", fake.restages(), fake.reissues())
+	}
+
+	// One event per DISTINCT report — a sustained disagreement must not spam the audit log.
+	r.reconcileOnce(context.Background())
+	if got := eventTypes(t, st, "cU"); len(got) != 1 {
+		t.Errorf("re-surfaced on the same report: %d events, want 1", len(got))
+	}
+}
+
+// A freshly staged secret (or a just-restaged one) is NORMAL inside the grace window and must not
+// alarm — RestageHostPBSSecret deliberately sets consumed_at back to NULL.
+func TestR39_FreshlyStagedSecretDoesNotAlarm(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hF", "cF", true, true)
+	seedReport(t, st, "hF", "cF", "applied")
+	if _, err := st.SaveHostPBSSecret("hF", "just-minted"); err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	// A deliberate re-stage clears consumed_at — the Scenario-F edge case.
+	if _, err := st.ConsumeHostPBSSecret("hF"); err != nil {
+		t.Fatalf("consume: %v", err)
+	}
+	if _, err := st.RestageHostPBSSecret("hF"); err != nil {
+		t.Fatalf("restage: %v", err)
+	}
+
+	r := newRec(st, &fakeActions{restageResult: true})
+	r.reconcileOnce(context.Background())
+
+	if got := eventTypes(t, st, "cF"); len(got) != 0 {
+		t.Errorf("a just-staged secret alarmed inside the grace window: %v", got)
+	}
+}
+
+// A CONSUMED secret never alarms, however old.
+func TestR39_ConsumedSecretNeverAlarms(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hC", "cC", true, true)
+	seedReport(t, st, "hC", "cC", "applied")
+	if _, err := st.SaveHostPBSSecret("hC", "taken"); err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	if _, err := st.ConsumeHostPBSSecret("hC"); err != nil {
+		t.Fatalf("consume: %v", err)
+	}
+	ageSecret(t, st, "hC", 30*24*time.Hour)
+
+	r := newRec(st, &fakeActions{restageResult: true})
+	r.reconcileOnce(context.Background())
+	if got := eventTypes(t, st, "cC"); len(got) != 0 {
+		t.Errorf("a consumed secret alarmed: %v", got)
+	}
+}
+
+// A box that HONESTLY reports a stuck state is already being healed; its unconsumed secret is the
+// symptom, not a contradiction, so the gauge stays quiet and only the heal event is written.
+func TestR39_HonestStuckStateDoesNotDoubleReport(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hH", "cH", true, true)
+	seedReport(t, st, "hH", "cH", stateWaitingSecret)
+	if _, err := st.SaveHostPBSSecret("hH", "staged"); err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	ageSecret(t, st, "hH", 3*time.Hour)
+
+	r := newRec(st, &fakeActions{restageResult: true})
+	r.reconcileOnce(context.Background())
+
+	for _, e := range eventTypes(t, st, "cH") {
+		if e == eventUnconsumed {
+			t.Error("an honestly-stuck box must not also raise the honesty gauge — it is the symptom being healed")
+		}
+	}
+}