hub v0.68.0 — auth_failed self-heal, consumed_at honesty gauge, wrapper drift (R-39 + R-50b(a))

Completes the hub half of R-39's fleet fix on top of the generation core (c484aa2). pbsdrheal gains an auth_failed TRIGGER — a new trigger in the existing machine, not a new machine. A box whose credential PBS rejects escalates to a fresh mint, never a re-stage (which would re-feed the secret PBS just rejected), through the EXISTING damper: a 401 flap must not become a secret-minting chain. With the generation stamp this closes the loop end to end — agent proves the 401, hub re-keys, generation advances, descriptor hash moves, agent re-consumes. consumed_at honesty gauge: a staged secret still unconsumed past a 15-minute grace while the box reports `applied` is surfaced with its own event. That is the exact 2026-07-18 fingerprint and a disagreement no single tier can see alone. Deliberately a SURFACE, not a heal — auto-re-issuing on it would mint a second secret on top of an unconsumed one, which is the mint/consume race R-39(a) already recorded. One event per distinct report, and an honestly-stuck box does not double-report (its unconsumed secret is the symptom being healed, not a contradiction). R-50b(a): ArtifactManifest.WrapperSHA256 + operator field + host-page drift surface. The PBS wrapper is root-owned 0755 and the pinned sudoers vector, yet installed unversioned from raw/branch/main and absent from every manifest. Agents >=0.91.0 report the installed hash; a mismatch is surfaced. An unknown on EITHER side reads as quiet, never as drift — lighting every host amber on rollout day is how a warning becomes background noise. The delivery channel itself stays R-50b(b)/(c). Compatibility unchanged: safe for 0.90.0 agents (unknown JSON key dropped); the re-arm and auth-honesty guarantees need agent >=0.91.0, so MinAgent moves only after the fleet has self-updated. Tests: auth_failed escalate/debounce/recovery-forgets-streak; honesty gauge incl. grace window, the restage edge (consumed_at deliberately NULLed), consumed-never-alarms, and honest-stuck-no-double-report; wrapper drift incl. both unknown directions. Red-proof run at the assertion level: removing the auth_failed arm fails the escalation tests with reissues=0.
2026-07-21 10:01:35 +02:00
parent c484aa204e
commit 107f74ea3c
10 changed files with 561 additions and 38 deletions
@@ -33,8 +33,15 @@ import (
 // (the secret is untouched by verify-before-consume), self-heals when the tunnel recovers, and must
 // stay LOUD for the operator if it does not — re-staging a secret would not help it.
 const (
-	stateWaitingSecret = "waiting_secret"
+	stateWaitingSecret  = "waiting_secret"
 	stateConsumedFailed = "consumed_failed"
+	// stateAuthFailed (R-39, agent >= 0.91.0) — the box HAS a credential, the descriptor is applied,
+	// and PBS rejects it with 401. Before 0.91.0 this state could not exist: the agent's verify loop
+	// read the credential file directly as non-root, always failed with "permission denied", and
+	// skipped — so an applied-and-dead tier was invisible to both tiers. It is healed like
+	// consumed_failed (escalate to a fresh mint), never by a re-stage: re-staging re-feeds the SAME
+	// secret PBS just rejected.
+	stateAuthFailed = "auth_failed"
 )

 // Audit event types (store.SaveEvent; hub-internal, not gated by allowedEventTypes). Distinct per
@@ -43,8 +50,18 @@ const (
 	eventRestaged       = "pbsdr_selfheal_restaged"        // re-armed the stored secret (routine)
 	eventReissued       = "pbsdr_selfheal_reissued"        // no stored secret → minted a fresh one
 	eventConsumedFailed = "pbsdr_selfheal_consumed_failed" // burned secret → minted a fresh one (a real problem was remediated)
+	eventAuthFailed     = "pbsdr_selfheal_auth_failed"     // PBS rejected the box's credential (401) → minted a fresh one
+	eventUnconsumed     = "pbsdr_unconsumed_secret"        // staged secret never consumed under an `applied` box (surfaced, NOT auto-healed)
 )

+// unconsumedGrace bounds how long a staged-but-unconsumed secret is NORMAL before it is a lie.
+//
+// A fresh mint (or a deliberate re-stage) is consumed on the agent's next tick — well inside one
+// ~15-minute report cycle. Beyond this window, an unconsumed secret sitting under a box that reports
+// `applied` is the exact fingerprint of the 2026-07-18 N100 failure: hub minted, agent
+// short-circuited, both tiers green, box serving a revoked credential.
+const unconsumedGrace = 15 * time.Minute
+
 // Actions is the mutation seam — fakes in tests count calls without SSH/ep0. Restage flips a stored
 // secret's consumed flag (returns restaged=false when NO row exists → the caller escalates). Reissue
 // mints a fresh ep0 token + stores a fresh consume-once secret + bumps the descriptor.
@@ -71,7 +88,9 @@ func (a storeActions) Reissue(ctx context.Context, customerID string) error {
 }

 // NewActions builds the production mutation seam.
-func NewActions(st *store.Store, reissuer Reissuer) Actions { return storeActions{st: st, reissuer: reissuer} }
+func NewActions(st *store.Store, reissuer Reissuer) Actions {
+	return storeActions{st: st, reissuer: reissuer}
+}

 // debounceState tracks, per host, the last DISTINCT report observed and how many consecutive
 // distinct reports it has held the current stuck state — so a fresh box that briefly shows
@@ -86,15 +105,18 @@ type debounceState struct {
 // Reconciler re-arms stuck PBS-DR hosts. DECLARATIVE + IDEMPOTENT: a tick over a converged fleet
 // writes nothing (Scenario C). It reads the hub DB (the source of truth) — never the box.
 type Reconciler struct {
-	store    *store.Store
-	act      Actions
-	interval time.Duration
-	debounceReports int // distinct stuck reports required before healing (default 2)
-	onlyHost string // "" = whole fleet; non-empty restricts the work set to one host (supervised rollout)
-	trigger  chan struct{}
-	logger   *log.Logger
+	store           *store.Store
+	act             Actions
+	interval        time.Duration
+	debounceReports int    // distinct stuck reports required before healing (default 2)
+	onlyHost        string // "" = whole fleet; non-empty restricts the work set to one host (supervised rollout)
+	trigger         chan struct{}
+	logger          *log.Logger

 	deb map[string]debounceState
+	// unconsumedSeen remembers the last report id already surfaced per host (Scenario F), so a
+	// sustained disagreement produces one event per fresh report rather than one per tick.
+	unconsumedSeen map[string]int64
 }

 // NewReconciler builds the reconciler. interval defaults to 5m, debounceReports to 2.
@@ -110,6 +132,7 @@ func NewReconciler(st *store.Store, act Actions, logger *log.Logger) *Reconciler
 		trigger:         make(chan struct{}, 1),
 		logger:          logger,
 		deb:             map[string]debounceState{},
+		unconsumedSeen:  map[string]int64{},
 	}
 }

@@ -161,6 +184,14 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) {
 			delete(r.deb, row.HostID)
 			continue
 		}
+		// R-39 consumed_at HONESTY (Scenario F). Deliberately a SURFACE, not a heal: the remediation
+		// for a genuinely stuck box is the auth_failed / waiting_secret / consumed_failed machinery
+		// above, driven by what the BOX reports. This check catches the disagreement itself — the hub
+		// staged a credential the box never took, while the box claims to be applied — which is a
+		// state no single tier can detect alone. Auto-re-issuing on it would mint a second secret on
+		// top of an unconsumed one: exactly the mint/consume race R-39(a) already recorded.
+		r.checkUnconsumed(row)
+
 		switch row.ReportedState {
 		case stateWaitingSecret:
 			if r.confirm(row) {
@@ -170,6 +201,10 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) {
 			if r.confirm(row) {
 				r.healConsumedFailed(ctx, row)
 			}
+		case stateAuthFailed:
+			if r.confirm(row) {
+				r.healAuthFailed(ctx, row)
+			}
 		default:
 			// applied | adopted | disabled | verify_failed | "" (no report) | anything else → no-op.
 			delete(r.deb, row.HostID)
@@ -183,6 +218,31 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) {
 	}
 }

+// checkUnconsumed surfaces the applied-but-never-consumed disagreement (Scenario F). One event per
+// distinct report, so a sustained state does not spam the audit log every tick.
+func (r *Reconciler) checkUnconsumed(row store.PBSDRHealRow) {
+	if row.SecretUnconsumedFor <= unconsumedGrace {
+		return // no staged secret, already consumed, or still inside the normal pickup window
+	}
+	// Only meaningful when the box claims to be converged. A box that honestly reports
+	// waiting_secret / consumed_failed / auth_failed is already being healed above and its unconsumed
+	// secret is the SYMPTOM being fixed, not a contradiction.
+	if row.ReportedState != "applied" && row.ReportedState != "adopted" {
+		return
+	}
+	if st, ok := r.unconsumedSeen[row.HostID]; ok && st == row.ReportID {
+		return // already surfaced for this exact report
+	}
+	if r.unconsumedSeen == nil {
+		r.unconsumedSeen = map[string]int64{}
+	}
+	r.unconsumedSeen[row.HostID] = row.ReportID
+	r.logger.Printf("[WARN] pbsdrheal: host %s (customer %s) reports pbs_dr=%s while a staged one-time secret has been UNCONSUMED for %s — the box is not using the credential the hub issued (R-39 fingerprint)",
+		row.HostID, row.CustomerID, row.ReportedState, row.SecretUnconsumedFor.Round(time.Minute))
+	r.event(row.CustomerID, eventUnconsumed, "warning",
+		"PBS-DR honesty check: the box reports the DR tier as applied, but a one-time credential issued by the hub has never been consumed. The tier may be authenticating with a superseded credential.")
+}
+
 // confirm advances the per-host debounce and reports whether the stuck state has held across
 // >= debounceReports DISTINCT reports. A re-observed same report (same reportID) never advances the
 // streak — the debounce counts fresh evidence, not reconciler ticks.
@@ -235,6 +295,24 @@ func (r *Reconciler) healConsumedFailed(ctx context.Context, row store.PBSDRHeal
 	}
 }

+// healAuthFailed escalates a box whose credential PBS rejects (401) to a fresh mint. This is the leg
+// that closes the R-39 loop end to end: the agent now PROVES the credential is dead instead of
+// silently skipping, the hub re-keys, the fresh mint advances the secret generation, the descriptor
+// hash moves, and the agent finally re-consumes (Scenario A). Before this, an applied-and-401 tier
+// stayed green forever and would have surfaced first at a real restore.
+//
+// A re-stage is deliberately NOT attempted: the stored secret IS the one PBS just rejected, so
+// re-arming it would burn a tick and change nothing. Damping is the SHARED confirm() — a 401 flap
+// must not turn into a secret-minting chain.
+func (r *Reconciler) healAuthFailed(ctx context.Context, row store.PBSDRHealRow) {
+	r.logger.Printf("[WARN] pbsdrheal: host %s (customer %s) reports auth_failed (PBS rejects its credential) — escalating to Re-issue", row.HostID, row.CustomerID)
+	if r.reissue(ctx, row) {
+		r.event(row.CustomerID, eventAuthFailed, "warning",
+			"PBS-DR self-heal: a box reported auth_failed (the DR endpoint rejected its stored credential); re-issued fresh endpoint credentials so the agent can re-consume and converge.")
+		r.resetAfterHeal(row)
+	}
+}
+
 // reissue runs the escalation; returns true on success (the caller then records the audit event).
 func (r *Reconciler) reissue(ctx context.Context, row store.PBSDRHealRow) bool {
 	if err := r.act.Reissue(ctx, row.CustomerID); err != nil {
@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"sync"
 	"testing"
+	"time"

 	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
 )
@@ -75,6 +76,16 @@ func seedReport(t *testing.T, st *store.Store, hostID, customerID, state string)
 	}
 }

+// ageSecret back-dates a staged secret's created_at so grace-window behaviour is testable without
+// sleeping. Writes the house SQLite datetime format.
+func ageSecret(t *testing.T, st *store.Store, hostID string, by time.Duration) {
+	t.Helper()
+	when := time.Now().UTC().Add(-by).Format("2006-01-02 15:04:05")
+	if err := st.SetHostPBSSecretCreatedAtForTest(hostID, when); err != nil {
+		t.Fatalf("age secret %s: %v", hostID, err)
+	}
+}
+
 func boolStr(b bool) string {
 	if b {
 		return "true"
@@ -228,7 +239,7 @@ func TestScenarioE_ConsumedFailedEscalates(t *testing.T) {
 // F — DR-OFF (disabled descriptor) and enabled-but-unprovisioned hosts are never in the work set.
 func TestScenarioF_OutOfScopeHostsUntouched(t *testing.T) {
 	st := newHealStore(t)
-	seedHost(t, st, "hOff", "cOff", false, true)   // descriptor disabled
+	seedHost(t, st, "hOff", "cOff", false, true) // descriptor disabled
 	seedReport(t, st, "hOff", "cOff", stateWaitingSecret)
 	seedHost(t, st, "hUnprov", "cUnprov", true, false) // enabled but never provisioned
 	seedReport(t, st, "hUnprov", "cUnprov", stateWaitingSecret)
@@ -274,3 +285,190 @@ func TestNoReHealOnSameReport(t *testing.T) {
 		t.Fatalf("re-healed the same stuck report: restages=%d, want 1", fake.restages())
 	}
 }
+
+// R-39 — auth_failed is escalated to a fresh mint, never re-staged.
+//
+// This is the leg that closes the loop end to end: the agent (>=0.91.0) now PROVES the credential is
+// rejected instead of silently skipping, and the hub re-keys, which advances the secret generation,
+// which moves the descriptor hash, which finally re-arms the agent. A re-stage would re-feed the very
+// secret PBS just rejected.
+//
+// COMPANION RED-PROOF (run + recorded): delete the `case stateAuthFailed` arm → the state falls to
+// the default no-op branch and this test FAILS with reissues=0.
+func TestR39_AuthFailedEscalatesToReissue(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hAF", "cAF", true, true)
+	seedReport(t, st, "hAF", "cAF", stateAuthFailed)
+	fake := &fakeActions{restageResult: true} // a re-stageable secret exists — it must STILL not be used
+	r := newRec(st, fake)
+
+	r.reconcileOnce(context.Background())
+
+	if fake.restages() != 0 {
+		t.Errorf("restages=%d, want 0 — re-staging re-feeds the credential PBS just rejected", fake.restages())
+	}
+	if fake.reissues() != 1 {
+		t.Fatalf("reissues=%d, want 1 — an applied-and-401 tier must be re-keyed, not left green", fake.reissues())
+	}
+	if got := eventTypes(t, st, "cAF"); len(got) != 1 || got[0] != eventAuthFailed {
+		t.Errorf("events = %v, want [%s] (distinct auth_failed signal, not borrowed from consumed_failed)", got, eventAuthFailed)
+	}
+}
+
+// The damper is the burn protection: a 401 flap must not become a secret-minting chain. auth_failed
+// uses the SAME confirm() debounce as the other triggers — one report is not enough.
+func TestR39_AuthFailedIsDebounced(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hAF2", "cAF2", true, true)
+	fake := &fakeActions{restageResult: true}
+	r := newRec(st, fake)
+	r.debounceReports = 2 // require two DISTINCT stuck reports
+
+	seedReport(t, st, "hAF2", "cAF2", stateAuthFailed)
+	r.reconcileOnce(context.Background())
+	if fake.reissues() != 0 {
+		t.Fatalf("re-issued on the FIRST auth_failed report (reissues=%d) — the damper is bypassed", fake.reissues())
+	}
+
+	seedReport(t, st, "hAF2", "cAF2", stateAuthFailed) // a second, distinct report
+	r.reconcileOnce(context.Background())
+	if fake.reissues() != 1 {
+		t.Fatalf("reissues=%d after two distinct auth_failed reports, want 1", fake.reissues())
+	}
+
+	// And the same confirmed report must not re-heal on the next tick.
+	r.reconcileOnce(context.Background())
+	if fake.reissues() != 1 {
+		t.Errorf("re-healed on a stale report: reissues=%d, want 1", fake.reissues())
+	}
+}
+
+// A host that recovers (auth_failed → applied) is a pure no-op, and its streak is forgotten.
+func TestR39_AuthFailedRecoveryIsNoOp(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hAF3", "cAF3", true, true)
+	fake := &fakeActions{restageResult: true}
+	r := newRec(st, fake)
+	r.debounceReports = 2 // so a single stuck report is not yet actionable
+
+	seedReport(t, st, "hAF3", "cAF3", stateAuthFailed)
+	r.reconcileOnce(context.Background())        // streak 1 — below the damper, nothing done
+	seedReport(t, st, "hAF3", "cAF3", "applied") // the box healed on its own
+	r.reconcileOnce(context.Background())
+
+	if fake.reissues() != 0 || fake.restages() != 0 {
+		t.Errorf("a recovered host was acted on: restages=%d reissues=%d, want 0/0", fake.restages(), fake.reissues())
+	}
+
+	// The streak must be FORGOTTEN, not merely paused: a later single auth_failed must start over
+	// rather than instantly tripping the damper it had half-filled before recovering.
+	seedReport(t, st, "hAF3", "cAF3", stateAuthFailed)
+	r.reconcileOnce(context.Background())
+	if fake.reissues() != 0 {
+		t.Errorf("a stale pre-recovery streak survived and fired early: reissues=%d, want 0", fake.reissues())
+	}
+}
+
+// R-39 Scenario F — the consumed_at HONESTY gauge.
+//
+// The July-18 fingerprint is a disagreement no single tier can see: the hub staged a credential, the
+// box never took it, and the box nonetheless reports `applied`. This surfaces it. It deliberately
+// does NOT auto-heal — minting on top of an unconsumed secret is the mint/consume race R-39(a)
+// already recorded.
+//
+// COMPANION RED-PROOF (run + recorded): delete the r.checkUnconsumed(row) call → no event is written
+// and TestR39_UnconsumedSecretUnderAppliedIsSurfaced FAILS.
+func TestR39_UnconsumedSecretUnderAppliedIsSurfaced(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hU", "cU", true, true)
+	seedReport(t, st, "hU", "cU", "applied") // the box claims converged
+	if _, err := st.SaveHostPBSSecret("hU", "never-taken"); err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	// Age the staged secret past the grace window.
+	ageSecret(t, st, "hU", 3*time.Hour)
+
+	fake := &fakeActions{restageResult: true}
+	r := newRec(st, fake)
+	r.reconcileOnce(context.Background())
+
+	if got := eventTypes(t, st, "cU"); len(got) != 1 || got[0] != eventUnconsumed {
+		t.Fatalf("events = %v, want [%s]", got, eventUnconsumed)
+	}
+	// SURFACE, not heal: an applied box is not in any heal arm, so nothing was minted or re-staged.
+	if fake.reissues() != 0 || fake.restages() != 0 {
+		t.Errorf("the honesty gauge must not mutate: restages=%d reissues=%d, want 0/0", fake.restages(), fake.reissues())
+	}
+
+	// One event per DISTINCT report — a sustained disagreement must not spam the audit log.
+	r.reconcileOnce(context.Background())
+	if got := eventTypes(t, st, "cU"); len(got) != 1 {
+		t.Errorf("re-surfaced on the same report: %d events, want 1", len(got))
+	}
+}
+
+// A freshly staged secret (or a just-restaged one) is NORMAL inside the grace window and must not
+// alarm — RestageHostPBSSecret deliberately sets consumed_at back to NULL.
+func TestR39_FreshlyStagedSecretDoesNotAlarm(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hF", "cF", true, true)
+	seedReport(t, st, "hF", "cF", "applied")
+	if _, err := st.SaveHostPBSSecret("hF", "just-minted"); err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	// A deliberate re-stage clears consumed_at — the Scenario-F edge case.
+	if _, err := st.ConsumeHostPBSSecret("hF"); err != nil {
+		t.Fatalf("consume: %v", err)
+	}
+	if _, err := st.RestageHostPBSSecret("hF"); err != nil {
+		t.Fatalf("restage: %v", err)
+	}
+
+	r := newRec(st, &fakeActions{restageResult: true})
+	r.reconcileOnce(context.Background())
+
+	if got := eventTypes(t, st, "cF"); len(got) != 0 {
+		t.Errorf("a just-staged secret alarmed inside the grace window: %v", got)
+	}
+}
+
+// A CONSUMED secret never alarms, however old.
+func TestR39_ConsumedSecretNeverAlarms(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hC", "cC", true, true)
+	seedReport(t, st, "hC", "cC", "applied")
+	if _, err := st.SaveHostPBSSecret("hC", "taken"); err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	if _, err := st.ConsumeHostPBSSecret("hC"); err != nil {
+		t.Fatalf("consume: %v", err)
+	}
+	ageSecret(t, st, "hC", 30*24*time.Hour)
+
+	r := newRec(st, &fakeActions{restageResult: true})
+	r.reconcileOnce(context.Background())
+	if got := eventTypes(t, st, "cC"); len(got) != 0 {
+		t.Errorf("a consumed secret alarmed: %v", got)
+	}
+}
+
+// A box that HONESTLY reports a stuck state is already being healed; its unconsumed secret is the
+// symptom, not a contradiction, so the gauge stays quiet and only the heal event is written.
+func TestR39_HonestStuckStateDoesNotDoubleReport(t *testing.T) {
+	st := newHealStore(t)
+	seedHost(t, st, "hH", "cH", true, true)
+	seedReport(t, st, "hH", "cH", stateWaitingSecret)
+	if _, err := st.SaveHostPBSSecret("hH", "staged"); err != nil {
+		t.Fatalf("mint: %v", err)
+	}
+	ageSecret(t, st, "hH", 3*time.Hour)
+
+	r := newRec(st, &fakeActions{restageResult: true})
+	r.reconcileOnce(context.Background())
+
+	for _, e := range eventTypes(t, st, "cH") {
+		if e == eventUnconsumed {
+			t.Error("an honestly-stuck box must not also raise the honesty gauge — it is the symptom being healed")
+		}
+	}
+}