hub v0.68.0 — auth_failed self-heal, consumed_at honesty gauge, wrapper drift (R-39 + R-50b(a))
Completes the hub half of R-39's fleet fix on top of the generation core (c484aa2).
pbsdrheal gains an auth_failed TRIGGER — a new trigger in the existing machine, not a
new machine. A box whose credential PBS rejects escalates to a fresh mint, never a
re-stage (which would re-feed the secret PBS just rejected), through the EXISTING damper:
a 401 flap must not become a secret-minting chain. With the generation stamp this closes
the loop end to end — agent proves the 401, hub re-keys, generation advances, descriptor
hash moves, agent re-consumes.
consumed_at honesty gauge: a staged secret still unconsumed past a 15-minute grace while
the box reports `applied` is surfaced with its own event. That is the exact 2026-07-18
fingerprint and a disagreement no single tier can see alone. Deliberately a SURFACE, not
a heal — auto-re-issuing on it would mint a second secret on top of an unconsumed one,
which is the mint/consume race R-39(a) already recorded. One event per distinct report,
and an honestly-stuck box does not double-report (its unconsumed secret is the symptom
being healed, not a contradiction).
R-50b(a): ArtifactManifest.WrapperSHA256 + operator field + host-page drift surface. The
PBS wrapper is root-owned 0755 and the pinned sudoers vector, yet installed unversioned
from raw/branch/main and absent from every manifest. Agents >=0.91.0 report the installed
hash; a mismatch is surfaced. An unknown on EITHER side reads as quiet, never as drift —
lighting every host amber on rollout day is how a warning becomes background noise. The
delivery channel itself stays R-50b(b)/(c).
Compatibility unchanged: safe for 0.90.0 agents (unknown JSON key dropped); the re-arm
and auth-honesty guarantees need agent >=0.91.0, so MinAgent moves only after the fleet
has self-updated.
Tests: auth_failed escalate/debounce/recovery-forgets-streak; honesty gauge incl. grace
window, the restage edge (consumed_at deliberately NULLed), consumed-never-alarms, and
honest-stuck-no-double-report; wrapper drift incl. both unknown directions. Red-proof run
at the assertion level: removing the auth_failed arm fails the escalation tests with
reissues=0.
This commit is contained in:
@@ -3,6 +3,7 @@ package store
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// PBS DR tier (SLICE 1): the HOST-scoped one-time PBS token secret — the host/agent twin of the
|
||||
@@ -102,6 +103,17 @@ type PBSDRHealRow struct {
|
||||
DescriptorProvisioned bool // desired_json pbs_dr.namespace != "" (was provisioned, not a bare enable)
|
||||
ReportedState string // latest report pbs_dr.state ("" = no report / no stanza)
|
||||
ReportID int64 // id of that latest host_report (0 = none); the debounce distinctness key
|
||||
|
||||
// R-39 consumed_at honesty (Scenario F). SecretUnconsumedFor is how long an UNCONSUMED secret has
|
||||
// been staged for this host (0 when none is staged, or when it has already been consumed).
|
||||
//
|
||||
// A staged-but-unconsumed secret sitting under a box that reports `applied` is the exact
|
||||
// fingerprint of the 2026-07-18 N100 failure: the hub minted, the agent short-circuited, and both
|
||||
// tiers reported success while the box served a revoked credential. Briefly unconsumed is NORMAL
|
||||
// (a fresh mint, or a deliberate re-stage, is consumed on the agent's next tick) — only a
|
||||
// SUSTAINED one is a lie, which is why the caller applies a grace window rather than alarming on
|
||||
// presence alone.
|
||||
SecretUnconsumedFor time.Duration
|
||||
}
|
||||
|
||||
// PBSDRHealStates returns one row per host: its descriptor enable/provision flags + the agent's
|
||||
@@ -110,11 +122,13 @@ type PBSDRHealRow struct {
|
||||
// (the reconciler filters to enabled+provisioned hosts).
|
||||
func (s *Store) PBSDRHealStates() ([]PBSDRHealRow, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT h.host_id, h.customer_id, h.desired_json, latest.mx, hr.report_json
|
||||
SELECT h.host_id, h.customer_id, h.desired_json, latest.mx, hr.report_json,
|
||||
ps.created_at, ps.consumed_at
|
||||
FROM hosts h
|
||||
LEFT JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest
|
||||
ON latest.host_id = h.host_id
|
||||
LEFT JOIN host_reports hr ON hr.id = latest.mx`)
|
||||
LEFT JOIN host_reports hr ON hr.id = latest.mx
|
||||
LEFT JOIN host_pbs_secrets ps ON ps.host_id = h.host_id`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -123,8 +137,9 @@ func (s *Store) PBSDRHealStates() ([]PBSDRHealRow, error) {
|
||||
for rows.Next() {
|
||||
var hostID, customerID, desiredJSON string
|
||||
var reportID sql.NullInt64
|
||||
var reportJSON sql.NullString
|
||||
if err := rows.Scan(&hostID, &customerID, &desiredJSON, &reportID, &reportJSON); err != nil {
|
||||
var reportJSON, secretCreatedAt, secretConsumedAt sql.NullString
|
||||
if err := rows.Scan(&hostID, &customerID, &desiredJSON, &reportID, &reportJSON,
|
||||
&secretCreatedAt, &secretConsumedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r := PBSDRHealRow{HostID: hostID, CustomerID: customerID}
|
||||
@@ -151,7 +166,27 @@ func (s *Store) PBSDRHealStates() ([]PBSDRHealRow, error) {
|
||||
r.ReportedState = rr.PBSDR.State
|
||||
}
|
||||
}
|
||||
// Unconsumed-secret age: only when a secret is staged AND still unconsumed.
|
||||
if secretCreatedAt.Valid && !secretConsumedAt.Valid {
|
||||
// SQLite datetime('now') is UTC and format-varied — parseSQLiteTime is the house parser
|
||||
// (it returns the zero time on an unparseable value, which we treat as "unknown", never
|
||||
// as "infinitely stale").
|
||||
if created := parseSQLiteTime(secretCreatedAt.String); !created.IsZero() {
|
||||
if age := time.Since(created); age > 0 {
|
||||
r.SecretUnconsumedFor = age
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetHostPBSSecretCreatedAtForTest back-dates a staged secret's created_at. TEST-ONLY seam: the
|
||||
// grace-window behaviour of the consumed_at honesty gauge (R-39 Scenario F) is otherwise only
|
||||
// observable by sleeping for 15 minutes. It touches nothing else — not the value, not consumed_at,
|
||||
// not the generation.
|
||||
func (s *Store) SetHostPBSSecretCreatedAtForTest(hostID, sqliteTime string) error {
|
||||
_, err := s.db.Exec(`UPDATE host_pbs_secrets SET created_at = ? WHERE host_id = ?`, sqliteTime, hostID)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1476,6 +1476,17 @@ type ArtifactManifest struct {
|
||||
// box whose agent is below it (Part D) — mechanising the publish-train "agent BEFORE controller
|
||||
// floor" rule instead of leaving it to operator discipline.
|
||||
MinAgent string `json:"min_agent"`
|
||||
// WrapperSHA256 is the sha256 of the PBS-DR apply wrapper (configs/felhom-pbs-apply) the operator
|
||||
// has vouched (R-50b(a), v0.68.0).
|
||||
//
|
||||
// Unlike the agent binary and the golden, this artifact is installed from
|
||||
// `raw/branch/main` by felhom-host-install.sh — UNVERSIONED, with no tag, no pin and no checksum.
|
||||
// It is a root-owned 0755 file and the pinned sudoers vector for the PBS storage verbs, so "which
|
||||
// wrapper is on this host?" was previously unanswerable from any manifest: two hosts installed a
|
||||
// week apart could carry different privileged code while reporting the same agent version.
|
||||
// Recording the hash here does not fix the delivery channel (that is R-50b(b)/(c)) — it makes
|
||||
// DRIFT VISIBLE, which is the cheap honest first step.
|
||||
WrapperSHA256 string `json:"wrapper_sha256"`
|
||||
}
|
||||
|
||||
// hub_settings keys for the artifact manifest (BUNDLE slice). Stored as discrete key/value rows in
|
||||
@@ -1487,6 +1498,7 @@ const (
|
||||
settingArtifactGoldenVersion = "artifact_golden_version"
|
||||
settingArtifactGoldenSHA256 = "artifact_golden_sha256"
|
||||
settingArtifactMinAgent = "artifact_min_agent"
|
||||
settingArtifactWrapperSHA256 = "artifact_wrapper_sha256" // R-50b(a): the vouched felhom-pbs-apply hash
|
||||
)
|
||||
|
||||
// settingOperatorPasswordHash is the hub_settings key for the operator login password bcrypt hash,
|
||||
@@ -1534,6 +1546,7 @@ func (s *Store) GetArtifactManifest() ArtifactManifest {
|
||||
GoldenVersion: s.getSetting(settingArtifactGoldenVersion),
|
||||
GoldenSHA256: s.getSetting(settingArtifactGoldenSHA256),
|
||||
MinAgent: s.getSetting(settingArtifactMinAgent),
|
||||
WrapperSHA256: s.getSetting(settingArtifactWrapperSHA256),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1552,7 +1565,10 @@ func (s *Store) SetArtifactManifest(m ArtifactManifest) error {
|
||||
if err := s.setSetting(settingArtifactGoldenSHA256, m.GoldenSHA256); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.setSetting(settingArtifactMinAgent, m.MinAgent)
|
||||
if err := s.setSetting(settingArtifactMinAgent, m.MinAgent); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.setSetting(settingArtifactWrapperSHA256, m.WrapperSHA256)
|
||||
}
|
||||
|
||||
// EffectiveMinControllerVersion resolves the floor that actually applies to a customer: the
|
||||
|
||||
Reference in New Issue
Block a user