diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 6ae1347..b87820c 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,6 +1,37 @@ # Felhom Hub — Changelog -## v0.72.0 — R-70 + R-71(c): the offsite last mile becomes visible, and burned credentials heal themselves (2026-07-23) +## v0.73.0 — `offsite_stale` no longer cries wolf on a newborn tier (ISO-train v1.25.0 Part 7) (2026-07-23) + +Origin (operator, 2026-07-23 12:01 CEST): demo-hp's offsite was repaired and escrowed at 10:01Z and +`offsite_stale` fired MINUTES later (`last_run:"" … threshold 48h0m0s`) — the never-ran branch had +no time anchor, so "enabled + escrowed + never ran" was instantly stale on the first tick, with +remedy copy ("check the controller/schedule") that was wrong for the moment's true state. + +**The fix (no new constant, one boundary) — `monitor.OffsiteChecker.isStale`:** + +- **Boundary ruling (recorded):** never-ran staleness is owned by `offsite_stale` ONLY in the + v0.72.0 `applied` delivery state; pre-applied never-ran shapes belong to + `offsite_delivery_stuck` alone — **one state, one owner, never both**. Structurally this was + already true (`Check` nil-skips reports without the offsite object, and a report CARRYING the + object IS `applied` by the v0.72.0 definition) — now it is pinned by an explicit boundary test + that also asserts the delivery checker remains that shape's only voice. +- **Anchor:** never-ran staleness = `applied` AND (now − anchor) > the EXISTING 48 h threshold, + where anchor = the newest of **`one_time_secrets.consumed_at`** (delivery completed; via the + v0.72.0 `GetOneTimeSecretInfo`) and the customer's **escrow-blob timestamp** + (`host_escrow.updated_at`/`created_at` via the new `LatestEscrowTimeForCustomer` — reduced in + Go, not SQL MAX, because the two timestamp formats would misorder lexicographically). Runs + become possible only at the ceremony, so the ceremony anchors the clock. No separate grace + knob: the existing threshold, anchored properly, IS the grace. +- Anchor-less legacy shape (no secret row, no escrow row): pre-fix behavior kept — stale on + sight, fail toward visibility. Ran-before behavior byte-untouched (explicit both-direction + tests: old run + fresh anchor still fires; fresh run + old anchor stays silent). +- One INFO log line on the FIRST observation of a deferred newborn (`never-ran within the + anchored threshold (anchor …)`) — the anchored evaluation is provable live without per-sweep + spam. + +Red-proof: never-ran branch reverted to the pre-fix `return true` → the fresh-anchor fixture and +the escrow-anchor fixture both fired `offsite_stale:warning` (FAIL observed) → fix restored. + Origin: `documentation/audits/DIAG-f10-demo-hp-offsite-2026-07-23.md` — demo-hp sat 2 days with the customer card claiming "Provisioned … delivered to the controller once" (static copy) while the box diff --git a/hub/internal/monitor/offsite.go b/hub/internal/monitor/offsite.go index 5b6cba7..add340e 100644 --- a/hub/internal/monitor/offsite.go +++ b/hub/internal/monitor/offsite.go @@ -73,7 +73,7 @@ func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNo oc.fillStates[c.CustomerID] = bandOK seeded++ } - if !oc.isStale(off) { + if !oc.isStale(c.CustomerID, off) { oc.staleStates[c.CustomerID] = "ok" } } @@ -101,14 +101,28 @@ func (oc *OffsiteChecker) fillBand(off *offsiteReport) string { } // isStale: enabled + ESCROWED (the only state where runs are expected) with no run in >staleAfter (or -// never). Pending/disabled = normal onboarding, never stale. A recent-but-failing run is NOT stale -// (backup_failed owns that signal). -func (oc *OffsiteChecker) isStale(off *offsiteReport) bool { +// never ran, ANCHORED — see below). Pending/disabled = normal onboarding, never stale. A +// recent-but-failing run is NOT stale (backup_failed owns that signal). +// +// Part-7 (v0.73.0) — the never-ran branch no longer fires on sight. The 2026-07-23 cry-wolf: +// demo-hp's tier was repaired and escrowed at 10:01Z and offsite_stale fired MINUTES later +// (`last_run:"" … threshold 48h`), because "enabled + escrowed + never ran" had no time anchor. +// Boundary: reaching this code at all means the latest report CARRIES the offsite object — i.e. +// the v0.72.0 delivery state is `applied` (Check nil-skips everything else); pre-applied +// never-ran shapes are offsite_delivery_stuck's alone — ONE STATE, ONE OWNER, never both. +// Anchor: the newest of one_time_secrets.consumed_at (delivery completed) and the customer's +// escrow-blob timestamp (host_escrow.updated_at/created_at — runs become POSSIBLE only at the +// ceremony). The EXISTING staleAfter threshold, anchored there, IS the grace — no new knob. +func (oc *OffsiteChecker) isStale(customerID string, off *offsiteReport) bool { if !off.Enabled || off.EscrowState != "escrowed" { return false } if off.LastRun == "" { - return true + anchor := oc.neverRanAnchor(customerID) + if anchor.IsZero() { + return true // legacy shape (no secret timestamps, no escrow row) — fail toward visibility, as before + } + return oc.now().Sub(anchor) > oc.staleAfter } t, err := time.Parse(time.RFC3339, off.LastRun) if err != nil { @@ -117,6 +131,19 @@ func (oc *OffsiteChecker) isStale(off *offsiteReport) bool { return oc.now().Sub(t) > oc.staleAfter } +// neverRanAnchor returns the newest hub-held timestamp from which a never-ran-but-applied tier's +// staleness may be counted (zero when the hub holds neither — the pre-v0.5x legacy shape). +func (oc *OffsiteChecker) neverRanAnchor(customerID string) time.Time { + var anchor time.Time + if info, err := oc.store.GetOneTimeSecretInfo(customerID); err == nil && info != nil && info.ConsumedAt.After(anchor) { + anchor = info.ConsumedAt + } + if t, err := oc.store.LatestEscrowTimeForCustomer(customerID); err == nil && t.After(anchor) { + anchor = t + } + return anchor +} + // Check evaluates every customer's latest report. Escalation-only emits; recovery re-arms silently. func (oc *OffsiteChecker) Check() { customers, err := oc.store.GetCustomers() @@ -156,12 +183,21 @@ func (oc *OffsiteChecker) Check() { // STALENESS (binary, warn-severity) newStale := "ok" - if oc.isStale(off) { + if oc.isStale(c.CustomerID, off) { newStale = "stale" } if newStale == "stale" && oc.staleStates[c.CustomerID] != "stale" { oc.emitStale(c.CustomerID, off) } + // Part-7: make the anchored never-ran evaluation VISIBLE once (first observation of the + // shape), so a live newborn tier's deferral is provable from the log without spamming + // every sweep. + if off.LastRun == "" && newStale == "ok" && off.Enabled && off.EscrowState == "escrowed" { + if _, known := oc.staleStates[c.CustomerID]; !known { + oc.logger.Printf("[INFO] Offsite staleness: %s never-ran within the anchored threshold (anchor %s) — newborn tier, not stale", + c.CustomerID, oc.neverRanAnchor(c.CustomerID).UTC().Format(time.RFC3339)) + } + } oc.staleStates[c.CustomerID] = newStale } for k := range oc.fillStates { diff --git a/hub/internal/monitor/offsite_neverran_test.go b/hub/internal/monitor/offsite_neverran_test.go new file mode 100644 index 0000000..05720c3 --- /dev/null +++ b/hub/internal/monitor/offsite_neverran_test.go @@ -0,0 +1,193 @@ +package monitor + +import ( + "io" + "log" + "path/filepath" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// Part-7 (v0.73.0): the never-ran offsite-staleness branch is ANCHORED — a newborn tier +// (applied + escrowed + never ran) is not stale until the EXISTING 48 h threshold has elapsed +// since the newest of one_time_secrets.consumed_at / the escrow-blob timestamp. Origin: the +// 2026-07-23 cry-wolf — demo-hp escrowed 10:01Z, offsite_stale fired minutes later. +// One state, one owner: pre-applied never-ran shapes belong to offsite_delivery_stuck alone. + +func newNeverRanStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.New(filepath.Join(t.TempDir(), "nr.db"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + return st +} + +func sqliteAgo(d time.Duration) string { + return time.Now().UTC().Add(-d).Format("2006-01-02 15:04:05") +} + +// seedConsumedSecret stages + consumes a secret for the customer, back-dating consumed_at. +func seedConsumedSecret(t *testing.T, st *store.Store, customerID string, consumedAgo time.Duration) { + t.Helper() + if err := st.SaveOneTimeSecret(customerID, "x"); err != nil { + t.Fatal(err) + } + if err := st.SetOneTimeSecretTimesForTest(customerID, sqliteAgo(consumedAgo+time.Hour), sqliteAgo(consumedAgo)); err != nil { + t.Fatal(err) + } +} + +func neverRanChecker(st *store.Store, events *[]string) *OffsiteChecker { + return NewOffsiteChecker(st, 0, func(_, eventType, severity, _, _, _ string) { + *events = append(*events, eventType+":"+severity) + }, log.New(io.Discard, "", 0)) +} + +// The fix: applied + escrowed + never ran with a FRESH anchor (consumed 1 h ago) → NO event. +// COMPANION RED-PROOF: replace the anchored never-ran branch with the pre-fix `return true` → +// this fixture fires offsite_stale on the first sweep → FAIL observed → restored. +func TestOffsiteStale_NeverRanFreshAnchor_Silent(t *testing.T) { + st := newNeverRanStore(t) + seedConsumedSecret(t, st, "c1", time.Hour) + saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50)) + + var events []string + oc := neverRanChecker(st, &events) + oc.Check() + oc.Check() + + if len(events) != 0 { + t.Fatalf("events = %v, want none — a newborn tier (anchor 1h ago, threshold 48h) is NOT stale", events) + } + if got := oc.GetStaleState("c1"); got != "ok" { + t.Fatalf("stale state = %q, want ok", got) + } +} + +// The threshold still bites: anchor 49 h ago (consumed_at) + never ran → exactly one offsite_stale +// with the existing copy shape. +func TestOffsiteStale_NeverRanOldAnchor_FiresOnce(t *testing.T) { + st := newNeverRanStore(t) + seedConsumedSecret(t, st, "c1", 49*time.Hour) + saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50)) + + var events []string + oc := neverRanChecker(st, &events) + oc.Check() + oc.Check() // deduped across sweeps + + if len(events) != 1 || events[0] != "offsite_stale:warning" { + t.Fatalf("events = %v, want exactly [offsite_stale:warning]", events) + } + evs, err := st.GetRecentEvents("c1", 10) + if err != nil || len(evs) != 1 { + t.Fatalf("saved events = %v (%v), want exactly 1", evs, err) + } + if evs[0].EventType != "offsite_stale" { + t.Fatalf("event type = %s", evs[0].EventType) + } +} + +// The escrow-timestamp leg of the anchor: consumed_at long ago but the CEREMONY (escrow blob) +// happened 1 h ago → the newest anchor wins → silent. (The ceremony is the moment runs become +// possible; a delayed ceremony must not make the tier instantly stale.) +func TestOffsiteStale_NeverRanEscrowAnchorWins(t *testing.T) { + st := newNeverRanStore(t) + seedConsumedSecret(t, st, "c1", 72*time.Hour) // delivery long ago + if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k"}); err != nil { + t.Fatal(err) + } + // escrow blob stored now (SaveHostEscrow stamps updated_at with datetime('now')) + if _, err := st.SaveHostEscrow("h1", []byte("blob"), "fp", "zero_knowledge", time.Now().UTC().Format(time.RFC3339), "sha"); err != nil { + t.Fatal(err) + } + saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50)) + + var events []string + oc := neverRanChecker(st, &events) + oc.Check() + + if len(events) != 0 { + t.Fatalf("events = %v, want none — the newest anchor (escrow 'now') defers staleness", events) + } +} + +// THE BOUNDARY (one state, one owner): a never-ran tier that is NOT applied (report carries no +// offsite object) produces ZERO offsite_stale regardless of age — and offsite_delivery_stuck +// (the v0.72.0 checker) remains that shape's only voice. +func TestOffsiteStale_NotApplied_NeverFires_DeliveryCheckerOwnsIt(t *testing.T) { + st := newNeverRanStore(t) + // the burned-credential shape: consumed 49h ago, offsite ENABLED in config, report WITHOUT offsite + if err := st.SaveCustomerConfig(&store.CustomerConfig{ + CustomerID: "c1", APIKey: "k", RetrievalPassword: "p", + ConfigJSON: `{"offsite":{"enabled":true,"type":"shared","host":"h","user":"u","repo_path":"/r","quota_gb":50}}`, + }); err != nil { + t.Fatal(err) + } + seedConsumedSecret(t, st, "c1", 49*time.Hour) + saveOffsiteReport(t, st, "c1", "") // no offsite object = not applied + + var staleEvents []string + oc := neverRanChecker(st, &staleEvents) + oc.Check() + if len(staleEvents) != 0 { + t.Fatalf("offsite_stale events = %v, want none — a pre-applied shape is NEVER offsite_stale's", staleEvents) + } + + // ...and the delivery checker IS that shape's voice. + var deliveryEvents []string + dc := NewOffsiteDeliveryChecker(st, nil, func(_, eventType, severity, _, _, _ string) { + deliveryEvents = append(deliveryEvents, eventType+":"+severity) + }, log.New(io.Discard, "", 0)) + dc.Check() + if len(deliveryEvents) != 1 || deliveryEvents[0] != "offsite_delivery_stuck:warning" { + t.Fatalf("delivery events = %v, want exactly [offsite_delivery_stuck:warning]", deliveryEvents) + } +} + +// Legacy fallback: applied + escrowed + never ran with NO anchor at all (no secret row, no escrow +// row) keeps the pre-fix behavior — stale on sight (fail toward visibility). +func TestOffsiteStale_NeverRanNoAnchor_LegacyFailsTowardVisibility(t *testing.T) { + st := newNeverRanStore(t) + saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50)) + + var events []string + oc := neverRanChecker(st, &events) + oc.Check() + + if len(events) != 1 || events[0] != "offsite_stale:warning" { + t.Fatalf("events = %v, want [offsite_stale:warning] — the anchor-less legacy shape stays visible", events) + } +} + +// Ran-before behavior byte-untouched: recent run silent, 49h-old run fires — with a consumed_at +// anchor present that must NOT interfere in either direction. +func TestOffsiteStale_RanBefore_Untouched(t *testing.T) { + st := newNeverRanStore(t) + seedConsumedSecret(t, st, "c1", time.Hour) // fresh anchor must not mask an old run + old := time.Now().UTC().Add(-49 * time.Hour).Format(time.RFC3339) + saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", old, "ok", 0, 50)) + + var events []string + oc := neverRanChecker(st, &events) + oc.Check() + if len(events) != 1 || events[0] != "offsite_stale:warning" { + t.Fatalf("events = %v, want [offsite_stale:warning] — a 49h-old run is stale regardless of a fresh anchor", events) + } + + st2 := newNeverRanStore(t) + seedConsumedSecret(t, st2, "c2", 72*time.Hour) // old anchor must not stale a fresh run + recent := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) + saveOffsiteReport(t, st2, "c2", offsiteJSON(true, "escrowed", recent, "ok", 0, 50)) + + var events2 []string + oc2 := neverRanChecker(st2, &events2) + oc2.Check() + if len(events2) != 0 { + t.Fatalf("events = %v, want none — a 1h-old run is fresh regardless of an old anchor", events2) + } +} diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index dbbb4a1..94acdb9 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -967,6 +967,36 @@ func (s *Store) CountReportsOffsiteSince(customerID string, since time.Time) (to return total, withOffsite, rows.Err() } +// LatestEscrowTimeForCustomer returns the newest escrow-blob timestamp across the customer's +// hosts (updated_at, falling back to created_at), or zero when no blob exists. Part-7 v0.73.0: +// one leg of the never-ran offsite-staleness anchor — the escrow ceremony is the moment runs +// become POSSIBLE, so the 48 h staleness threshold counts from it, not from the dawn of time. +// Reduced in Go (not SQL MAX) because created_at formats vary (RFC3339 vs SQLite space form) and +// lexicographic MAX would misorder them; parseSQLiteTime handles both. +func (s *Store) LatestEscrowTimeForCustomer(customerID string) (time.Time, error) { + rows, err := s.db.Query(` + SELECT he.created_at, he.updated_at FROM host_escrow he + INNER JOIN hosts h ON h.host_id = he.host_id + WHERE h.customer_id = ?`, customerID) + if err != nil { + return time.Time{}, err + } + defer rows.Close() + var newest time.Time + for rows.Next() { + var createdAt, updatedAt string + if err := rows.Scan(&createdAt, &updatedAt); err != nil { + return time.Time{}, err + } + for _, raw := range []string{createdAt, updatedAt} { + if t := parseSQLiteTime(raw); t.After(newest) { + newest = t + } + } + } + return newest, rows.Err() +} + // LastEventAt returns the created_at of the most recent event of the given type for a customer // (zero time when none). Durable across hub restarts — used as the cooldown/rate-limit source for // hub-emitted detector events (R-70/R-71c: a repeating pattern must surface as repeating events on