hub v0.73.0 — offsite_stale anchored on newborn tiers (never-ran = applied-only + consumed_at/escrow anchor; one state one owner)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NKSN3gSg4TKVBBqkwW2djR

hub/CHANGELOG.md

@@ -1,6 +1,37 @@
 # Felhom Hub — Changelog
 
-## v0.72.0 — R-70 + R-71(c): the offsite last mile becomes visible, and burned credentials heal themselves (2026-07-23)
+## v0.73.0 — `offsite_stale` no longer cries wolf on a newborn tier (ISO-train v1.25.0 Part 7) (2026-07-23)
+
+Origin (operator, 2026-07-23 12:01 CEST): demo-hp's offsite was repaired and escrowed at 10:01Z and
+`offsite_stale` fired MINUTES later (`last_run:"" … threshold 48h0m0s`) — the never-ran branch had
+no time anchor, so "enabled + escrowed + never ran" was instantly stale on the first tick, with
+remedy copy ("check the controller/schedule") that was wrong for the moment's true state.
+
+**The fix (no new constant, one boundary) — `monitor.OffsiteChecker.isStale`:**
+
+- **Boundary ruling (recorded):** never-ran staleness is owned by `offsite_stale` ONLY in the
+  v0.72.0 `applied` delivery state; pre-applied never-ran shapes belong to
+  `offsite_delivery_stuck` alone — **one state, one owner, never both**. Structurally this was
+  already true (`Check` nil-skips reports without the offsite object, and a report CARRYING the
+  object IS `applied` by the v0.72.0 definition) — now it is pinned by an explicit boundary test
+  that also asserts the delivery checker remains that shape's only voice.
+- **Anchor:** never-ran staleness = `applied` AND (now − anchor) > the EXISTING 48 h threshold,
+  where anchor = the newest of **`one_time_secrets.consumed_at`** (delivery completed; via the
+  v0.72.0 `GetOneTimeSecretInfo`) and the customer's **escrow-blob timestamp**
+  (`host_escrow.updated_at`/`created_at` via the new `LatestEscrowTimeForCustomer` — reduced in
+  Go, not SQL MAX, because the two timestamp formats would misorder lexicographically). Runs
+  become possible only at the ceremony, so the ceremony anchors the clock. No separate grace
+  knob: the existing threshold, anchored properly, IS the grace.
+- Anchor-less legacy shape (no secret row, no escrow row): pre-fix behavior kept — stale on
+  sight, fail toward visibility. Ran-before behavior byte-untouched (explicit both-direction
+  tests: old run + fresh anchor still fires; fresh run + old anchor stays silent).
+- One INFO log line on the FIRST observation of a deferred newborn (`never-ran within the
+  anchored threshold (anchor …)`) — the anchored evaluation is provable live without per-sweep
+  spam.
+
+Red-proof: never-ran branch reverted to the pre-fix `return true` → the fresh-anchor fixture and
+the escrow-anchor fixture both fired `offsite_stale:warning` (FAIL observed) → fix restored.
+
 
 Origin: `documentation/audits/DIAG-f10-demo-hp-offsite-2026-07-23.md` — demo-hp sat 2 days with the
 customer card claiming "Provisioned … delivered to the controller once" (static copy) while the box

hub/internal/monitor/offsite.go

@@ -73,7 +73,7 @@ func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNo
 			oc.fillStates[c.CustomerID] = bandOK
 			seeded++
 		}
-		if !oc.isStale(off) {
+		if !oc.isStale(c.CustomerID, off) {
 			oc.staleStates[c.CustomerID] = "ok"
 		}
 	}
@@ -101,14 +101,28 @@ func (oc *OffsiteChecker) fillBand(off *offsiteReport) string {
 }
 
 // isStale: enabled + ESCROWED (the only state where runs are expected) with no run in >staleAfter (or
-// never). Pending/disabled = normal onboarding, never stale. A recent-but-failing run is NOT stale
-// (backup_failed owns that signal).
-func (oc *OffsiteChecker) isStale(off *offsiteReport) bool {
+// never ran, ANCHORED — see below). Pending/disabled = normal onboarding, never stale. A
+// recent-but-failing run is NOT stale (backup_failed owns that signal).
+//
+// Part-7 (v0.73.0) — the never-ran branch no longer fires on sight. The 2026-07-23 cry-wolf:
+// demo-hp's tier was repaired and escrowed at 10:01Z and offsite_stale fired MINUTES later
+// (`last_run:"" … threshold 48h`), because "enabled + escrowed + never ran" had no time anchor.
+// Boundary: reaching this code at all means the latest report CARRIES the offsite object — i.e.
+// the v0.72.0 delivery state is `applied` (Check nil-skips everything else); pre-applied
+// never-ran shapes are offsite_delivery_stuck's alone — ONE STATE, ONE OWNER, never both.
+// Anchor: the newest of one_time_secrets.consumed_at (delivery completed) and the customer's
+// escrow-blob timestamp (host_escrow.updated_at/created_at — runs become POSSIBLE only at the
+// ceremony). The EXISTING staleAfter threshold, anchored there, IS the grace — no new knob.
+func (oc *OffsiteChecker) isStale(customerID string, off *offsiteReport) bool {
 	if !off.Enabled || off.EscrowState != "escrowed" {
 		return false
 	}
 	if off.LastRun == "" {
-		return true
+		anchor := oc.neverRanAnchor(customerID)
+		if anchor.IsZero() {
+			return true // legacy shape (no secret timestamps, no escrow row) — fail toward visibility, as before
+		}
+		return oc.now().Sub(anchor) > oc.staleAfter
 	}
 	t, err := time.Parse(time.RFC3339, off.LastRun)
 	if err != nil {
@@ -117,6 +131,19 @@ func (oc *OffsiteChecker) isStale(off *offsiteReport) bool {
 	return oc.now().Sub(t) > oc.staleAfter
 }
 
+// neverRanAnchor returns the newest hub-held timestamp from which a never-ran-but-applied tier's
+// staleness may be counted (zero when the hub holds neither — the pre-v0.5x legacy shape).
+func (oc *OffsiteChecker) neverRanAnchor(customerID string) time.Time {
+	var anchor time.Time
+	if info, err := oc.store.GetOneTimeSecretInfo(customerID); err == nil && info != nil && info.ConsumedAt.After(anchor) {
+		anchor = info.ConsumedAt
+	}
+	if t, err := oc.store.LatestEscrowTimeForCustomer(customerID); err == nil && t.After(anchor) {
+		anchor = t
+	}
+	return anchor
+}
+
 // Check evaluates every customer's latest report. Escalation-only emits; recovery re-arms silently.
 func (oc *OffsiteChecker) Check() {
 	customers, err := oc.store.GetCustomers()
@@ -156,12 +183,21 @@ func (oc *OffsiteChecker) Check() {
 
 		// STALENESS (binary, warn-severity)
 		newStale := "ok"
-		if oc.isStale(off) {
+		if oc.isStale(c.CustomerID, off) {
 			newStale = "stale"
 		}
 		if newStale == "stale" && oc.staleStates[c.CustomerID] != "stale" {
 			oc.emitStale(c.CustomerID, off)
 		}
+		// Part-7: make the anchored never-ran evaluation VISIBLE once (first observation of the
+		// shape), so a live newborn tier's deferral is provable from the log without spamming
+		// every sweep.
+		if off.LastRun == "" && newStale == "ok" && off.Enabled && off.EscrowState == "escrowed" {
+			if _, known := oc.staleStates[c.CustomerID]; !known {
+				oc.logger.Printf("[INFO] Offsite staleness: %s never-ran within the anchored threshold (anchor %s) — newborn tier, not stale",
+					c.CustomerID, oc.neverRanAnchor(c.CustomerID).UTC().Format(time.RFC3339))
+			}
+		}
 		oc.staleStates[c.CustomerID] = newStale
 	}
 	for k := range oc.fillStates {

hub/internal/monitor/offsite_neverran_test.go

@@ -0,0 +1,193 @@
+package monitor
+
+import (
+	"io"
+	"log"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// Part-7 (v0.73.0): the never-ran offsite-staleness branch is ANCHORED — a newborn tier
+// (applied + escrowed + never ran) is not stale until the EXISTING 48 h threshold has elapsed
+// since the newest of one_time_secrets.consumed_at / the escrow-blob timestamp. Origin: the
+// 2026-07-23 cry-wolf — demo-hp escrowed 10:01Z, offsite_stale fired minutes later.
+// One state, one owner: pre-applied never-ran shapes belong to offsite_delivery_stuck alone.
+
+func newNeverRanStore(t *testing.T) *store.Store {
+	t.Helper()
+	st, err := store.New(filepath.Join(t.TempDir(), "nr.db"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	return st
+}
+
+func sqliteAgo(d time.Duration) string {
+	return time.Now().UTC().Add(-d).Format("2006-01-02 15:04:05")
+}
+
+// seedConsumedSecret stages + consumes a secret for the customer, back-dating consumed_at.
+func seedConsumedSecret(t *testing.T, st *store.Store, customerID string, consumedAgo time.Duration) {
+	t.Helper()
+	if err := st.SaveOneTimeSecret(customerID, "x"); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.SetOneTimeSecretTimesForTest(customerID, sqliteAgo(consumedAgo+time.Hour), sqliteAgo(consumedAgo)); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func neverRanChecker(st *store.Store, events *[]string) *OffsiteChecker {
+	return NewOffsiteChecker(st, 0, func(_, eventType, severity, _, _, _ string) {
+		*events = append(*events, eventType+":"+severity)
+	}, log.New(io.Discard, "", 0))
+}
+
+// The fix: applied + escrowed + never ran with a FRESH anchor (consumed 1 h ago) → NO event.
+// COMPANION RED-PROOF: replace the anchored never-ran branch with the pre-fix `return true` →
+// this fixture fires offsite_stale on the first sweep → FAIL observed → restored.
+func TestOffsiteStale_NeverRanFreshAnchor_Silent(t *testing.T) {
+	st := newNeverRanStore(t)
+	seedConsumedSecret(t, st, "c1", time.Hour)
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+	oc.Check()
+
+	if len(events) != 0 {
+		t.Fatalf("events = %v, want none — a newborn tier (anchor 1h ago, threshold 48h) is NOT stale", events)
+	}
+	if got := oc.GetStaleState("c1"); got != "ok" {
+		t.Fatalf("stale state = %q, want ok", got)
+	}
+}
+
+// The threshold still bites: anchor 49 h ago (consumed_at) + never ran → exactly one offsite_stale
+// with the existing copy shape.
+func TestOffsiteStale_NeverRanOldAnchor_FiresOnce(t *testing.T) {
+	st := newNeverRanStore(t)
+	seedConsumedSecret(t, st, "c1", 49*time.Hour)
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+	oc.Check() // deduped across sweeps
+
+	if len(events) != 1 || events[0] != "offsite_stale:warning" {
+		t.Fatalf("events = %v, want exactly [offsite_stale:warning]", events)
+	}
+	evs, err := st.GetRecentEvents("c1", 10)
+	if err != nil || len(evs) != 1 {
+		t.Fatalf("saved events = %v (%v), want exactly 1", evs, err)
+	}
+	if evs[0].EventType != "offsite_stale" {
+		t.Fatalf("event type = %s", evs[0].EventType)
+	}
+}
+
+// The escrow-timestamp leg of the anchor: consumed_at long ago but the CEREMONY (escrow blob)
+// happened 1 h ago → the newest anchor wins → silent. (The ceremony is the moment runs become
+// possible; a delayed ceremony must not make the tier instantly stale.)
+func TestOffsiteStale_NeverRanEscrowAnchorWins(t *testing.T) {
+	st := newNeverRanStore(t)
+	seedConsumedSecret(t, st, "c1", 72*time.Hour) // delivery long ago
+	if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k"}); err != nil {
+		t.Fatal(err)
+	}
+	// escrow blob stored now (SaveHostEscrow stamps updated_at with datetime('now'))
+	if _, err := st.SaveHostEscrow("h1", []byte("blob"), "fp", "zero_knowledge", time.Now().UTC().Format(time.RFC3339), "sha"); err != nil {
+		t.Fatal(err)
+	}
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+
+	if len(events) != 0 {
+		t.Fatalf("events = %v, want none — the newest anchor (escrow 'now') defers staleness", events)
+	}
+}
+
+// THE BOUNDARY (one state, one owner): a never-ran tier that is NOT applied (report carries no
+// offsite object) produces ZERO offsite_stale regardless of age — and offsite_delivery_stuck
+// (the v0.72.0 checker) remains that shape's only voice.
+func TestOffsiteStale_NotApplied_NeverFires_DeliveryCheckerOwnsIt(t *testing.T) {
+	st := newNeverRanStore(t)
+	// the burned-credential shape: consumed 49h ago, offsite ENABLED in config, report WITHOUT offsite
+	if err := st.SaveCustomerConfig(&store.CustomerConfig{
+		CustomerID: "c1", APIKey: "k", RetrievalPassword: "p",
+		ConfigJSON: `{"offsite":{"enabled":true,"type":"shared","host":"h","user":"u","repo_path":"/r","quota_gb":50}}`,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	seedConsumedSecret(t, st, "c1", 49*time.Hour)
+	saveOffsiteReport(t, st, "c1", "") // no offsite object = not applied
+
+	var staleEvents []string
+	oc := neverRanChecker(st, &staleEvents)
+	oc.Check()
+	if len(staleEvents) != 0 {
+		t.Fatalf("offsite_stale events = %v, want none — a pre-applied shape is NEVER offsite_stale's", staleEvents)
+	}
+
+	// ...and the delivery checker IS that shape's voice.
+	var deliveryEvents []string
+	dc := NewOffsiteDeliveryChecker(st, nil, func(_, eventType, severity, _, _, _ string) {
+		deliveryEvents = append(deliveryEvents, eventType+":"+severity)
+	}, log.New(io.Discard, "", 0))
+	dc.Check()
+	if len(deliveryEvents) != 1 || deliveryEvents[0] != "offsite_delivery_stuck:warning" {
+		t.Fatalf("delivery events = %v, want exactly [offsite_delivery_stuck:warning]", deliveryEvents)
+	}
+}
+
+// Legacy fallback: applied + escrowed + never ran with NO anchor at all (no secret row, no escrow
+// row) keeps the pre-fix behavior — stale on sight (fail toward visibility).
+func TestOffsiteStale_NeverRanNoAnchor_LegacyFailsTowardVisibility(t *testing.T) {
+	st := newNeverRanStore(t)
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+
+	if len(events) != 1 || events[0] != "offsite_stale:warning" {
+		t.Fatalf("events = %v, want [offsite_stale:warning] — the anchor-less legacy shape stays visible", events)
+	}
+}
+
+// Ran-before behavior byte-untouched: recent run silent, 49h-old run fires — with a consumed_at
+// anchor present that must NOT interfere in either direction.
+func TestOffsiteStale_RanBefore_Untouched(t *testing.T) {
+	st := newNeverRanStore(t)
+	seedConsumedSecret(t, st, "c1", time.Hour) // fresh anchor must not mask an old run
+	old := time.Now().UTC().Add(-49 * time.Hour).Format(time.RFC3339)
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", old, "ok", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+	if len(events) != 1 || events[0] != "offsite_stale:warning" {
+		t.Fatalf("events = %v, want [offsite_stale:warning] — a 49h-old run is stale regardless of a fresh anchor", events)
+	}
+
+	st2 := newNeverRanStore(t)
+	seedConsumedSecret(t, st2, "c2", 72*time.Hour) // old anchor must not stale a fresh run
+	recent := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339)
+	saveOffsiteReport(t, st2, "c2", offsiteJSON(true, "escrowed", recent, "ok", 0, 50))
+
+	var events2 []string
+	oc2 := neverRanChecker(st2, &events2)
+	oc2.Check()
+	if len(events2) != 0 {
+		t.Fatalf("events = %v, want none — a 1h-old run is fresh regardless of an old anchor", events2)
+	}
+}

hub/internal/store/store.go

@@ -967,6 +967,36 @@ func (s *Store) CountReportsOffsiteSince(customerID string, since time.Time) (to
 	return total, withOffsite, rows.Err()
 }
 
+// LatestEscrowTimeForCustomer returns the newest escrow-blob timestamp across the customer's
+// hosts (updated_at, falling back to created_at), or zero when no blob exists. Part-7 v0.73.0:
+// one leg of the never-ran offsite-staleness anchor — the escrow ceremony is the moment runs
+// become POSSIBLE, so the 48 h staleness threshold counts from it, not from the dawn of time.
+// Reduced in Go (not SQL MAX) because created_at formats vary (RFC3339 vs SQLite space form) and
+// lexicographic MAX would misorder them; parseSQLiteTime handles both.
+func (s *Store) LatestEscrowTimeForCustomer(customerID string) (time.Time, error) {
+	rows, err := s.db.Query(`
+		SELECT he.created_at, he.updated_at FROM host_escrow he
+		INNER JOIN hosts h ON h.host_id = he.host_id
+		WHERE h.customer_id = ?`, customerID)
+	if err != nil {
+		return time.Time{}, err
+	}
+	defer rows.Close()
+	var newest time.Time
+	for rows.Next() {
+		var createdAt, updatedAt string
+		if err := rows.Scan(&createdAt, &updatedAt); err != nil {
+			return time.Time{}, err
+		}
+		for _, raw := range []string{createdAt, updatedAt} {
+			if t := parseSQLiteTime(raw); t.After(newest) {
+				newest = t
+			}
+		}
+	}
+	return newest, rows.Err()
+}
+
 // LastEventAt returns the created_at of the most recent event of the given type for a customer
 // (zero time when none). Durable across hub restarts — used as the cooldown/rate-limit source for
 // hub-emitted detector events (R-70/R-71c: a repeating pattern must surface as repeating events on