hub v0.73.0 — offsite_stale anchored on newborn tiers (never-ran = applied-only + consumed_at/escrow anchor; one state one owner)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NKSN3gSg4TKVBBqkwW2djR
2026-07-23 13:25:22 +02:00
parent 527d81cf70
commit b03a53ddcf
4 changed files with 297 additions and 7 deletions
@@ -73,7 +73,7 @@ func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNo
 			oc.fillStates[c.CustomerID] = bandOK
 			seeded++
 		}
-		if !oc.isStale(off) {
+		if !oc.isStale(c.CustomerID, off) {
 			oc.staleStates[c.CustomerID] = "ok"
 		}
 	}
@@ -101,14 +101,28 @@ func (oc *OffsiteChecker) fillBand(off *offsiteReport) string {
 }

 // isStale: enabled + ESCROWED (the only state where runs are expected) with no run in >staleAfter (or
-// never). Pending/disabled = normal onboarding, never stale. A recent-but-failing run is NOT stale
-// (backup_failed owns that signal).
-func (oc *OffsiteChecker) isStale(off *offsiteReport) bool {
+// never ran, ANCHORED — see below). Pending/disabled = normal onboarding, never stale. A
+// recent-but-failing run is NOT stale (backup_failed owns that signal).
+//
+// Part-7 (v0.73.0) — the never-ran branch no longer fires on sight. The 2026-07-23 cry-wolf:
+// demo-hp's tier was repaired and escrowed at 10:01Z and offsite_stale fired MINUTES later
+// (`last_run:"" … threshold 48h`), because "enabled + escrowed + never ran" had no time anchor.
+// Boundary: reaching this code at all means the latest report CARRIES the offsite object — i.e.
+// the v0.72.0 delivery state is `applied` (Check nil-skips everything else); pre-applied
+// never-ran shapes are offsite_delivery_stuck's alone — ONE STATE, ONE OWNER, never both.
+// Anchor: the newest of one_time_secrets.consumed_at (delivery completed) and the customer's
+// escrow-blob timestamp (host_escrow.updated_at/created_at — runs become POSSIBLE only at the
+// ceremony). The EXISTING staleAfter threshold, anchored there, IS the grace — no new knob.
+func (oc *OffsiteChecker) isStale(customerID string, off *offsiteReport) bool {
 	if !off.Enabled || off.EscrowState != "escrowed" {
 		return false
 	}
 	if off.LastRun == "" {
-		return true
+		anchor := oc.neverRanAnchor(customerID)
+		if anchor.IsZero() {
+			return true // legacy shape (no secret timestamps, no escrow row) — fail toward visibility, as before
+		}
+		return oc.now().Sub(anchor) > oc.staleAfter
 	}
 	t, err := time.Parse(time.RFC3339, off.LastRun)
 	if err != nil {
@@ -117,6 +131,19 @@ func (oc *OffsiteChecker) isStale(off *offsiteReport) bool {
 	return oc.now().Sub(t) > oc.staleAfter
 }

+// neverRanAnchor returns the newest hub-held timestamp from which a never-ran-but-applied tier's
+// staleness may be counted (zero when the hub holds neither — the pre-v0.5x legacy shape).
+func (oc *OffsiteChecker) neverRanAnchor(customerID string) time.Time {
+	var anchor time.Time
+	if info, err := oc.store.GetOneTimeSecretInfo(customerID); err == nil && info != nil && info.ConsumedAt.After(anchor) {
+		anchor = info.ConsumedAt
+	}
+	if t, err := oc.store.LatestEscrowTimeForCustomer(customerID); err == nil && t.After(anchor) {
+		anchor = t
+	}
+	return anchor
+}
+
 // Check evaluates every customer's latest report. Escalation-only emits; recovery re-arms silently.
 func (oc *OffsiteChecker) Check() {
 	customers, err := oc.store.GetCustomers()
@@ -156,12 +183,21 @@ func (oc *OffsiteChecker) Check() {

 		// STALENESS (binary, warn-severity)
 		newStale := "ok"
-		if oc.isStale(off) {
+		if oc.isStale(c.CustomerID, off) {
 			newStale = "stale"
 		}
 		if newStale == "stale" && oc.staleStates[c.CustomerID] != "stale" {
 			oc.emitStale(c.CustomerID, off)
 		}
+		// Part-7: make the anchored never-ran evaluation VISIBLE once (first observation of the
+		// shape), so a live newborn tier's deferral is provable from the log without spamming
+		// every sweep.
+		if off.LastRun == "" && newStale == "ok" && off.Enabled && off.EscrowState == "escrowed" {
+			if _, known := oc.staleStates[c.CustomerID]; !known {
+				oc.logger.Printf("[INFO] Offsite staleness: %s never-ran within the anchored threshold (anchor %s) — newborn tier, not stale",
+					c.CustomerID, oc.neverRanAnchor(c.CustomerID).UTC().Format(time.RFC3339))
+			}
+		}
 		oc.staleStates[c.CustomerID] = newStale
 	}
 	for k := range oc.fillStates {
@@ -0,0 +1,193 @@
+package monitor
+
+import (
+	"io"
+	"log"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// Part-7 (v0.73.0): the never-ran offsite-staleness branch is ANCHORED — a newborn tier
+// (applied + escrowed + never ran) is not stale until the EXISTING 48 h threshold has elapsed
+// since the newest of one_time_secrets.consumed_at / the escrow-blob timestamp. Origin: the
+// 2026-07-23 cry-wolf — demo-hp escrowed 10:01Z, offsite_stale fired minutes later.
+// One state, one owner: pre-applied never-ran shapes belong to offsite_delivery_stuck alone.
+
+func newNeverRanStore(t *testing.T) *store.Store {
+	t.Helper()
+	st, err := store.New(filepath.Join(t.TempDir(), "nr.db"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	return st
+}
+
+func sqliteAgo(d time.Duration) string {
+	return time.Now().UTC().Add(-d).Format("2006-01-02 15:04:05")
+}
+
+// seedConsumedSecret stages + consumes a secret for the customer, back-dating consumed_at.
+func seedConsumedSecret(t *testing.T, st *store.Store, customerID string, consumedAgo time.Duration) {
+	t.Helper()
+	if err := st.SaveOneTimeSecret(customerID, "x"); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.SetOneTimeSecretTimesForTest(customerID, sqliteAgo(consumedAgo+time.Hour), sqliteAgo(consumedAgo)); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func neverRanChecker(st *store.Store, events *[]string) *OffsiteChecker {
+	return NewOffsiteChecker(st, 0, func(_, eventType, severity, _, _, _ string) {
+		*events = append(*events, eventType+":"+severity)
+	}, log.New(io.Discard, "", 0))
+}
+
+// The fix: applied + escrowed + never ran with a FRESH anchor (consumed 1 h ago) → NO event.
+// COMPANION RED-PROOF: replace the anchored never-ran branch with the pre-fix `return true` →
+// this fixture fires offsite_stale on the first sweep → FAIL observed → restored.
+func TestOffsiteStale_NeverRanFreshAnchor_Silent(t *testing.T) {
+	st := newNeverRanStore(t)
+	seedConsumedSecret(t, st, "c1", time.Hour)
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+	oc.Check()
+
+	if len(events) != 0 {
+		t.Fatalf("events = %v, want none — a newborn tier (anchor 1h ago, threshold 48h) is NOT stale", events)
+	}
+	if got := oc.GetStaleState("c1"); got != "ok" {
+		t.Fatalf("stale state = %q, want ok", got)
+	}
+}
+
+// The threshold still bites: anchor 49 h ago (consumed_at) + never ran → exactly one offsite_stale
+// with the existing copy shape.
+func TestOffsiteStale_NeverRanOldAnchor_FiresOnce(t *testing.T) {
+	st := newNeverRanStore(t)
+	seedConsumedSecret(t, st, "c1", 49*time.Hour)
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+	oc.Check() // deduped across sweeps
+
+	if len(events) != 1 || events[0] != "offsite_stale:warning" {
+		t.Fatalf("events = %v, want exactly [offsite_stale:warning]", events)
+	}
+	evs, err := st.GetRecentEvents("c1", 10)
+	if err != nil || len(evs) != 1 {
+		t.Fatalf("saved events = %v (%v), want exactly 1", evs, err)
+	}
+	if evs[0].EventType != "offsite_stale" {
+		t.Fatalf("event type = %s", evs[0].EventType)
+	}
+}
+
+// The escrow-timestamp leg of the anchor: consumed_at long ago but the CEREMONY (escrow blob)
+// happened 1 h ago → the newest anchor wins → silent. (The ceremony is the moment runs become
+// possible; a delayed ceremony must not make the tier instantly stale.)
+func TestOffsiteStale_NeverRanEscrowAnchorWins(t *testing.T) {
+	st := newNeverRanStore(t)
+	seedConsumedSecret(t, st, "c1", 72*time.Hour) // delivery long ago
+	if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k"}); err != nil {
+		t.Fatal(err)
+	}
+	// escrow blob stored now (SaveHostEscrow stamps updated_at with datetime('now'))
+	if _, err := st.SaveHostEscrow("h1", []byte("blob"), "fp", "zero_knowledge", time.Now().UTC().Format(time.RFC3339), "sha"); err != nil {
+		t.Fatal(err)
+	}
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+
+	if len(events) != 0 {
+		t.Fatalf("events = %v, want none — the newest anchor (escrow 'now') defers staleness", events)
+	}
+}
+
+// THE BOUNDARY (one state, one owner): a never-ran tier that is NOT applied (report carries no
+// offsite object) produces ZERO offsite_stale regardless of age — and offsite_delivery_stuck
+// (the v0.72.0 checker) remains that shape's only voice.
+func TestOffsiteStale_NotApplied_NeverFires_DeliveryCheckerOwnsIt(t *testing.T) {
+	st := newNeverRanStore(t)
+	// the burned-credential shape: consumed 49h ago, offsite ENABLED in config, report WITHOUT offsite
+	if err := st.SaveCustomerConfig(&store.CustomerConfig{
+		CustomerID: "c1", APIKey: "k", RetrievalPassword: "p",
+		ConfigJSON: `{"offsite":{"enabled":true,"type":"shared","host":"h","user":"u","repo_path":"/r","quota_gb":50}}`,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	seedConsumedSecret(t, st, "c1", 49*time.Hour)
+	saveOffsiteReport(t, st, "c1", "") // no offsite object = not applied
+
+	var staleEvents []string
+	oc := neverRanChecker(st, &staleEvents)
+	oc.Check()
+	if len(staleEvents) != 0 {
+		t.Fatalf("offsite_stale events = %v, want none — a pre-applied shape is NEVER offsite_stale's", staleEvents)
+	}
+
+	// ...and the delivery checker IS that shape's voice.
+	var deliveryEvents []string
+	dc := NewOffsiteDeliveryChecker(st, nil, func(_, eventType, severity, _, _, _ string) {
+		deliveryEvents = append(deliveryEvents, eventType+":"+severity)
+	}, log.New(io.Discard, "", 0))
+	dc.Check()
+	if len(deliveryEvents) != 1 || deliveryEvents[0] != "offsite_delivery_stuck:warning" {
+		t.Fatalf("delivery events = %v, want exactly [offsite_delivery_stuck:warning]", deliveryEvents)
+	}
+}
+
+// Legacy fallback: applied + escrowed + never ran with NO anchor at all (no secret row, no escrow
+// row) keeps the pre-fix behavior — stale on sight (fail toward visibility).
+func TestOffsiteStale_NeverRanNoAnchor_LegacyFailsTowardVisibility(t *testing.T) {
+	st := newNeverRanStore(t)
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+
+	if len(events) != 1 || events[0] != "offsite_stale:warning" {
+		t.Fatalf("events = %v, want [offsite_stale:warning] — the anchor-less legacy shape stays visible", events)
+	}
+}
+
+// Ran-before behavior byte-untouched: recent run silent, 49h-old run fires — with a consumed_at
+// anchor present that must NOT interfere in either direction.
+func TestOffsiteStale_RanBefore_Untouched(t *testing.T) {
+	st := newNeverRanStore(t)
+	seedConsumedSecret(t, st, "c1", time.Hour) // fresh anchor must not mask an old run
+	old := time.Now().UTC().Add(-49 * time.Hour).Format(time.RFC3339)
+	saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", old, "ok", 0, 50))
+
+	var events []string
+	oc := neverRanChecker(st, &events)
+	oc.Check()
+	if len(events) != 1 || events[0] != "offsite_stale:warning" {
+		t.Fatalf("events = %v, want [offsite_stale:warning] — a 49h-old run is stale regardless of a fresh anchor", events)
+	}
+
+	st2 := newNeverRanStore(t)
+	seedConsumedSecret(t, st2, "c2", 72*time.Hour) // old anchor must not stale a fresh run
+	recent := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339)
+	saveOffsiteReport(t, st2, "c2", offsiteJSON(true, "escrowed", recent, "ok", 0, 50))
+
+	var events2 []string
+	oc2 := neverRanChecker(st2, &events2)
+	oc2.Check()
+	if len(events2) != 0 {
+		t.Fatalf("events = %v, want none — a 1h-old run is fresh regardless of an old anchor", events2)
+	}
+}