hub v0.73.0 — offsite_stale anchored on newborn tiers (never-ran = applied-only + consumed_at/escrow anchor; one state one owner)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NKSN3gSg4TKVBBqkwW2djR
This commit is contained in:
2026-07-23 13:25:22 +02:00
parent 527d81cf70
commit b03a53ddcf
4 changed files with 297 additions and 7 deletions
+42 -6
View File
@@ -73,7 +73,7 @@ func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNo
oc.fillStates[c.CustomerID] = bandOK
seeded++
}
if !oc.isStale(off) {
if !oc.isStale(c.CustomerID, off) {
oc.staleStates[c.CustomerID] = "ok"
}
}
@@ -101,14 +101,28 @@ func (oc *OffsiteChecker) fillBand(off *offsiteReport) string {
}
// isStale: enabled + ESCROWED (the only state where runs are expected) with no run in >staleAfter (or
// never). Pending/disabled = normal onboarding, never stale. A recent-but-failing run is NOT stale
// (backup_failed owns that signal).
func (oc *OffsiteChecker) isStale(off *offsiteReport) bool {
// never ran, ANCHORED — see below). Pending/disabled = normal onboarding, never stale. A
// recent-but-failing run is NOT stale (backup_failed owns that signal).
//
// Part-7 (v0.73.0) — the never-ran branch no longer fires on sight. The 2026-07-23 cry-wolf:
// demo-hp's tier was repaired and escrowed at 10:01Z and offsite_stale fired MINUTES later
// (`last_run:"" … threshold 48h`), because "enabled + escrowed + never ran" had no time anchor.
// Boundary: reaching this code at all means the latest report CARRIES the offsite object — i.e.
// the v0.72.0 delivery state is `applied` (Check nil-skips everything else); pre-applied
// never-ran shapes are offsite_delivery_stuck's alone — ONE STATE, ONE OWNER, never both.
// Anchor: the newest of one_time_secrets.consumed_at (delivery completed) and the customer's
// escrow-blob timestamp (host_escrow.updated_at/created_at — runs become POSSIBLE only at the
// ceremony). The EXISTING staleAfter threshold, anchored there, IS the grace — no new knob.
func (oc *OffsiteChecker) isStale(customerID string, off *offsiteReport) bool {
if !off.Enabled || off.EscrowState != "escrowed" {
return false
}
if off.LastRun == "" {
return true
anchor := oc.neverRanAnchor(customerID)
if anchor.IsZero() {
return true // legacy shape (no secret timestamps, no escrow row) — fail toward visibility, as before
}
return oc.now().Sub(anchor) > oc.staleAfter
}
t, err := time.Parse(time.RFC3339, off.LastRun)
if err != nil {
@@ -117,6 +131,19 @@ func (oc *OffsiteChecker) isStale(off *offsiteReport) bool {
return oc.now().Sub(t) > oc.staleAfter
}
// neverRanAnchor returns the newest hub-held timestamp from which a never-ran-but-applied tier's
// staleness may be counted (zero when the hub holds neither — the pre-v0.5x legacy shape).
func (oc *OffsiteChecker) neverRanAnchor(customerID string) time.Time {
var anchor time.Time
if info, err := oc.store.GetOneTimeSecretInfo(customerID); err == nil && info != nil && info.ConsumedAt.After(anchor) {
anchor = info.ConsumedAt
}
if t, err := oc.store.LatestEscrowTimeForCustomer(customerID); err == nil && t.After(anchor) {
anchor = t
}
return anchor
}
// Check evaluates every customer's latest report. Escalation-only emits; recovery re-arms silently.
func (oc *OffsiteChecker) Check() {
customers, err := oc.store.GetCustomers()
@@ -156,12 +183,21 @@ func (oc *OffsiteChecker) Check() {
// STALENESS (binary, warn-severity)
newStale := "ok"
if oc.isStale(off) {
if oc.isStale(c.CustomerID, off) {
newStale = "stale"
}
if newStale == "stale" && oc.staleStates[c.CustomerID] != "stale" {
oc.emitStale(c.CustomerID, off)
}
// Part-7: make the anchored never-ran evaluation VISIBLE once (first observation of the
// shape), so a live newborn tier's deferral is provable from the log without spamming
// every sweep.
if off.LastRun == "" && newStale == "ok" && off.Enabled && off.EscrowState == "escrowed" {
if _, known := oc.staleStates[c.CustomerID]; !known {
oc.logger.Printf("[INFO] Offsite staleness: %s never-ran within the anchored threshold (anchor %s) — newborn tier, not stale",
c.CustomerID, oc.neverRanAnchor(c.CustomerID).UTC().Format(time.RFC3339))
}
}
oc.staleStates[c.CustomerID] = newStale
}
for k := range oc.fillStates {