hub v0.73.0 — offsite_stale anchored on newborn tiers (never-ran = applied-only + consumed_at/escrow anchor; one state one owner)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NKSN3gSg4TKVBBqkwW2djR
This commit is contained in:
@@ -73,7 +73,7 @@ func NewOffsiteChecker(s *store.Store, staleAfter time.Duration, onEvent EventNo
|
||||
oc.fillStates[c.CustomerID] = bandOK
|
||||
seeded++
|
||||
}
|
||||
if !oc.isStale(off) {
|
||||
if !oc.isStale(c.CustomerID, off) {
|
||||
oc.staleStates[c.CustomerID] = "ok"
|
||||
}
|
||||
}
|
||||
@@ -101,14 +101,28 @@ func (oc *OffsiteChecker) fillBand(off *offsiteReport) string {
|
||||
}
|
||||
|
||||
// isStale: enabled + ESCROWED (the only state where runs are expected) with no run in >staleAfter (or
|
||||
// never). Pending/disabled = normal onboarding, never stale. A recent-but-failing run is NOT stale
|
||||
// (backup_failed owns that signal).
|
||||
func (oc *OffsiteChecker) isStale(off *offsiteReport) bool {
|
||||
// never ran, ANCHORED — see below). Pending/disabled = normal onboarding, never stale. A
|
||||
// recent-but-failing run is NOT stale (backup_failed owns that signal).
|
||||
//
|
||||
// Part-7 (v0.73.0) — the never-ran branch no longer fires on sight. The 2026-07-23 cry-wolf:
|
||||
// demo-hp's tier was repaired and escrowed at 10:01Z and offsite_stale fired MINUTES later
|
||||
// (`last_run:"" … threshold 48h`), because "enabled + escrowed + never ran" had no time anchor.
|
||||
// Boundary: reaching this code at all means the latest report CARRIES the offsite object — i.e.
|
||||
// the v0.72.0 delivery state is `applied` (Check nil-skips everything else); pre-applied
|
||||
// never-ran shapes are offsite_delivery_stuck's alone — ONE STATE, ONE OWNER, never both.
|
||||
// Anchor: the newest of one_time_secrets.consumed_at (delivery completed) and the customer's
|
||||
// escrow-blob timestamp (host_escrow.updated_at/created_at — runs become POSSIBLE only at the
|
||||
// ceremony). The EXISTING staleAfter threshold, anchored there, IS the grace — no new knob.
|
||||
func (oc *OffsiteChecker) isStale(customerID string, off *offsiteReport) bool {
|
||||
if !off.Enabled || off.EscrowState != "escrowed" {
|
||||
return false
|
||||
}
|
||||
if off.LastRun == "" {
|
||||
return true
|
||||
anchor := oc.neverRanAnchor(customerID)
|
||||
if anchor.IsZero() {
|
||||
return true // legacy shape (no secret timestamps, no escrow row) — fail toward visibility, as before
|
||||
}
|
||||
return oc.now().Sub(anchor) > oc.staleAfter
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, off.LastRun)
|
||||
if err != nil {
|
||||
@@ -117,6 +131,19 @@ func (oc *OffsiteChecker) isStale(off *offsiteReport) bool {
|
||||
return oc.now().Sub(t) > oc.staleAfter
|
||||
}
|
||||
|
||||
// neverRanAnchor returns the newest hub-held timestamp from which a never-ran-but-applied tier's
|
||||
// staleness may be counted (zero when the hub holds neither — the pre-v0.5x legacy shape).
|
||||
func (oc *OffsiteChecker) neverRanAnchor(customerID string) time.Time {
|
||||
var anchor time.Time
|
||||
if info, err := oc.store.GetOneTimeSecretInfo(customerID); err == nil && info != nil && info.ConsumedAt.After(anchor) {
|
||||
anchor = info.ConsumedAt
|
||||
}
|
||||
if t, err := oc.store.LatestEscrowTimeForCustomer(customerID); err == nil && t.After(anchor) {
|
||||
anchor = t
|
||||
}
|
||||
return anchor
|
||||
}
|
||||
|
||||
// Check evaluates every customer's latest report. Escalation-only emits; recovery re-arms silently.
|
||||
func (oc *OffsiteChecker) Check() {
|
||||
customers, err := oc.store.GetCustomers()
|
||||
@@ -156,12 +183,21 @@ func (oc *OffsiteChecker) Check() {
|
||||
|
||||
// STALENESS (binary, warn-severity)
|
||||
newStale := "ok"
|
||||
if oc.isStale(off) {
|
||||
if oc.isStale(c.CustomerID, off) {
|
||||
newStale = "stale"
|
||||
}
|
||||
if newStale == "stale" && oc.staleStates[c.CustomerID] != "stale" {
|
||||
oc.emitStale(c.CustomerID, off)
|
||||
}
|
||||
// Part-7: make the anchored never-ran evaluation VISIBLE once (first observation of the
|
||||
// shape), so a live newborn tier's deferral is provable from the log without spamming
|
||||
// every sweep.
|
||||
if off.LastRun == "" && newStale == "ok" && off.Enabled && off.EscrowState == "escrowed" {
|
||||
if _, known := oc.staleStates[c.CustomerID]; !known {
|
||||
oc.logger.Printf("[INFO] Offsite staleness: %s never-ran within the anchored threshold (anchor %s) — newborn tier, not stale",
|
||||
c.CustomerID, oc.neverRanAnchor(c.CustomerID).UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
oc.staleStates[c.CustomerID] = newStale
|
||||
}
|
||||
for k := range oc.fillStates {
|
||||
|
||||
Reference in New Issue
Block a user