Files
felhom.eu/hub/internal/monitor/offsite_neverran_test.go
T
admin 91cabdde1b
gates / gates (push) Successful in 7s
hub v0.93.0: the retention keeps the key it was built to keep (R-198) + three honesty fixes (R-197, R-192, R-196)
R-198 — host_escrow_superseded shipped with `blob` (the K-escrow / PBS datastore key) and
identity_blob was added to host_escrow LATER, never here. The offsite restic REPOSITORY
password lives in identity_blob. So demoteCurrentEscrowTx -- whose own comment calls it "THE
ONE escrow row-copy routine" -- retained the whole-guest key and silently dropped the off-site
data key, which is the secret the retention was built to preserve. And because the copy happens
as the new blob overwrites the old, the destroying act was the ESCROW CEREMONY: the exact thing
a rebuilt box tells its customer to run, on a card promising in Hungarian that the old backups
stay recoverable. Both demo boxes crossed that line on 2026-08-04.

  - identity_blob added to the table (CREATE + additive ALTER) and carried in the shared copy
    routine, so BOTH callers are fixed at once: re-escrow and host-delete demotion.
  - ListSupersededEscrow reads it back; store.HostEscrow gains IdentityBlob.
  - CountCurrentEscrowWithIdentity is the census of who the fix protects.
  - Nothing is backfillable: pre-v0.93.0 retained rows have no blob and their sources are gone.
  - Tests assert the CONSEQUENCE (a retained row can still yield a repo password), which is why
    the pre-existing retention test stayed green for two months asserting the mechanism.

R-197 — SaveHostEscrow returns the hash it replaced; the escrow PUT raises
offsite_repo_key_changed (warning, operator-only, edge-triggered) when both hashes are known and
differ. No hash value travels. Severity chosen for the world v0.93.0 creates: with the identity
blob retained, a changed key is "this history now depends on an older recovery code", not a loss.

R-192 (half) — the stuck alert now reports the two shapes it actually covers, burned and
regressed, each stating its own measurement; the regressed text withdraws the Re-issue
recommendation. Every self-heal refusal leaves a notification_log row with its reason. The
guard's logic is unchanged; its 500-oldest-reports scoping stays OPEN and the window is named in
the alert text so the limitation travels with the number. offsite_delivery_stuck and
offsite_credential_restaged are added to operatorOnlyEvents -- neither was registered and neither
has a customerMessages entry, which is not a block.

R-196 — five comments (not the three the spec expected) claimed ReissueCredentials rotates the
restic repo password. It resets the PROVIDER password and cannot touch the repo password, which
is generated on the box. All five corrected; the staleness mark documented as precautionary. The
BEHAVIOUR stays open.

Not in this release: R-199, R-200, R-201 remain open -- the chain that hands the key back is
still unassembled. Part 5 hit its gate; the orphan card is untouched (R-202).
2026-08-04 12:56:58 +02:00

194 lines
7.5 KiB
Go

package monitor
import (
"io"
"log"
"path/filepath"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
// Part-7 (v0.73.0): the never-ran offsite-staleness branch is ANCHORED — a newborn tier
// (applied + escrowed + never ran) is not stale until the EXISTING 48 h threshold has elapsed
// since the newest of one_time_secrets.consumed_at / the escrow-blob timestamp. Origin: the
// 2026-07-23 cry-wolf — demo-hp escrowed 10:01Z, offsite_stale fired minutes later.
// One state, one owner: pre-applied never-ran shapes belong to offsite_delivery_stuck alone.
func newNeverRanStore(t *testing.T) *store.Store {
t.Helper()
st, err := store.New(filepath.Join(t.TempDir(), "nr.db"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
return st
}
func sqliteAgo(d time.Duration) string {
return time.Now().UTC().Add(-d).Format("2006-01-02 15:04:05")
}
// seedConsumedSecret stages + consumes a secret for the customer, back-dating consumed_at.
func seedConsumedSecret(t *testing.T, st *store.Store, customerID string, consumedAgo time.Duration) {
t.Helper()
if err := st.SaveOneTimeSecret(customerID, "x"); err != nil {
t.Fatal(err)
}
if err := st.SetOneTimeSecretTimesForTest(customerID, sqliteAgo(consumedAgo+time.Hour), sqliteAgo(consumedAgo)); err != nil {
t.Fatal(err)
}
}
func neverRanChecker(st *store.Store, events *[]string) *OffsiteChecker {
return NewOffsiteChecker(st, 0, func(_, eventType, severity, _, _, _ string) {
*events = append(*events, eventType+":"+severity)
}, log.New(io.Discard, "", 0))
}
// The fix: applied + escrowed + never ran with a FRESH anchor (consumed 1 h ago) → NO event.
// COMPANION RED-PROOF: replace the anchored never-ran branch with the pre-fix `return true` →
// this fixture fires offsite_stale on the first sweep → FAIL observed → restored.
func TestOffsiteStale_NeverRanFreshAnchor_Silent(t *testing.T) {
st := newNeverRanStore(t)
seedConsumedSecret(t, st, "c1", time.Hour)
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
var events []string
oc := neverRanChecker(st, &events)
oc.Check()
oc.Check()
if len(events) != 0 {
t.Fatalf("events = %v, want none — a newborn tier (anchor 1h ago, threshold 48h) is NOT stale", events)
}
if got := oc.GetStaleState("c1"); got != "ok" {
t.Fatalf("stale state = %q, want ok", got)
}
}
// The threshold still bites: anchor 49 h ago (consumed_at) + never ran → exactly one offsite_stale
// with the existing copy shape.
func TestOffsiteStale_NeverRanOldAnchor_FiresOnce(t *testing.T) {
st := newNeverRanStore(t)
seedConsumedSecret(t, st, "c1", 49*time.Hour)
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
var events []string
oc := neverRanChecker(st, &events)
oc.Check()
oc.Check() // deduped across sweeps
if len(events) != 1 || events[0] != "offsite_stale:warning" {
t.Fatalf("events = %v, want exactly [offsite_stale:warning]", events)
}
evs, err := st.GetRecentEvents("c1", 10)
if err != nil || len(evs) != 1 {
t.Fatalf("saved events = %v (%v), want exactly 1", evs, err)
}
if evs[0].EventType != "offsite_stale" {
t.Fatalf("event type = %s", evs[0].EventType)
}
}
// The escrow-timestamp leg of the anchor: consumed_at long ago but the CEREMONY (escrow blob)
// happened 1 h ago → the newest anchor wins → silent. (The ceremony is the moment runs become
// possible; a delayed ceremony must not make the tier instantly stale.)
func TestOffsiteStale_NeverRanEscrowAnchorWins(t *testing.T) {
st := newNeverRanStore(t)
seedConsumedSecret(t, st, "c1", 72*time.Hour) // delivery long ago
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k"}); err != nil {
t.Fatal(err)
}
// escrow blob stored now (SaveHostEscrow stamps updated_at with datetime('now'))
if _, _, err := st.SaveHostEscrow("h1", []byte("blob"), "fp", "zero_knowledge", time.Now().UTC().Format(time.RFC3339), "sha"); err != nil {
t.Fatal(err)
}
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
var events []string
oc := neverRanChecker(st, &events)
oc.Check()
if len(events) != 0 {
t.Fatalf("events = %v, want none — the newest anchor (escrow 'now') defers staleness", events)
}
}
// THE BOUNDARY (one state, one owner): a never-ran tier that is NOT applied (report carries no
// offsite object) produces ZERO offsite_stale regardless of age — and offsite_delivery_stuck
// (the v0.72.0 checker) remains that shape's only voice.
func TestOffsiteStale_NotApplied_NeverFires_DeliveryCheckerOwnsIt(t *testing.T) {
st := newNeverRanStore(t)
// the burned-credential shape: consumed 49h ago, offsite ENABLED in config, report WITHOUT offsite
if err := st.SaveCustomerConfig(&store.CustomerConfig{
CustomerID: "c1", APIKey: "k", RetrievalPassword: "p",
ConfigJSON: `{"offsite":{"enabled":true,"type":"shared","host":"h","user":"u","repo_path":"/r","quota_gb":50}}`,
}); err != nil {
t.Fatal(err)
}
seedConsumedSecret(t, st, "c1", 49*time.Hour)
saveOffsiteReport(t, st, "c1", "") // no offsite object = not applied
var staleEvents []string
oc := neverRanChecker(st, &staleEvents)
oc.Check()
if len(staleEvents) != 0 {
t.Fatalf("offsite_stale events = %v, want none — a pre-applied shape is NEVER offsite_stale's", staleEvents)
}
// ...and the delivery checker IS that shape's voice.
var deliveryEvents []string
dc := NewOffsiteDeliveryChecker(st, nil, func(_, eventType, severity, _, _, _ string) {
deliveryEvents = append(deliveryEvents, eventType+":"+severity)
}, log.New(io.Discard, "", 0))
dc.Check()
if len(deliveryEvents) != 1 || deliveryEvents[0] != "offsite_delivery_stuck:warning" {
t.Fatalf("delivery events = %v, want exactly [offsite_delivery_stuck:warning]", deliveryEvents)
}
}
// Legacy fallback: applied + escrowed + never ran with NO anchor at all (no secret row, no escrow
// row) keeps the pre-fix behavior — stale on sight (fail toward visibility).
func TestOffsiteStale_NeverRanNoAnchor_LegacyFailsTowardVisibility(t *testing.T) {
st := newNeverRanStore(t)
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", "", "", 0, 50))
var events []string
oc := neverRanChecker(st, &events)
oc.Check()
if len(events) != 1 || events[0] != "offsite_stale:warning" {
t.Fatalf("events = %v, want [offsite_stale:warning] — the anchor-less legacy shape stays visible", events)
}
}
// Ran-before behavior byte-untouched: recent run silent, 49h-old run fires — with a consumed_at
// anchor present that must NOT interfere in either direction.
func TestOffsiteStale_RanBefore_Untouched(t *testing.T) {
st := newNeverRanStore(t)
seedConsumedSecret(t, st, "c1", time.Hour) // fresh anchor must not mask an old run
old := time.Now().UTC().Add(-49 * time.Hour).Format(time.RFC3339)
saveOffsiteReport(t, st, "c1", offsiteJSON(true, "escrowed", old, "ok", 0, 50))
var events []string
oc := neverRanChecker(st, &events)
oc.Check()
if len(events) != 1 || events[0] != "offsite_stale:warning" {
t.Fatalf("events = %v, want [offsite_stale:warning] — a 49h-old run is stale regardless of a fresh anchor", events)
}
st2 := newNeverRanStore(t)
seedConsumedSecret(t, st2, "c2", 72*time.Hour) // old anchor must not stale a fresh run
recent := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339)
saveOffsiteReport(t, st2, "c2", offsiteJSON(true, "escrowed", recent, "ok", 0, 50))
var events2 []string
oc2 := neverRanChecker(st2, &events2)
oc2.Check()
if len(events2) != 0 {
t.Fatalf("events = %v, want none — a 1h-old run is fresh regardless of an old anchor", events2)
}
}