hub v0.72.0 — R-70 + R-71c: offsite delivery-state detector, card, stuck event, R-39(a)-guarded self-heal restage

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NKSN3gSg4TKVBBqkwW2djR
2026-07-23 12:59:04 +02:00
parent c801cee647
commit 1133aade73
13 changed files with 1179 additions and 3 deletions
@@ -0,0 +1,302 @@
+package monitor
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// R-70/R-71c checker tests. Real store, fake reissuer (records calls AND mimics the production
+// side effect — a restage IS a SaveOneTimeSecret — so a clobber is observable on the row itself),
+// captured onEvent, injected clock.
+
+const (
+	dtReportNoOffsite   = `{"health":{"status":"ok"}}`
+	dtReportWithOffsite = `{"health":{"status":"ok"},"offsite":{"enabled":true}}`
+	dtOffsiteConfig     = `{"offsite":{"enabled":true,"type":"shared","host":"h","user":"u","repo_path":"/home/felhom-repo","quota_gb":50}}`
+)
+
+type fakeReissuer struct {
+	mu    sync.Mutex
+	st    *store.Store
+	calls []string
+}
+
+func (f *fakeReissuer) ReissueOffsiteForCustomer(_ context.Context, customerID string) error {
+	f.mu.Lock()
+	f.calls = append(f.calls, customerID)
+	f.mu.Unlock()
+	// The production path's essential side effect: ReissueCredentials → SaveOneTimeSecret
+	// (last-write-wins clobber). Mimicked so the R-39(a) tests can observe what a wrongly-fired
+	// heal would DO to the row, not merely that it was called.
+	return f.st.SaveOneTimeSecret(customerID, "fresh-from-heal")
+}
+
+func (f *fakeReissuer) count() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.calls)
+}
+
+type dtHarness struct {
+	st       *store.Store
+	reissuer *fakeReissuer
+	checker  *OffsiteDeliveryChecker
+	events   *[]string // "type:severity"
+}
+
+func newDTHarness(t *testing.T, withReissuer bool) dtHarness {
+	t.Helper()
+	st, err := store.New(filepath.Join(t.TempDir(), "d.db"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	if err := st.SaveCustomerConfig(&store.CustomerConfig{
+		CustomerID: "c1", CustomerName: "C", Domain: "c1.hu", APIKey: "k", RetrievalPassword: "p",
+		ConfigJSON: dtOffsiteConfig,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	events := &[]string{}
+	var mu sync.Mutex
+	onEvent := func(_, eventType, severity, _, _, _ string) {
+		mu.Lock()
+		*events = append(*events, eventType+":"+severity)
+		mu.Unlock()
+	}
+	var ri *fakeReissuer
+	var riIface OffsiteReissuer
+	if withReissuer {
+		ri = &fakeReissuer{st: st}
+		riIface = ri
+	}
+	c := NewOffsiteDeliveryChecker(st, riIface, onEvent, log.New(io.Discard, "", 0))
+	return dtHarness{st: st, reissuer: ri, checker: c, events: events}
+}
+
+// burnedFixture puts c1 into the F10 shape: consumed >1h ago, N offbox-less reports since.
+func (h dtHarness) burnedFixture(t *testing.T, reports int) {
+	t.Helper()
+	if err := h.st.SaveOneTimeSecret("c1", "x"); err != nil {
+		t.Fatal(err)
+	}
+	consumed := time.Now().UTC().Add(-2 * time.Hour).Format("2006-01-02 15:04:05")
+	staged := time.Now().UTC().Add(-3 * time.Hour).Format("2006-01-02 15:04:05")
+	if err := h.st.SetOneTimeSecretTimesForTest("c1", staged, consumed); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < reports; i++ {
+		if err := h.st.SaveReport("c1", []byte(dtReportNoOffsite)); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func (h dtHarness) savedEvents(t *testing.T, eventType string) []store.Event {
+	t.Helper()
+	all, err := h.st.GetRecentEvents("c1", 50)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var out []store.Event
+	for _, e := range all {
+		if e.EventType == eventType {
+			out = append(out, e)
+		}
+	}
+	return out
+}
+
+// Scenario: the stuck event fires once with severity WARNING, and the durable 24h cooldown holds
+// across a second pass (delete the LastEventAt guard in maybeEmitStuck → this fails with 2 events —
+// the executable red-proof of the cooldown).
+func TestDeliveryChecker_StuckEvent_WarningOncePer24h(t *testing.T) {
+	h := newDTHarness(t, false)
+	h.burnedFixture(t, 5)
+
+	h.checker.Check()
+	h.checker.Check() // same tick shape again — cooldown must swallow it
+
+	saved := h.savedEvents(t, "offsite_delivery_stuck")
+	if len(saved) != 1 {
+		t.Fatalf("stuck events = %d, want exactly 1 (24h per-customer cooldown)", len(saved))
+	}
+	if saved[0].Severity != "warning" {
+		t.Fatalf("severity = %q, want warning (operator email tier; never info-silent, never critical)", saved[0].Severity)
+	}
+	var details map[string]any
+	if err := json.Unmarshal([]byte(saved[0].DetailsJSON), &details); err != nil || details["consumed_at"] == "" {
+		t.Fatalf("details must carry consumed_at (mióta), got %s err %v", saved[0].DetailsJSON, err)
+	}
+	if got := *h.events; len(got) != 1 || got[0] != "offsite_delivery_stuck:warning" {
+		t.Fatalf("dispatched = %v, want exactly [offsite_delivery_stuck:warning]", got)
+	}
+}
+
+// Scenario: the R-71c self-heal fires EXACTLY once — one reissue call, the restaged event
+// (warning), and the 24h rate limit blocks a re-trigger even when the shape recurs (delete the
+// LastEventAt guard in maybeHeal → the second pass fires again and this fails — the rate-limit
+// red-proof).
+func TestDeliveryChecker_SelfHeal_FiresOnceAndRateLimits(t *testing.T) {
+	h := newDTHarness(t, true)
+	h.burnedFixture(t, 4)
+
+	h.checker.Check()
+	if h.reissuer.count() != 1 {
+		t.Fatalf("reissue calls = %d, want 1", h.reissuer.count())
+	}
+	restaged := h.savedEvents(t, "offsite_credential_restaged")
+	if len(restaged) != 1 || restaged[0].Severity != "warning" {
+		t.Fatalf("restaged events = %v, want exactly 1 with severity warning (the operator ALWAYS learns a heal fired)", restaged)
+	}
+
+	// The shape recurs (box burned the fresh one too): back into consumed_awaiting_apply.
+	h.burnedFixture(t, 4)
+	h.checker.Check()
+	if h.reissuer.count() != 1 {
+		t.Fatalf("reissue calls after recurrence = %d, want STILL 1 (one restage per customer per 24h; repeats surface as events only)", h.reissuer.count())
+	}
+}
+
+// Not-enough-evidence gates: under healMinReports offbox-less reports → no heal; any offbox
+// evidence since consume (regressed-apply shape) → no heal. The stuck EVENT still fires (age gate
+// alone) — visibility never waits for the heal's stricter bar.
+func TestDeliveryChecker_SelfHeal_EvidenceGates(t *testing.T) {
+	h := newDTHarness(t, true)
+	h.burnedFixture(t, 3) // 3 < healMinReports
+	h.checker.Check()
+	if h.reissuer.count() != 0 {
+		t.Fatalf("reissue with 3 reports = %d calls, want 0 (needs >= 4)", h.reissuer.count())
+	}
+	if len(h.savedEvents(t, "offsite_delivery_stuck")) != 1 {
+		t.Fatal("stuck event must fire regardless of the heal's stricter evidence bar")
+	}
+
+	// add offbox evidence AFTER consume, then more offbox-less reports — mixed history: no heal
+	if err := h.st.SaveReport("c1", []byte(dtReportWithOffsite)); err != nil {
+		t.Fatal(err)
+	}
+	if err := h.st.SaveReport("c1", []byte(dtReportNoOffsite)); err != nil {
+		t.Fatal(err)
+	}
+	h.checker.Check()
+	if h.reissuer.count() != 0 {
+		t.Fatalf("reissue on mixed offbox history = %d calls, want 0 (regressed-apply is the operator's call)", h.reissuer.count())
+	}
+}
+
+// THE CLOBBER RED-PROOF (R-39(a), mandatory per spec): an operator Re-issue lands between the
+// checker's derive and its act (simulated via the onEvent hook, which runs after the stuck event
+// and before maybeHeal). The act-time guard re-reads the row, finds it UNCONSUMED, and refuses —
+// zero reissue calls, row untouched. Remove the `info.ConsumedAt.IsZero()` refusal in maybeHeal →
+// the fake fires SaveOneTimeSecret and this test FAILS on both assertions (calls=1, row clobbered)
+// — proving the fixture would have been clobbered.
+func TestDeliveryChecker_R39aGuard_NeverRestagesOverUnconsumed(t *testing.T) {
+	h := newDTHarness(t, true)
+	h.burnedFixture(t, 4)
+
+	// The TOCTOU: the moment the stuck event dispatches, the "operator" stages a fresh secret.
+	operatorStaged := "2026-07-23 12:00:00"
+	*h.events = nil
+	base := h.checker.onEvent
+	h.checker.onEvent = func(cid, et, sev, msg, det, src string) {
+		if et == "offsite_delivery_stuck" {
+			if err := h.st.SaveOneTimeSecret("c1", "operator-fresh"); err != nil {
+				t.Errorf("mid-tick stage: %v", err)
+			}
+			if err := h.st.SetOneTimeSecretTimesForTest("c1", operatorStaged, ""); err != nil {
+				t.Errorf("mid-tick stamp: %v", err)
+			}
+		}
+		base(cid, et, sev, msg, det, src)
+	}
+
+	h.checker.Check()
+
+	if h.reissuer.count() != 0 {
+		t.Fatalf("reissue calls = %d, want 0 — the R-39(a) guard must refuse over an unconsumed secret", h.reissuer.count())
+	}
+	info, err := h.st.GetOneTimeSecretInfo("c1")
+	if err != nil || info == nil {
+		t.Fatalf("secret row: %v / %v", info, err)
+	}
+	if !info.ConsumedAt.IsZero() || !info.CreatedAt.Equal(time.Date(2026, 7, 23, 12, 0, 0, 0, time.UTC)) {
+		t.Fatalf("the operator's fresh secret was CLOBBERED (created_at=%v consumed_at=%v) — R-39(a) violated", info.CreatedAt, info.ConsumedAt)
+	}
+	if len(h.savedEvents(t, "offsite_credential_restaged")) != 0 {
+		t.Fatal("no restaged event may exist for a refused heal")
+	}
+}
+
+// The demo-felhom live shape, full Check(): applied + stale unconsumed staged secret → ZERO events,
+// ZERO reissue calls, row untouched. Precedence (applied wins) is the first line of defense; the
+// R-39(a) guard is the second.
+func TestDeliveryChecker_AppliedWithStaleStaged_Untouched(t *testing.T) {
+	h := newDTHarness(t, true)
+	if err := h.st.SaveReport("c1", []byte(dtReportWithOffsite)); err != nil {
+		t.Fatal(err)
+	}
+	if err := h.st.SaveOneTimeSecret("c1", "stale"); err != nil {
+		t.Fatal(err)
+	}
+	if err := h.st.SetOneTimeSecretTimesForTest("c1", "2026-07-21 08:29:29", ""); err != nil {
+		t.Fatal(err)
+	}
+
+	h.checker.Check()
+
+	if h.reissuer.count() != 0 {
+		t.Fatalf("reissue calls = %d, want 0 (demo-felhom shape is healthy)", h.reissuer.count())
+	}
+	if len(*h.events) != 0 {
+		t.Fatalf("events = %v, want none", *h.events)
+	}
+	info, _ := h.st.GetOneTimeSecretInfo("c1")
+	if info == nil || !info.ConsumedAt.IsZero() || !info.CreatedAt.Equal(time.Date(2026, 7, 21, 8, 29, 29, 0, time.UTC)) {
+		t.Fatalf("fixture row mutated: %+v — the live specimen must survive the checker untouched", info)
+	}
+}
+
+// Guard rails: young consumed state (inside stuckAfter) is silent; offsite-disabled and
+// non-active customers are skipped entirely.
+func TestDeliveryChecker_QuietShapes(t *testing.T) {
+	h := newDTHarness(t, true)
+	// consumed 5 minutes ago — normal convergence window
+	if err := h.st.SaveOneTimeSecret("c1", "x"); err != nil {
+		t.Fatal(err)
+	}
+	recent := time.Now().UTC().Add(-5 * time.Minute).Format("2006-01-02 15:04:05")
+	if err := h.st.SetOneTimeSecretTimesForTest("c1", recent, recent); err != nil {
+		t.Fatal(err)
+	}
+	for i := 0; i < 5; i++ {
+		if err := h.st.SaveReport("c1", []byte(dtReportNoOffsite)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	h.checker.Check()
+	if len(*h.events) != 0 || h.reissuer.count() != 0 {
+		t.Fatalf("young consumed state must be silent, got events=%v calls=%d", *h.events, h.reissuer.count())
+	}
+
+	// offsite disabled → skipped even in a stuck-looking shape
+	if err := h.st.SaveCustomerConfig(&store.CustomerConfig{
+		CustomerID: "c1", CustomerName: "C", Domain: "c1.hu", APIKey: "k", RetrievalPassword: "p",
+		ConfigJSON: `{"offsite":{"enabled":false}}`,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	h.burnedFixture(t, 5)
+	h.checker.Check()
+	if len(*h.events) != 0 || h.reissuer.count() != 0 {
+		t.Fatalf("disabled offsite must be skipped, got events=%v calls=%d", *h.events, h.reissuer.count())
+	}
+}