felhom.eu/hub/internal/monitor/restoretest_test.go

package monitor

import (
	"encoding/json"
	"io"
	"log"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// R-85 Part 2 — the restore-test result must be HEARD.
//
// Before this, a failed restore-test was a `[WARN]` line in the ingest handler and nothing else —
// no event, no notification, no gauge. That was true for the LOCAL tier that was already being
// tested, which means the loudest DR signal this system produces was in practice inaudible.

type capturedEvent struct {
	customerID, eventType, severity, message string
}

func rtStore(t *testing.T) *store.Store {
	t.Helper()
	st, err := store.New(filepath.Join(t.TempDir(), "rt.db"), log.New(io.Discard, "", 0))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { st.Close() })
	if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "k", RetrievalPassword: "p"}); err != nil {
		t.Fatal(err)
	}
	if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
		t.Fatal(err)
	}
	return st
}

// rtReport builds a host-report carrying storage targets and zero or one restore-test.
func rtReport(t *testing.T, tier string, pass bool, testedAt time.Time, errMsg string) string {
	t.Helper()
	type rt struct {
		SourceArchive string `json:"source_archive"`
		SourceTier    string `json:"source_tier"`
		Pass          bool   `json:"pass"`
		Error         string `json:"error,omitempty"`
		TestedAt      string `json:"tested_at"`
	}
	type stg struct {
		Name    string `json:"name"`
		Type    string `json:"type"`
		Content string `json:"content"`
	}
	payload := struct {
		RestoreTests   []rt  `json:"restore_tests"`
		StorageTargets []stg `json:"storage_targets"`
	}{
		StorageTargets: []stg{
			{Name: "local", Type: "local", Content: "backup,iso"},
			{Name: "felhom-pbs", Type: "pbs", Content: "backup"},
		},
	}
	if tier != "" {
		payload.RestoreTests = []rt{{
			SourceArchive: tier + ":backup/ct/9201/x", SourceTier: tier,
			Pass: pass, Error: errMsg, TestedAt: testedAt.UTC().Format(time.RFC3339),
		}}
	}
	b, err := json.Marshal(payload)
	if err != nil {
		t.Fatal(err)
	}
	return string(b)
}

func rtChecker(st *store.Store, got *[]capturedEvent, logTo io.Writer) *RestoreTestChecker {
	if logTo == nil {
		logTo = io.Discard
	}
	return NewRestoreTestChecker(st, func(cid, et, sev, msg, _, _ string) {
		*got = append(*got, capturedEvent{cid, et, sev, msg})
	}, log.New(logTo, "", 0))
}

// ── SCENARIO B — a failed restore-test is HEARD ──────────────────────────────────────────────
//
// The assertion is that a NOTIFICATION IS EMITTED, not that a handler ran. The hollow version of
// this test checks for a log line, which passes against exactly the code this task replaces.
//
// COMPANION RED-PROOF (observed): make checkFailure `return` before emit (the pre-R-85 shape — log
// and stop) and this fails with "a FAILED restore-test must EMIT an operator event; got 0 event(s)".
// Restored.
func TestRestoreTest_FailureEmitsAnOperatorEvent(t *testing.T) {
	st := rtStore(t)
	now := time.Now().UTC()
	if err := st.SaveHostReport("h1", "c1",
		[]byte(rtReport(t, "pbs", false, now, "restore task: context deadline exceeded")),
		store.HostReportDenorm{}); err != nil {
		t.Fatal(err)
	}
	var got []capturedEvent
	rtChecker(st, &got, nil).Check()

	var fail *capturedEvent
	for i := range got {
		if got[i].eventType == EventRestoreTestFailed {
			fail = &got[i]
		}
	}
	if fail == nil {
		t.Fatalf("a FAILED restore-test must EMIT an operator event; got %d event(s): %+v", len(got), got)
	}
	if fail.severity != "error" {
		t.Fatalf("a failed restore-test is an error, got severity %q", fail.severity)
	}
	if !strings.Contains(fail.message, "pbs") {
		t.Fatalf("the TIER must be named — 'a restore-test failed' without saying which tier is not actionable; got %q", fail.message)
	}
}

// Edge-triggered: a permanently failing tier must not emit on every sweep.
func TestRestoreTest_FailureDoesNotSpam(t *testing.T) {
	st := rtStore(t)
	now := time.Now().UTC()
	st.SaveHostReport("h1", "c1", []byte(rtReport(t, "pbs", false, now, "boom")), store.HostReportDenorm{})
	var got []capturedEvent
	c := rtChecker(st, &got, nil)
	c.Check()
	c.Check()
	c.Check()
	n := 0
	for _, e := range got {
		if e.eventType == EventRestoreTestFailed {
			n++
		}
	}
	if n != 1 {
		t.Fatalf("a persistently failing tier must report ONCE per failing run, got %d", n)
	}
}

// A PASS must not emit a failure event.
func TestRestoreTest_PassEmitsNoFailure(t *testing.T) {
	st := rtStore(t)
	now := time.Now().UTC()
	st.SaveHostReport("h1", "c1", []byte(rtReport(t, "pbs", true, now, "")), store.HostReportDenorm{})
	var got []capturedEvent
	rtChecker(st, &got, nil).Check()
	for _, e := range got {
		if e.eventType == EventRestoreTestFailed {
			t.Fatalf("a PASSING restore-test must not emit a failure; got %+v", e)
		}
	}
}

// ── SCENARIO D — a newborn box does not alarm  (the R-81 lesson) ─────────────────────────────
//
// COMPANION RED-PROOF (observed): remove the anchor branch from assessRestoreProven (treat "never
// proven" as MISSED outright) and this fails with "a newborn box must NOT alarm; got
// restore_test_stale ...". That is the fourth instance of no-signal-as-bad-signal, and it would
// have been written by the same hand that just fixed the third.
func TestRestoreTest_NewbornDoesNotAlarm(t *testing.T) {
	st := rtStore(t)
	now := time.Now().UTC()
	// A report with NO restore-test at all, and first contact one hour ago.
	st.SaveHostReport("h1", "c1", []byte(rtReport(t, "", false, now, "")), store.HostReportDenorm{})

	var got []capturedEvent
	var logbuf strings.Builder
	rtChecker(st, &got, &logbuf).Check()

	for _, e := range got {
		if e.eventType == EventRestoreTestStale {
			t.Fatalf("a newborn box must NOT alarm; got %s: %s", e.eventType, e.message)
		}
	}
	// ...but the deferral must be VISIBLE, or quiet is indistinguishable from not-checked.
	if !strings.Contains(logbuf.String(), "not restore-proven yet") {
		t.Fatalf("the deferred verdict must be logged; log:\n%s", logbuf.String())
	}
}

// The boundary, pinned by name so a refactor has to delete an obviously-named contract. R-86 made
// the limit per-tier, so the anchor is now measured against THE TIER'S OWN window — here the local
// tier's, which clamps to the 7-day floor and so keeps this contract numerically identical to the
// one the flat constant expressed.
func TestRestoreTest_Contract_UnprovenIsUnknownUntilTheAnchorElapses(t *testing.T) {
	now := time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC)
	window := restoreProvenWindow("local", 24*time.Hour, true)
	if window != restoreProvenWindowFloor {
		t.Fatalf("precondition: a daily local tier must clamp to the floor; got %s", window)
	}
	cases := []struct {
		name       string
		watched    time.Duration
		wantMissed bool
	}{
		{"newborn, 1h", time.Hour, false},
		{"just inside", window - time.Minute, false},
		{"exactly at the limit", window, false},
		{"just outside", window + time.Minute, true},
		{"long past", 30 * 24 * time.Hour, true},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			got := assessRestoreProven("local", time.Time{}, now.Add(-c.watched), now, window)
			if got.missed() != c.wantMissed {
				t.Fatalf("CONTRACT VIOLATED: unproven for %s (limit %s) → missed=%v, want %v (reason %q)",
					c.watched, window, got.missed(), c.wantMissed, got.reason)
			}
			if !c.wantMissed && got.verdict != verdictUnknown {
				t.Fatalf("a deferred tier must be UNKNOWN (visible), not OK; got verdict=%d", got.verdict)
			}
		})
	}
}

// ── SCENARIO C — an unproven tier becomes visible, and is DISTINCT from a failure ────────────

func TestRestoreTest_StaleIsSeparateFromFailure(t *testing.T) {
	now := time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC)

	// A DAILY tier judged on its own rhythm: the window clamps to the 7-day floor.
	daily := restoreProvenWindow("local", 24*time.Hour, true)
	stale := assessRestoreProven("local", now.Add(-9*24*time.Hour), now.Add(-60*24*time.Hour), now, daily)
	if !stale.missed() {
		t.Fatalf("a daily tier last proven 9 days ago (limit %s) must be stale; got %q", daily, stale.reason)
	}
	// The wording must not read as "broken" — that is the other signal.
	if !strings.Contains(stale.reason, "unverified, not known-broken") {
		t.Fatalf("staleness must say UNVERIFIED, not broken — merging the two is the thing this avoids; got %q", stale.reason)
	}
	fresh := assessRestoreProven("local", now.Add(-2*24*time.Hour), now.Add(-60*24*time.Hour), now, daily)
	if fresh.verdict != verdictOK {
		t.Fatalf("a tier proven 2 days ago is fine; got verdict=%d reason=%q", fresh.verdict, fresh.reason)
	}
}

// The two signals must be DIFFERENT event types — merging them would collapse "your DR is broken"
// into "your DR is unverified", and the second is the one that quietly becomes the first.
func TestRestoreTest_EventTypesAreDistinct(t *testing.T) {
	if EventRestoreTestFailed == EventRestoreTestStale {
		t.Fatal("failure and staleness must be distinct event types")
	}
}

// A tier the box does not HAVE is never reported stale — the Slice-C gate, for the same reason:
// without it, absence-is-not-failure would be re-introduced one level down.
func TestRestoreTest_TierNotPresentIsNeverStale(t *testing.T) {
	var hr hostReportRestoreTests
	if err := json.Unmarshal([]byte(`{"storage_targets":[{"name":"local","type":"local","content":"backup"}]}`), &hr); err != nil {
		t.Fatal(err)
	}
	tiers := expectedRestoreTiers(hr)
	for _, tr := range tiers {
		if tr == "pbs" {
			t.Fatalf("a box with no PBS storage must not expect a pbs tier; got %v", tiers)
		}
	}
	if len(tiers) != 1 || tiers[0] != "local" {
		t.Fatalf("want just the local tier; got %v", tiers)
	}
}

// The window scan recovers per-tier proof even though each report carries only the LATEST run.
func TestRestoreTest_LastProvenPerTierAcrossTheWindow(t *testing.T) {
	now := time.Now().UTC()
	rows := []store.HostReportRow{
		{ReceivedAt: now.Add(-1 * time.Hour), ReportJSON: rtReportRaw("pbs", true, now.Add(-2*time.Hour))},
		{ReceivedAt: now.Add(-3 * time.Hour), ReportJSON: rtReportRaw("local", true, now.Add(-4*time.Hour))},
		{ReceivedAt: now.Add(-5 * time.Hour), ReportJSON: `{{{malformed`},
	}
	got := lastProvenPerTier(rows)
	if _, ok := got["pbs"]; !ok {
		t.Fatalf("the pbs tier's proof must be recovered from the window; got %v", got)
	}
	if _, ok := got["local"]; !ok {
		t.Fatalf("the local tier's proof must be recovered even though a LATER report shows only pbs; got %v", got)
	}
}

func rtReportRaw(tier string, pass bool, at time.Time) string {
	return `{"restore_tests":[{"source_archive":"` + tier + `:x","source_tier":"` + tier +
		`","pass":` + boolStr(pass) + `,"tested_at":"` + at.UTC().Format(time.RFC3339) + `"}]}`
}

func boolStr(b bool) string {
	if b {
		return "true"
	}
	return "false"
}

// ── SCENARIO G — a healthy WEEKLY tier is never reported stale (R-86 Part 2) ─────────────────
//
// This is the test that pins the false alarm this change would otherwise have CREATED. The agent
// now proves a tier once per archive generation, so a weekly offsite tier is proved weekly — in
// perfect health. Against the old flat 7-day window it would sit on the line and alarm every night.
//
// COMPANION RED-PROOF (observed 2026-08-03): pin the window flat, as it was —
//
//   - window := restoreProvenWindow(tier, observed, observedOK)
//   - window := restoreProvenWindowFloor   // the pre-R-86 flat 7 days
//
// →	--- FAIL: TestRestoreTest_HealthyWeeklyTierIsNeverStale
//
//	week 0: a weekly tier proved on its own archive must never be stale (proof age 172h0m0s,
//	window 168h0m0s); verdict=2 reason="pbs tier: last successful restore-test was 172h0m0s ago
//	(limit 168h0m0s, this tier's own backup rhythm) — the tier is unverified, not known-broken"
//
// Restored. The mutation is one line because the whole of Part 2 is one decision: whose rhythm.
//
// NOTE, because it is the finding this test nearly hid: the FIRST version of this fixture had NO
// jitter, and it PASSED under the mutation. A perfectly regular weekly tier's proof age reaches
// EXACTLY 168h just before the next proof, and `age > window` is false by a hair — a hollow test
// that would have shipped Part 1 and its false alarm together. The jitter below is what makes this
// a test, and it is also the truth about the old constant: a healthy weekly tier did not merely sit
// near the line, it sat ON it, so any ordinary delay tipped it over.
func TestRestoreTest_HealthyWeeklyTierIsNeverStale(t *testing.T) {
	start := time.Date(2026, 6, 1, 3, 0, 0, 0, time.UTC)
	firstContact := start.Add(-24 * time.Hour)

	// The observable rhythm of a weekly tier, as the hub would compute it from the reports. No
	// assertion about the window ITSELF here on purpose: that is the mechanism, and it is pinned in
	// TestRestoreProvenWindow_Contract. What this test asserts is the CONSEQUENCE — does the alarm
	// fire? — because R-97b proved a mechanism and shipped a broken consequence anyway.
	weekly := restoreProvenWindow("pbs", 7*24*time.Hour, true)

	// Walk several weeks of a HEALTHY tier, with the jitter a real one has: the backup does not land
	// to the second, and a restore-test can be deferred one evaluation behind a running backup.
	//
	// The jitter is the point. A perfectly regular weekly tier's proof reaches an age of EXACTLY one
	// interval (168h) just before the next proof, and against a flat 168h window `age > window` is
	// false by a hair — so a regular fixture would pass against the very constant this task must
	// change, and prove nothing. That is the brief's "sits exactly on that line": every real-world
	// delay pushes it over, and the alarm is about a system that is working.
	settle, evalLatency := 24*time.Hour, 6*time.Hour
	archiveLate := []time.Duration{0, 4 * time.Hour, 2 * time.Hour, 6 * time.Hour, 0, 3 * time.Hour}
	deferred := []time.Duration{0, 0, 6 * time.Hour, 0, 0, 6 * time.Hour} // one evaluation behind a backup

	archiveAt := func(week int) time.Time {
		return start.AddDate(0, 0, 7*week).Add(archiveLate[week])
	}
	provenAt := func(week int) time.Time {
		return archiveAt(week).Add(settle + evalLatency + deferred[week])
	}

	var worst time.Duration
	for week := 0; week+1 < len(archiveLate); week++ {
		// The widest the proof's age ever gets: the instant before the NEXT week's proof lands.
		now := provenAt(week + 1).Add(-time.Second)
		age := now.Sub(provenAt(week))
		if age > worst {
			worst = age
		}
		v := assessRestoreProven("pbs", provenAt(week), firstContact, now, weekly)
		if v.verdict != verdictOK {
			t.Fatalf("week %d: a weekly tier proved on its own archive must never be stale (proof age %s, window %s); verdict=%d reason=%q",
				week, age.Round(time.Hour), weekly, v.verdict, v.reason)
		}
	}
	// The fixture must actually EXERCISE the boundary — a jitter-free walk would sit at exactly one
	// interval and pass against a flat 7-day window, which is the hollow version of this test.
	if worst <= restoreProvenWindowFloor {
		t.Fatalf("this fixture never exceeds the old flat window (worst proof age %s) — it cannot detect the defect it exists for", worst)
	}
}

// ...and a weekly tier that genuinely STOPS being proved must still alarm. A window that never
// fires is not a fix, it is a deletion.
func TestRestoreTest_WeeklyTierThatStopsBeingProvedStillAlarms(t *testing.T) {
	now := time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC)
	weekly := restoreProvenWindow("pbs", 7*24*time.Hour, true)

	v := assessRestoreProven("pbs", now.Add(-weekly-time.Hour), now.Add(-90*24*time.Hour), now, weekly)
	if !v.missed() {
		t.Fatalf("a weekly tier unproven for longer than its own window MUST alarm; got verdict=%d reason=%q", v.verdict, v.reason)
	}
	if !strings.Contains(v.reason, weekly.String()) {
		t.Fatalf("the alarm must state the window it was judged against (R-100's corollary); got %q", v.reason)
	}
}

// The window's own contract: derived from the tier's rhythm, floored, capped, and never dependent
// on an unobservable history for the tier that would suffer most from a wrong answer.
func TestRestoreProvenWindow_Contract(t *testing.T) {
	cases := []struct {
		name       string
		tier       string
		observed   time.Duration
		observedOK bool
		want       time.Duration
	}{
		{"daily local clamps to the floor", "local", 24 * time.Hour, true, restoreProvenWindowFloor},
		{"weekly pbs widens", "pbs", 7 * 24 * time.Hour, true, restoreProvenWindowCap},
		{"3-day tier sits between", "pbs", 72 * time.Hour, true, 12 * 24 * time.Hour},
		{"unobservable local falls back to its declared rhythm", "local", 0, false, restoreProvenWindowFloor},
		{"unobservable pbs falls back WIDE, not to the floor", "pbs", 0, false, restoreProvenWindowCap},
		{"a nonsense zero interval is ignored", "pbs", 0, true, restoreProvenWindowCap},
		// MEASURED ON THE LIVE BOX, and the reason observation may only WIDEN. demo-felhom's two
		// retained PBS snapshots sit 8h54m apart (one is a healing artefact), so a mean-gap estimate
		// reads a WEEKLY tier as nine-hourly. Taking that at face value gives 4x9h = 36h → the 7-day
		// floor → and a weekly tier proved weekly (~8.25d of proof age) alarms within a week of this
		// shipping, on the box it shipped to.
		{"a short observed gap must NOT tighten a weekly tier", "pbs", 8*time.Hour + 54*time.Minute, true, restoreProvenWindowCap},
		{"a short observed gap must not tighten the host tier either", "local", 30 * time.Minute, true, restoreProvenWindowFloor},
		// ...but a tier that genuinely runs SLOWER than its declared rhythm still widens.
		{"a genuinely slower tier widens", "local", 4 * 24 * time.Hour, true, restoreProvenWindowCap},
	}
	// The relationship Part 1 depends on: a weekly tier's window must be WIDER than a daily tier's,
	// or proving weekly (which is now correct behaviour) alarms on itself.
	if restoreProvenWindow("pbs", 7*24*time.Hour, true) <= restoreProvenWindow("local", 24*time.Hour, true) {
		t.Fatal("a weekly tier must earn a wider window than a daily one — otherwise R-86's agent half alarms about itself")
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			got := restoreProvenWindow(c.tier, c.observed, c.observedOK)
			if got != c.want {
				t.Fatalf("window(%s, observed=%s ok=%v) = %s, want %s", c.tier, c.observed, c.observedOK, got, c.want)
			}
			if got < restoreProvenWindowFloor || got > restoreProvenWindowCap {
				t.Fatalf("every window must stay inside [%s, %s]; got %s", restoreProvenWindowFloor, restoreProvenWindowCap, got)
			}
		})
	}
}

// The rhythm must be OBSERVED from the reports, not assumed — including the slice-A.4 rule that a
// PBS-targeted vzdump appears in both arrays and must be attributed by TARGET TYPE.
func TestObservedArchiveIntervals_FromReports(t *testing.T) {
	base := time.Date(2026, 7, 1, 2, 0, 0, 0, time.UTC)
	mk := func(localAt []time.Time, pbsAt []time.Time) string {
		type stg struct{ Name, Type, Content string }
		type bk struct {
			TargetID  string `json:"target_id"`
			Success   bool   `json:"success"`
			StartedAt string `json:"started_at"`
		}
		type snap struct {
			BackupTime string `json:"backup_time"`
		}
		payload := struct {
			StorageTargets []struct {
				Name    string `json:"name"`
				Type    string `json:"type"`
				Content string `json:"content"`
			} `json:"storage_targets"`
			Backups      []bk   `json:"backups"`
			PBSSnapshots []snap `json:"pbs_snapshots"`
		}{}
		payload.StorageTargets = append(payload.StorageTargets, struct {
			Name    string `json:"name"`
			Type    string `json:"type"`
			Content string `json:"content"`
		}{"felhom-backup", "dir", "backup"}, struct {
			Name    string `json:"name"`
			Type    string `json:"type"`
			Content string `json:"content"`
		}{"felhom-pbs", "pbs", "backup"})
		for _, at := range localAt {
			payload.Backups = append(payload.Backups, bk{"felhom-backup", true, at.Format(time.RFC3339)})
		}
		for _, at := range pbsAt {
			// The SAME archive appears as a vzdump record AND as a snapshot — slice A.4.
			payload.Backups = append(payload.Backups, bk{"felhom-pbs", true, at.Format(time.RFC3339)})
			payload.PBSSnapshots = append(payload.PBSSnapshots, snap{at.Format(time.RFC3339)})
		}
		b, err := json.Marshal(payload)
		if err != nil {
			t.Fatal(err)
		}
		return string(b)
	}

	rows := []store.HostReportRow{
		{ReportJSON: mk(
			[]time.Time{base, base.AddDate(0, 0, 1), base.AddDate(0, 0, 2)},
			[]time.Time{base, base.AddDate(0, 0, 7)},
		)},
		{ReportJSON: `{{{malformed`}, // must not blind the scan
	}

	got := observedArchiveIntervals(rows)
	if d, ok := got["local"]; !ok || d != 24*time.Hour {
		t.Fatalf("a daily host tier must be observed as ~24h; got %s ok=%v", d, ok)
	}
	if d, ok := got["pbs"]; !ok || d != 7*24*time.Hour {
		t.Fatalf("a weekly offsite tier must be observed as ~7d — and its vzdump record must NOT be "+
			"counted into the host tier (slice A.4); got %s ok=%v", d, ok)
	}

	// One generation is not a rhythm: unobservable, so the caller falls back to the declared one.
	single := []store.HostReportRow{{ReportJSON: mk(nil, []time.Time{base})}}
	if d, ok := observedArchiveIntervals(single)["pbs"]; ok {
		t.Fatalf("one archive cannot yield an interval; got %s", d)
	}
}