package localapi import ( "context" "fmt" "io" "log/slog" "net/http" "path/filepath" "sync" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/backup" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" ) // R-973 — the whole-guest backup cadence cap. A compromised controller calling POST /backup in a loop // pushed out every pre-compromise restore point: each success prunes the tier. These tests model the tier // the way Proxmox and ep0 keep it (keep-last N, pruned after each success) and ask what is LEFT. // keepLastTier is a whole-guest tier: every successful run adds an archive stamped with the clock and // prunes to keepLast, as vzdump's --prune-backups keep-last (local) and ep0's keep-last 2 (off-site) do. type keepLastTier struct { mu sync.Mutex clock *testClock keepLast int archives []time.Time fail bool runs int } func (k *keepLastTier) BackupWithSnapshotHook(_ context.Context, vmid int, _ func()) (hub.Backup, error) { k.mu.Lock() defer k.mu.Unlock() k.runs++ now := k.clock.Now() if k.fail { return hub.Backup{VMID: vmid, StartedAt: now.Format(time.RFC3339)}, fmt.Errorf("vzdump failed") } k.archives = append(k.archives, now) if len(k.archives) > k.keepLast { k.archives = k.archives[len(k.archives)-k.keepLast:] } return hub.Backup{VMID: vmid, Success: true, Archive: "x", StartedAt: now.Format(time.RFC3339)}, nil } // oldestBefore reports whether the tier still holds an archive made before t (a pre-compromise point). func (k *keepLastTier) oldestBefore(t time.Time) bool { k.mu.Lock() defer k.mu.Unlock() return len(k.archives) > 0 && k.archives[0].Before(t) } func (k *keepLastTier) count() (runs, kept int) { k.mu.Lock() defer k.mu.Unlock() return k.runs, len(k.archives) } type testClock struct { mu sync.Mutex t time.Time } func (c *testClock) Now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t } func (c *testClock) Advance(d time.Duration) { c.mu.Lock(); c.t = c.t.Add(d); c.mu.Unlock() } // capServer builds the two-tier box of `07` §6: local 24 h keep-last 3, off-site 7 d keep-last 2, the // success history on disk at path (a second capServer on the same path is an agent restart). func capServer(t *testing.T, path string, clock *testClock, local, offsite *keepLastTier) *Server { t.Helper() srv, err := NewServer(Options{ ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{}, Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local"}, {Name: "felhom-pbs"}}}, Tokens: staticTokens{"A": 8200}, BackupTiers: []BackupTier{ {TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: local}, {TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: offsite}, }, LastKnownBackups: backup.NewBackupSuccessState(path), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) if err != nil { t.Fatal(err) } srv.baseCtx = context.Background() srv.now = clock.Now return srv } // runBackup POSTs one backup and waits for the job to end. Returns the HTTP status of the POST. func runBackup(t *testing.T, srv *Server, target string) int { t.Helper() rr := do(t, srv.Handler(), "POST", "/backup?target="+target, "A", "") if rr.Code == http.StatusAccepted { waitFor(t, func() bool { srv.jobsMu.Lock() defer srv.jobsMu.Unlock() j := srv.jobs[backupJobKey{vmid: 8200, target: target}] return j != nil && !backupInFlight(j.Phase) }) } return rr.Code } // The off-site tier — the one `07` §8 row 10 counts on against ransomware. Before the cap ten calls in a // row left ep0 holding only the attacker's two copies. Now ONE runs (the tier was due), the other nine // are refused, and the newest pre-compromise copy survives; five days of flooding later it still does. // // COMPANION RED-PROOF (observed 2026-10-18): make capRefuses always return false → this fails with // "the off-site tier lost every pre-compromise copy: 10 runs" . Restored. func TestBackupCap_OffsiteFloodKeepsAPreCompromiseCopy(t *testing.T) { clock := &testClock{t: testNow} local := &keepLastTier{clock: clock, keepLast: 3} off := &keepLastTier{clock: clock, keepLast: 2} path := filepath.Join(t.TempDir(), "backup-success-state.json") srv := capServer(t, path, clock, local, off) // The honest history: two weekly copies, the newest 7 days and 1 hour old (so the tier is due). for _, age := range []time.Duration{14*24*time.Hour + time.Hour, 7*24*time.Hour + time.Hour} { clock.t = testNow.Add(-age) if code := runBackup(t, srv, "felhom-pbs"); code != http.StatusAccepted { t.Fatalf("seeding the honest weekly copy: %d", code) } } clock.t = testNow compromise := testNow refused := 0 for i := 0; i < 10; i++ { if runBackup(t, srv, "felhom-pbs") == http.StatusTooManyRequests { refused++ } clock.Advance(time.Minute) } runs, _ := off.count() if !off.oldestBefore(compromise) { t.Fatalf("the off-site tier lost every pre-compromise copy: %d runs", runs-2) } if runs-2 != 1 || refused != 9 { t.Fatalf("flood: %d runs, %d refused — want 1 run (the tier was due) and 9 refused", runs-2, refused) } for d := 0; d < 5; d++ { // five more days of flooding, every hour for h := 0; h < 24; h++ { clock.Advance(time.Hour) if code := runBackup(t, srv, "felhom-pbs"); code != http.StatusTooManyRequests { t.Fatalf("day %d hour %d: an off-site run was allowed %v after the last one", d, h, clock.Now().Sub(compromise)) } } } if !off.oldestBefore(compromise) { t.Fatal("five days of flooding pushed out the pre-compromise off-site copy") } } // The local tier: a flood runs exactly 3 in a day (the schedule's own day: two nights + one press) and // every further request is refused. Honest about the limit: with keep-last 3 those 3 replace the 3 kept // points — the cap slows a flush from minutes to a day; it cannot keep a point a 3-per-day pace prunes. func TestBackupCap_LocalFloodRunsAtMostThreeADay(t *testing.T) { clock := &testClock{t: testNow} local := &keepLastTier{clock: clock, keepLast: 3} srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2}) refused := 0 for i := 0; i < 20; i++ { if runBackup(t, srv, "local") == http.StatusTooManyRequests { refused++ } clock.Advance(10 * time.Minute) } if runs, _ := local.count(); runs != 3 || refused != 17 { t.Fatalf("local flood over ~3 h: %d runs, %d refused — want 3 and 17", runs, refused) } clock.Advance(24 * time.Hour) // a day later the window has moved on if code := runBackup(t, srv, "local"); code != http.StatusAccepted { t.Fatalf("a run a day later must be allowed: %d", code) } } // The schedule itself is never refused: a night at W+6h, a press at 10:00, the next night at W+2h (all // three inside one rolling 24 h) run; a second press that day is refused; nights keep running after it. func TestBackupCap_TheScheduledDayIsNeverRefused(t *testing.T) { day := time.Date(2026, 6, 10, 0, 0, 0, 0, time.UTC) clock := &testClock{t: day.Add(8*time.Hour + 29*time.Minute)} // W=02:30 → W+6h ≈ 08:29 local := &keepLastTier{clock: clock, keepLast: 3} srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2}) steps := []struct { at time.Time want int what string }{ {day.Add(8*time.Hour + 29*time.Minute), http.StatusAccepted, "night 1 at W+6h"}, {day.Add(10 * time.Hour), http.StatusAccepted, "a press at 10:00"}, {day.Add(24*time.Hour + 4*time.Hour + 30*time.Minute), http.StatusAccepted, "night 2 at W+2h (20 h later)"}, {day.Add(24*time.Hour + 7*time.Hour), http.StatusTooManyRequests, "a second press inside the same 24 h"}, {day.Add(48*time.Hour + 4*time.Hour + 30*time.Minute), http.StatusAccepted, "night 3"}, } for _, s := range steps { clock.t = s.at if got := runBackup(t, srv, "local"); got != s.want { t.Fatalf("%s: got %d, want %d", s.what, got, s.want) } } } // A restart must not reset the cap: the history is read back from disk. // // COMPANION RED-PROOF (observed 2026-10-18): make recentCapRuns ignore s.lastKnown → this fails with // "after a restart the 4th run in a day was allowed (got 202)". Restored. func TestBackupCap_SurvivesARestart(t *testing.T) { clock := &testClock{t: testNow} path := filepath.Join(t.TempDir(), "s.json") local := &keepLastTier{clock: clock, keepLast: 3} srv := capServer(t, path, clock, local, &keepLastTier{clock: clock, keepLast: 2}) for i := 0; i < 3; i++ { if code := runBackup(t, srv, "local"); code != http.StatusAccepted { t.Fatalf("run %d: %d", i+1, code) } clock.Advance(time.Minute) } srv2 := capServer(t, path, clock, local, &keepLastTier{clock: clock, keepLast: 2}) // the restart if code := runBackup(t, srv2, "local"); code != http.StatusTooManyRequests { t.Fatalf("after a restart the 4th run in a day was allowed (got %d)", code) } } // Failed runs prune nothing and must stay retryable: they do not count. func TestBackupCap_FailuresDoNotCount(t *testing.T) { clock := &testClock{t: testNow} local := &keepLastTier{clock: clock, keepLast: 3, fail: true} srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2}) for i := 0; i < 6; i++ { if code := runBackup(t, srv, "local"); code != http.StatusAccepted { t.Fatalf("failed attempt %d was refused: %d", i+1, code) } clock.Advance(15 * time.Minute) } local.mu.Lock() local.fail = false local.mu.Unlock() if code := runBackup(t, srv, "local"); code != http.StatusAccepted { t.Fatalf("a success after six failures was refused: %d", code) } } // A refusal raises the operator signal for 24 h, and the signal survives a restart. func TestBackupCap_RefusalRaisesTheOperatorSignal(t *testing.T) { clock := &testClock{t: testNow} path := filepath.Join(t.TempDir(), "s.json") off := &keepLastTier{clock: clock, keepLast: 2} srv := capServer(t, path, clock, &keepLastTier{clock: clock, keepLast: 3}, off) if _, _, _, ok := srv.BackupCapRefusal(); ok { t.Fatal("a fresh box must not raise the signal") } runBackup(t, srv, "felhom-pbs") clock.Advance(time.Hour) if code := runBackup(t, srv, "felhom-pbs"); code != http.StatusTooManyRequests { t.Fatalf("second off-site run an hour later: %d, want 429", code) } target, vmid, at, ok := srv.BackupCapRefusal() if !ok || target != "felhom-pbs" || vmid != 8200 || !at.Equal(clock.Now()) { t.Fatalf("signal after a refusal: ok=%v target=%q vmid=%d at=%v", ok, target, vmid, at) } clock.Advance(23 * time.Hour) srv2 := capServer(t, path, clock, &keepLastTier{clock: clock, keepLast: 3}, off) // a restart if _, _, _, ok := srv2.BackupCapRefusal(); !ok { t.Fatal("the signal did not survive a restart") } clock.Advance(2 * time.Hour) if _, _, _, ok := srv2.BackupCapRefusal(); ok { t.Fatal("the signal must clear 24 h after the last refusal") } } func TestBackupCadenceCap(t *testing.T) { cases := []struct { cadence time.Duration window time.Duration max int }{ {0, 24 * time.Hour, 3}, {24 * time.Hour, 24 * time.Hour, 3}, {12 * time.Hour, 24 * time.Hour, 3}, {7 * 24 * time.Hour, 6 * 24 * time.Hour, 1}, {36 * time.Hour, 18 * time.Hour, 1}, {30 * 24 * time.Hour, 29 * 24 * time.Hour, 1}, } for _, c := range cases { if w, m := backupCadenceCap(c.cadence); w != c.window || m != c.max { t.Errorf("cadence %v: got %v/%d, want %v/%d", c.cadence, w, m, c.window, c.max) } } }