felhom-agent/internal/localapi/backup_cap_test.go

package localapi

import (
	"context"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"path/filepath"
	"sync"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)

// R-973 — the whole-guest backup cadence cap. A compromised controller calling POST /backup in a loop
// pushed out every pre-compromise restore point: each success prunes the tier. These tests model the tier
// the way Proxmox and ep0 keep it (keep-last N, pruned after each success) and ask what is LEFT.

// keepLastTier is a whole-guest tier: every successful run adds an archive stamped with the clock and
// prunes to keepLast, as vzdump's --prune-backups keep-last (local) and ep0's keep-last 2 (off-site) do.
type keepLastTier struct {
	mu       sync.Mutex
	clock    *testClock
	keepLast int
	archives []time.Time
	fail     bool
	runs     int
}

func (k *keepLastTier) BackupWithSnapshotHook(_ context.Context, vmid int, _ func()) (hub.Backup, error) {
	k.mu.Lock()
	defer k.mu.Unlock()
	k.runs++
	now := k.clock.Now()
	if k.fail {
		return hub.Backup{VMID: vmid, StartedAt: now.Format(time.RFC3339)}, fmt.Errorf("vzdump failed")
	}
	k.archives = append(k.archives, now)
	if len(k.archives) > k.keepLast {
		k.archives = k.archives[len(k.archives)-k.keepLast:]
	}
	return hub.Backup{VMID: vmid, Success: true, Archive: "x", StartedAt: now.Format(time.RFC3339)}, nil
}

// oldestBefore reports whether the tier still holds an archive made before t (a pre-compromise point).
func (k *keepLastTier) oldestBefore(t time.Time) bool {
	k.mu.Lock()
	defer k.mu.Unlock()
	return len(k.archives) > 0 && k.archives[0].Before(t)
}

func (k *keepLastTier) count() (runs, kept int) {
	k.mu.Lock()
	defer k.mu.Unlock()
	return k.runs, len(k.archives)
}

type testClock struct {
	mu sync.Mutex
	t  time.Time
}

func (c *testClock) Now() time.Time          { c.mu.Lock(); defer c.mu.Unlock(); return c.t }
func (c *testClock) Advance(d time.Duration) { c.mu.Lock(); c.t = c.t.Add(d); c.mu.Unlock() }

// capServer builds the two-tier box of `07` §6: local 24 h keep-last 3, off-site 7 d keep-last 2, the
// success history on disk at path (a second capServer on the same path is an agent restart).
func capServer(t *testing.T, path string, clock *testClock, local, offsite *keepLastTier) *Server {
	t.Helper()
	srv, err := NewServer(Options{
		ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
		Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local"}, {Name: "felhom-pbs"}}},
		Tokens:  staticTokens{"A": 8200},
		BackupTiers: []BackupTier{
			{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: local},
			{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: offsite},
		},
		LastKnownBackups: backup.NewBackupSuccessState(path),
		Logger:           slog.New(slog.NewTextHandler(io.Discard, nil)),
	})
	if err != nil {
		t.Fatal(err)
	}
	srv.baseCtx = context.Background()
	srv.now = clock.Now
	return srv
}

// runBackup POSTs one backup and waits for the job to end. Returns the HTTP status of the POST.
func runBackup(t *testing.T, srv *Server, target string) int {
	t.Helper()
	rr := do(t, srv.Handler(), "POST", "/backup?target="+target, "A", "")
	if rr.Code == http.StatusAccepted {
		waitFor(t, func() bool {
			srv.jobsMu.Lock()
			defer srv.jobsMu.Unlock()
			j := srv.jobs[backupJobKey{vmid: 8200, target: target}]
			return j != nil && !backupInFlight(j.Phase)
		})
	}
	return rr.Code
}

// The off-site tier — the one `07` §8 row 10 counts on against ransomware. Before the cap ten calls in a
// row left ep0 holding only the attacker's two copies. Now ONE runs (the tier was due), the other nine
// are refused, and the newest pre-compromise copy survives; five days of flooding later it still does.
//
// COMPANION RED-PROOF (observed 2026-10-18): make capRefuses always return false → this fails with
// "the off-site tier lost every pre-compromise copy: 10 runs" . Restored.
func TestBackupCap_OffsiteFloodKeepsAPreCompromiseCopy(t *testing.T) {
	clock := &testClock{t: testNow}
	local := &keepLastTier{clock: clock, keepLast: 3}
	off := &keepLastTier{clock: clock, keepLast: 2}
	path := filepath.Join(t.TempDir(), "backup-success-state.json")
	srv := capServer(t, path, clock, local, off)

	// The honest history: two weekly copies, the newest 7 days and 1 hour old (so the tier is due).
	for _, age := range []time.Duration{14*24*time.Hour + time.Hour, 7*24*time.Hour + time.Hour} {
		clock.t = testNow.Add(-age)
		if code := runBackup(t, srv, "felhom-pbs"); code != http.StatusAccepted {
			t.Fatalf("seeding the honest weekly copy: %d", code)
		}
	}
	clock.t = testNow
	compromise := testNow

	refused := 0
	for i := 0; i < 10; i++ {
		if runBackup(t, srv, "felhom-pbs") == http.StatusTooManyRequests {
			refused++
		}
		clock.Advance(time.Minute)
	}
	runs, _ := off.count()
	if !off.oldestBefore(compromise) {
		t.Fatalf("the off-site tier lost every pre-compromise copy: %d runs", runs-2)
	}
	if runs-2 != 1 || refused != 9 {
		t.Fatalf("flood: %d runs, %d refused — want 1 run (the tier was due) and 9 refused", runs-2, refused)
	}
	for d := 0; d < 5; d++ { // five more days of flooding, every hour
		for h := 0; h < 24; h++ {
			clock.Advance(time.Hour)
			if code := runBackup(t, srv, "felhom-pbs"); code != http.StatusTooManyRequests {
				t.Fatalf("day %d hour %d: an off-site run was allowed %v after the last one", d, h, clock.Now().Sub(compromise))
			}
		}
	}
	if !off.oldestBefore(compromise) {
		t.Fatal("five days of flooding pushed out the pre-compromise off-site copy")
	}
}

// The local tier: a flood runs exactly 3 in a day (the schedule's own day: two nights + one press) and
// every further request is refused. Honest about the limit: with keep-last 3 those 3 replace the 3 kept
// points — the cap slows a flush from minutes to a day; it cannot keep a point a 3-per-day pace prunes.
func TestBackupCap_LocalFloodRunsAtMostThreeADay(t *testing.T) {
	clock := &testClock{t: testNow}
	local := &keepLastTier{clock: clock, keepLast: 3}
	srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2})
	refused := 0
	for i := 0; i < 20; i++ {
		if runBackup(t, srv, "local") == http.StatusTooManyRequests {
			refused++
		}
		clock.Advance(10 * time.Minute)
	}
	if runs, _ := local.count(); runs != 3 || refused != 17 {
		t.Fatalf("local flood over ~3 h: %d runs, %d refused — want 3 and 17", runs, refused)
	}
	clock.Advance(24 * time.Hour) // a day later the window has moved on
	if code := runBackup(t, srv, "local"); code != http.StatusAccepted {
		t.Fatalf("a run a day later must be allowed: %d", code)
	}
}

// The schedule itself is never refused: a night at W+6h, a press at 10:00, the next night at W+2h (all
// three inside one rolling 24 h) run; a second press that day is refused; nights keep running after it.
func TestBackupCap_TheScheduledDayIsNeverRefused(t *testing.T) {
	day := time.Date(2026, 6, 10, 0, 0, 0, 0, time.UTC)
	clock := &testClock{t: day.Add(8*time.Hour + 29*time.Minute)} // W=02:30 → W+6h ≈ 08:29
	local := &keepLastTier{clock: clock, keepLast: 3}
	srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2})
	steps := []struct {
		at   time.Time
		want int
		what string
	}{
		{day.Add(8*time.Hour + 29*time.Minute), http.StatusAccepted, "night 1 at W+6h"},
		{day.Add(10 * time.Hour), http.StatusAccepted, "a press at 10:00"},
		{day.Add(24*time.Hour + 4*time.Hour + 30*time.Minute), http.StatusAccepted, "night 2 at W+2h (20 h later)"},
		{day.Add(24*time.Hour + 7*time.Hour), http.StatusTooManyRequests, "a second press inside the same 24 h"},
		{day.Add(48*time.Hour + 4*time.Hour + 30*time.Minute), http.StatusAccepted, "night 3"},
	}
	for _, s := range steps {
		clock.t = s.at
		if got := runBackup(t, srv, "local"); got != s.want {
			t.Fatalf("%s: got %d, want %d", s.what, got, s.want)
		}
	}
}

// A restart must not reset the cap: the history is read back from disk.
//
// COMPANION RED-PROOF (observed 2026-10-18): make recentCapRuns ignore s.lastKnown → this fails with
// "after a restart the 4th run in a day was allowed (got 202)". Restored.
func TestBackupCap_SurvivesARestart(t *testing.T) {
	clock := &testClock{t: testNow}
	path := filepath.Join(t.TempDir(), "s.json")
	local := &keepLastTier{clock: clock, keepLast: 3}
	srv := capServer(t, path, clock, local, &keepLastTier{clock: clock, keepLast: 2})
	for i := 0; i < 3; i++ {
		if code := runBackup(t, srv, "local"); code != http.StatusAccepted {
			t.Fatalf("run %d: %d", i+1, code)
		}
		clock.Advance(time.Minute)
	}
	srv2 := capServer(t, path, clock, local, &keepLastTier{clock: clock, keepLast: 2}) // the restart
	if code := runBackup(t, srv2, "local"); code != http.StatusTooManyRequests {
		t.Fatalf("after a restart the 4th run in a day was allowed (got %d)", code)
	}
}

// Failed runs prune nothing and must stay retryable: they do not count.
func TestBackupCap_FailuresDoNotCount(t *testing.T) {
	clock := &testClock{t: testNow}
	local := &keepLastTier{clock: clock, keepLast: 3, fail: true}
	srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2})
	for i := 0; i < 6; i++ {
		if code := runBackup(t, srv, "local"); code != http.StatusAccepted {
			t.Fatalf("failed attempt %d was refused: %d", i+1, code)
		}
		clock.Advance(15 * time.Minute)
	}
	local.mu.Lock()
	local.fail = false
	local.mu.Unlock()
	if code := runBackup(t, srv, "local"); code != http.StatusAccepted {
		t.Fatalf("a success after six failures was refused: %d", code)
	}
}

// A refusal raises the operator signal for 24 h, and the signal survives a restart.
func TestBackupCap_RefusalRaisesTheOperatorSignal(t *testing.T) {
	clock := &testClock{t: testNow}
	path := filepath.Join(t.TempDir(), "s.json")
	off := &keepLastTier{clock: clock, keepLast: 2}
	srv := capServer(t, path, clock, &keepLastTier{clock: clock, keepLast: 3}, off)
	if _, _, _, ok := srv.BackupCapRefusal(); ok {
		t.Fatal("a fresh box must not raise the signal")
	}
	runBackup(t, srv, "felhom-pbs")
	clock.Advance(time.Hour)
	if code := runBackup(t, srv, "felhom-pbs"); code != http.StatusTooManyRequests {
		t.Fatalf("second off-site run an hour later: %d, want 429", code)
	}
	target, vmid, at, ok := srv.BackupCapRefusal()
	if !ok || target != "felhom-pbs" || vmid != 8200 || !at.Equal(clock.Now()) {
		t.Fatalf("signal after a refusal: ok=%v target=%q vmid=%d at=%v", ok, target, vmid, at)
	}
	clock.Advance(23 * time.Hour)
	srv2 := capServer(t, path, clock, &keepLastTier{clock: clock, keepLast: 3}, off) // a restart
	if _, _, _, ok := srv2.BackupCapRefusal(); !ok {
		t.Fatal("the signal did not survive a restart")
	}
	clock.Advance(2 * time.Hour)
	if _, _, _, ok := srv2.BackupCapRefusal(); ok {
		t.Fatal("the signal must clear 24 h after the last refusal")
	}
}

func TestBackupCadenceCap(t *testing.T) {
	cases := []struct {
		cadence time.Duration
		window  time.Duration
		max     int
	}{
		{0, 24 * time.Hour, 3},
		{24 * time.Hour, 24 * time.Hour, 3},
		{12 * time.Hour, 24 * time.Hour, 3},
		{7 * 24 * time.Hour, 6 * 24 * time.Hour, 1},
		{36 * time.Hour, 18 * time.Hour, 1},
		{30 * 24 * time.Hour, 29 * 24 * time.Hour, 1},
	}
	for _, c := range cases {
		if w, m := backupCadenceCap(c.cadence); w != c.window || m != c.max {
			t.Errorf("cadence %v: got %v/%d, want %v/%d", c.cadence, w, m, c.window, c.max)
		}
	}
}