Files
admin 3844df7c23 controllerswap: F1 verify hardening — reject crash-looping no-healthcheck image v0.47.0
controllerHealthy reads RestartCount (running&&rc>0 -> not ok) + signals needsDwell for no-healthcheck;
verify requires verifyDwell(=3) consecutive ok polls for a no-healthcheck image (real healthcheck
trusted immediately). Closes the F1 hole (alpine crash-loop passed the point-in-time check). Red-proof
+ dwell + real-image tests. No sudoers/orchestration change.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pg8ANF97SEeKYSN5Jxw3qJ
2026-06-29 22:45:41 +02:00

517 lines
17 KiB
Go

package localapi
import (
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)
// ---- fakes ------------------------------------------------------------------------------
// fakeGuests records every VMID it was called with, so a test can assert an op was NOT issued
// for a guest the caller is not scoped to.
type fakeGuests struct {
mu sync.Mutex
snapVMIDs []int
rbVMIDs []int
cfgVMIDs []int
mounts map[string]string // mpN -> value, returned by GuestConfig
failSnap bool
}
func (f *fakeGuests) GuestConfig(_ context.Context, vmid int) (proxmox.GuestConfig, error) {
f.mu.Lock()
f.cfgVMIDs = append(f.cfgVMIDs, vmid)
f.mu.Unlock()
extra := map[string]json.RawMessage{}
for k, v := range f.mounts {
b, _ := json.Marshal(v)
extra[k] = b
}
return proxmox.GuestConfig{Extra: extra}, nil
}
func (f *fakeGuests) Snapshot(_ context.Context, vmid int, _, _ string) (string, error) {
f.mu.Lock()
f.snapVMIDs = append(f.snapVMIDs, vmid)
f.mu.Unlock()
if f.failSnap {
return "", fmt.Errorf("boom")
}
return "UPID:snap", nil
}
func (f *fakeGuests) Rollback(_ context.Context, vmid int, _ string) (string, error) {
f.mu.Lock()
f.rbVMIDs = append(f.rbVMIDs, vmid)
f.mu.Unlock()
return "UPID:rb", nil
}
func (f *fakeGuests) WaitTask(_ context.Context, _ string, _ proxmox.WaitOptions) (proxmox.TaskStatus, error) {
return proxmox.TaskStatus{ExitStatus: "OK"}, nil
}
type fakeBackups struct {
mu sync.Mutex
vmids []int
gate chan struct{} // if non-nil, the backup blocks until it is closed (observe phases)
failErr string // if set, the backup returns this error (drives the failed phase)
fireSnapshot bool // if true, invoke onSnapshot (simulates snapshot-mode detection, 8B.2)
}
func (f *fakeBackups) BackupWithSnapshotHook(_ context.Context, vmid int, onSnapshot func()) (hub.Backup, error) {
f.mu.Lock()
f.vmids = append(f.vmids, vmid)
gate := f.gate
failErr := f.failErr
fire := f.fireSnapshot
f.mu.Unlock()
if fire && onSnapshot != nil {
onSnapshot() // snapshot taken (mid-backup) → phase snapshotted
}
if gate != nil {
<-gate
}
if failErr != "" {
return hub.Backup{VMID: vmid}, fmt.Errorf("%s", failErr)
}
return hub.Backup{VMID: vmid, Success: true, Archive: "local:backup/vzdump-x", StartedAt: "2026-06-10T00:00:00Z"}, nil
}
func (f *fakeBackups) called() []int {
f.mu.Lock()
defer f.mu.Unlock()
return append([]int(nil), f.vmids...)
}
type fakeStore struct {
mu sync.Mutex
backups []hub.Backup
tests []hub.RestoreTest
}
func (s *fakeStore) RecordBackup(b hub.Backup) {
s.mu.Lock()
s.backups = append(s.backups, b)
s.mu.Unlock()
}
func (s *fakeStore) Backups(context.Context) []hub.Backup {
s.mu.Lock()
defer s.mu.Unlock()
return append([]hub.Backup(nil), s.backups...)
}
func (s *fakeStore) RestoreTests(context.Context) []hub.RestoreTest {
s.mu.Lock()
defer s.mu.Unlock()
return append([]hub.RestoreTest(nil), s.tests...)
}
type fakeStorage struct{ targets []hub.StorageTarget }
func (f fakeStorage) Observe(context.Context) ([]hub.StorageTarget, error) { return f.targets, nil }
// staticTokens is a fixed token→guest map for server tests (the durable store is tested
// separately). Token "A" → guest 8200, token "B" → guest 9300.
type staticTokens map[string]int
func (m staticTokens) Lookup(tok string) (int, bool) { v, ok := m[tok]; return v, ok }
// ---- harness ----------------------------------------------------------------------------
// testNow is the fixed clock the test server uses, so /backup/due cadence math is deterministic.
var testNow = time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC)
func newTestServerS(t *testing.T, g *fakeGuests, b *fakeBackups, st *fakeStore, sv StorageView) *Server {
t.Helper()
if sv == nil {
sv = fakeStorage{}
}
srv, err := NewServer(Options{
ListenAddr: "127.0.0.1:0",
Guests: g,
Backups: b,
Store: st,
Storage: sv,
Tokens: staticTokens{"A": 8200, "B": 9300},
BackupCadence: 24 * time.Hour,
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
})
if err != nil {
t.Fatalf("new server: %v", err)
}
srv.baseCtx = context.Background()
srv.now = func() time.Time { return testNow }
return srv
}
func newTestServer(t *testing.T, g *fakeGuests, b *fakeBackups, st *fakeStore, sv StorageView) http.Handler {
t.Helper()
return newTestServerS(t, g, b, st, sv).Handler()
}
func do(t *testing.T, h http.Handler, method, path, token, body string) *httptest.ResponseRecorder {
t.Helper()
var r *http.Request
if body != "" {
r = httptest.NewRequest(method, path, strings.NewReader(body))
r.Header.Set("Content-Type", "application/json")
} else {
r = httptest.NewRequest(method, path, nil)
}
if token != "" {
r.Header.Set("Authorization", "Bearer "+token)
}
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
return w
}
// ---- auth -------------------------------------------------------------------------------
func TestAuth_AbsentAndWrongToken(t *testing.T) {
g := &fakeGuests{}
h := newTestServer(t, g, &fakeBackups{}, &fakeStore{}, nil)
if w := do(t, h, "GET", "/storage", "", ""); w.Code != http.StatusUnauthorized {
t.Fatalf("absent token: got %d, want 401", w.Code)
}
if w := do(t, h, "GET", "/storage", "nope", ""); w.Code != http.StatusUnauthorized {
t.Fatalf("wrong token: got %d, want 401", w.Code)
}
if len(g.cfgVMIDs) != 0 {
t.Fatal("guest config was read despite failed auth")
}
}
// ---- self-scoping: the headline security test -------------------------------------------
// A token for guest A (8200) cannot snapshot/rollback/backup guest B (9300): the agent refuses
// with 403 and the proxmox op is NEVER issued for B.
func TestSelfScoping_CrossGuestRefused(t *testing.T) {
g := &fakeGuests{}
b := &fakeBackups{}
h := newTestServer(t, g, b, &fakeStore{}, nil)
// token A targets guest B via explicit body vmid → 403, op not called.
if w := do(t, h, "POST", "/snapshot", "A", `{"vmid":9300,"snapname":"x"}`); w.Code != http.StatusForbidden {
t.Fatalf("cross-guest snapshot: got %d, want 403", w.Code)
}
if w := do(t, h, "POST", "/rollback", "A", `{"vmid":9300,"snapname":"x"}`); w.Code != http.StatusForbidden {
t.Fatalf("cross-guest rollback: got %d, want 403", w.Code)
}
if w := do(t, h, "POST", "/backup", "A", `{"vmid":9300}`); w.Code != http.StatusForbidden {
t.Fatalf("cross-guest backup: got %d, want 403", w.Code)
}
// token A targets guest B via explicit query vmid → 403.
if w := do(t, h, "GET", "/storage?vmid=9300", "A", ""); w.Code != http.StatusForbidden {
t.Fatalf("cross-guest storage query: got %d, want 403", w.Code)
}
if len(g.snapVMIDs) != 0 || len(g.rbVMIDs) != 0 {
t.Fatalf("a guest op was issued on a cross-guest request: snaps=%v rb=%v", g.snapVMIDs, g.rbVMIDs)
}
// the backup goroutine must never have started for B
time.Sleep(20 * time.Millisecond)
if called := b.called(); len(called) != 0 {
t.Fatalf("backup issued on a cross-guest request: %v", called)
}
}
// Own-guest ops call the proxmox op for the CORRECT VMID (the token's guest), even when the
// caller names its own guest explicitly.
func TestSelfScoping_OwnGuestUsesTokenVMID(t *testing.T) {
g := &fakeGuests{}
h := newTestServer(t, g, &fakeBackups{}, &fakeStore{}, nil)
if w := do(t, h, "POST", "/snapshot", "A", `{"snapname":"pre-deploy"}`); w.Code != http.StatusOK {
t.Fatalf("own snapshot: got %d (%s)", w.Code, w.Body.String())
}
if w := do(t, h, "POST", "/snapshot", "B", `{"vmid":9300,"snapname":"ok"}`); w.Code != http.StatusOK {
t.Fatalf("own snapshot (explicit matching vmid): got %d", w.Code)
}
if len(g.snapVMIDs) != 2 || g.snapVMIDs[0] != 8200 || g.snapVMIDs[1] != 9300 {
t.Fatalf("snapshot used wrong VMIDs: %v", g.snapVMIDs)
}
}
// ---- endpoints --------------------------------------------------------------------------
func TestStorage_ReturnsOnlyThisGuestMountsWithClass(t *testing.T) {
g := &fakeGuests{mounts: map[string]string{
"mp0": "fastpool:8,mp=/var/lib/docker,backup=1",
"mp1": "bulk:200,mp=/mnt/media,backup=0",
}}
sv := fakeStorage{targets: []hub.StorageTarget{
{Name: "fastpool", ClassHint: "fast"},
{Name: "bulk", ClassHint: "slow"},
}}
h := newTestServer(t, g, &fakeBackups{}, &fakeStore{}, sv)
w := do(t, h, "GET", "/storage", "A", "")
if w.Code != http.StatusOK {
t.Fatalf("storage: got %d (%s)", w.Code, w.Body.String())
}
var resp struct {
Data StorageResponse `json:"data"`
}
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode: %v", err)
}
if resp.Data.VMID != 8200 {
t.Fatalf("vmid: got %d want 8200", resp.Data.VMID)
}
byKey := map[string]MountInfo{}
for _, m := range resp.Data.Mounts {
byKey[m.Key] = m
}
if byKey["mp0"].Class != "fast" || byKey["mp0"].Storage != "fastpool" || byKey["mp0"].MountPoint != "/var/lib/docker" || !byKey["mp0"].Backup {
t.Fatalf("mp0 wrong: %+v", byKey["mp0"])
}
if byKey["mp1"].Class != "slow" || byKey["mp1"].Backup {
t.Fatalf("mp1 wrong: %+v", byKey["mp1"])
}
if g.cfgVMIDs[0] != 8200 {
t.Fatalf("guest config read for wrong vmid: %v", g.cfgVMIDs)
}
}
func TestRollback_RequiresSnapname(t *testing.T) {
g := &fakeGuests{}
h := newTestServer(t, g, &fakeBackups{}, &fakeStore{}, nil)
if w := do(t, h, "POST", "/rollback", "A", `{}`); w.Code != http.StatusBadRequest {
t.Fatalf("missing snapname: got %d, want 400", w.Code)
}
if len(g.rbVMIDs) != 0 {
t.Fatal("rollback issued without a snapname")
}
if w := do(t, h, "POST", "/rollback", "A", `{"snapname":"pre-deploy"}`); w.Code != http.StatusOK {
t.Fatalf("valid rollback: got %d", w.Code)
}
if len(g.rbVMIDs) != 1 || g.rbVMIDs[0] != 8200 {
t.Fatalf("rollback VMIDs: %v", g.rbVMIDs)
}
}
func TestSnapshot_RejectsBadName(t *testing.T) {
g := &fakeGuests{}
h := newTestServer(t, g, &fakeBackups{}, &fakeStore{}, nil)
if w := do(t, h, "POST", "/snapshot", "A", `{"snapname":"bad name/slash"}`); w.Code != http.StatusBadRequest {
t.Fatalf("bad snapname: got %d, want 400", w.Code)
}
if len(g.snapVMIDs) != 0 {
t.Fatal("snapshot issued with an invalid name")
}
}
func TestBackup_EnqueuesForTokenGuest(t *testing.T) {
g := &fakeGuests{}
b := &fakeBackups{}
st := &fakeStore{}
h := newTestServer(t, g, b, st, nil)
w := do(t, h, "POST", "/backup", "A", "")
if w.Code != http.StatusAccepted {
t.Fatalf("backup enqueue: got %d, want 202", w.Code)
}
// the goroutine should run and record for guest 8200
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if c := b.called(); len(c) == 1 && c[0] == 8200 {
break
}
time.Sleep(5 * time.Millisecond)
}
if c := b.called(); len(c) != 1 || c[0] != 8200 {
t.Fatalf("backup not enqueued for token guest: %v", c)
}
}
func dueOf(t *testing.T, h http.Handler) BackupDueResponse {
t.Helper()
w := do(t, h, "GET", "/backup/due", "A", "")
var resp struct {
Data BackupDueResponse `json:"data"`
}
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode due: %v", err)
}
return resp.Data
}
// testNow is 2026-06-10T12:00:00Z, cadence 24h.
func TestBackupDue_Cadence(t *testing.T) {
st := &fakeStore{}
h := newTestServer(t, &fakeGuests{}, &fakeBackups{}, st, nil)
// no backup recorded → due
if d := dueOf(t, h); !d.Due {
t.Fatal("expected due=true with no backup recorded")
}
// a successful backup 1h ago → NOT due (within the 24h window)
st.backups = []hub.Backup{{VMID: 8200, Success: true, StartedAt: "2026-06-10T11:00:00Z"}}
if d := dueOf(t, h); d.Due {
t.Fatalf("expected due=false 1h after a successful backup; reason=%q", d.Reason)
}
// a successful backup >24h ago → due again
st.backups = []hub.Backup{{VMID: 8200, Success: true, StartedAt: "2026-06-08T00:00:00Z"}}
if d := dueOf(t, h); !d.Due {
t.Fatal("expected due=true when the newest backup is older than the cadence")
}
// a FAILED backup 1h ago does NOT satisfy the cadence → still due
st.backups = []hub.Backup{{VMID: 8200, Success: false, StartedAt: "2026-06-10T11:00:00Z"}}
if d := dueOf(t, h); !d.Due {
t.Fatal("expected due=true: a failed backup must not count as a successful one")
}
}
// POST /backup returns a running phase; the job transitions running→done; a successful backup then
// flips /backup/due to false (the controller won't re-quiesce in a loop).
func TestBackupStatus_RunningToDone(t *testing.T) {
st := &fakeStore{}
b := &fakeBackups{}
srv := newTestServerS(t, &fakeGuests{}, b, st, nil)
h := srv.Handler()
// gate the backup goroutine so we can observe the running phase deterministically
b.gate = make(chan struct{})
w := do(t, h, "POST", "/backup", "A", "")
if w.Code != http.StatusAccepted {
t.Fatalf("POST /backup: got %d want 202", w.Code)
}
var br struct {
Data BackupResponse `json:"data"`
}
_ = json.Unmarshal(w.Body.Bytes(), &br)
if br.Data.Phase != PhaseRunning || br.Data.JobID == "" {
t.Fatalf("expected running phase + job id, got %+v", br.Data)
}
if ph := statusPhase(t, h); ph != PhaseRunning {
t.Fatalf("status while running: got %q want running", ph)
}
close(b.gate) // let the backup complete
// wait for done
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if statusPhase(t, h) == PhaseDone {
break
}
time.Sleep(5 * time.Millisecond)
}
if ph := statusPhase(t, h); ph != PhaseDone {
t.Fatalf("status after completion: got %q want done", ph)
}
}
func TestBackupStatus_RunningToFailed(t *testing.T) {
b := &fakeBackups{failErr: "vzdump exploded"}
srv := newTestServerS(t, &fakeGuests{}, b, &fakeStore{}, nil)
h := srv.Handler()
if do(t, h, "POST", "/backup", "A", "").Code != http.StatusAccepted {
t.Fatal("POST /backup not accepted")
}
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if statusPhase(t, h) == PhaseFailed {
break
}
time.Sleep(5 * time.Millisecond)
}
if ph := statusPhase(t, h); ph != PhaseFailed {
t.Fatalf("status after a failed backup: got %q want failed", ph)
}
}
// A second POST /backup while one is running does NOT start a second vzdump (single-flight).
func TestBackup_SingleFlight(t *testing.T) {
b := &fakeBackups{gate: make(chan struct{})}
srv := newTestServerS(t, &fakeGuests{}, b, &fakeStore{}, nil)
h := srv.Handler()
do(t, h, "POST", "/backup", "A", "")
do(t, h, "POST", "/backup", "A", "") // should be coalesced onto the running job
close(b.gate)
time.Sleep(30 * time.Millisecond)
if c := b.called(); len(c) != 1 {
t.Fatalf("expected a single vzdump for concurrent POSTs, got %d", len(c))
}
}
func statusPhase(t *testing.T, h http.Handler) string {
t.Helper()
w := do(t, h, "GET", "/backup/status", "A", "")
var resp struct {
Data BackupStatusResponse `json:"data"`
}
_ = json.Unmarshal(w.Body.Bytes(), &resp)
return resp.Data.Phase
}
func waitPhase(t *testing.T, h http.Handler, want string) {
t.Helper()
deadline := time.Now().Add(time.Second)
for time.Now().Before(deadline) {
if statusPhase(t, h) == want {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatalf("phase never reached %q (last=%q)", want, statusPhase(t, h))
}
// 8B.2: snapshot mode → the job flips to `snapshotted` mid-backup (before done), so the controller
// can resume its app early. The gated fake holds the backup open after firing onSnapshot.
func TestBackupStatus_Snapshotted_8B2(t *testing.T) {
b := &fakeBackups{gate: make(chan struct{}), fireSnapshot: true}
srv := newTestServerS(t, &fakeGuests{}, b, &fakeStore{}, nil)
h := srv.Handler()
if do(t, h, "POST", "/backup", "A", "").Code != http.StatusAccepted {
t.Fatal("POST /backup not accepted")
}
waitPhase(t, h, PhaseSnapshotted) // onSnapshot fired → snapshotted, while the backup is still open
close(b.gate) // let the backup complete
waitPhase(t, h, PhaseDone) // and it still proceeds to done
}
// 8B.2 fallback: stop/downgraded mode → onSnapshot never fires → phase never becomes `snapshotted`
// (the controller then resumes at done, exactly 8B).
func TestBackup_StopMode_NeverSnapshotted_8B2(t *testing.T) {
b := &fakeBackups{gate: make(chan struct{}), fireSnapshot: false}
srv := newTestServerS(t, &fakeGuests{}, b, &fakeStore{}, nil)
h := srv.Handler()
do(t, h, "POST", "/backup", "A", "")
time.Sleep(40 * time.Millisecond) // give the goroutine time; it must NOT snapshot
if ph := statusPhase(t, h); ph != PhaseRunning {
t.Fatalf("stop mode: phase %q, want running (snapshotted must not be emitted)", ph)
}
close(b.gate)
waitPhase(t, h, PhaseDone)
}
func TestBackupStatus_FiltersToThisGuest(t *testing.T) {
st := &fakeStore{backups: []hub.Backup{
{VMID: 9300, Success: true, StartedAt: "2026-06-10T10:00:00Z", Archive: "other"},
{VMID: 8200, Success: true, StartedAt: "2026-06-10T09:00:00Z", Archive: "mine-old"},
{VMID: 8200, Success: true, StartedAt: "2026-06-10T11:00:00Z", Archive: "mine-new"},
}}
h := newTestServer(t, &fakeGuests{}, &fakeBackups{}, st, nil)
w := do(t, h, "GET", "/backup/status", "A", "")
if !strings.Contains(w.Body.String(), "mine-new") || strings.Contains(w.Body.String(), "other") {
t.Fatalf("status not scoped to this guest / not latest: %s", w.Body.String())
}
}