package localapi import ( "context" "encoding/json" "errors" "net/http" "os" "path/filepath" "sort" "strings" "sync" "sync/atomic" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/backup" "gitea.dooplex.hu/admin/felhom-agent/internal/poweroff" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // fakePower records the power-off steps in order. type fakePower struct { mu sync.Mutex steps []string } func (f *fakePower) ShutdownGuest(_ context.Context, vmid int, timeout time.Duration) error { f.mu.Lock() defer f.mu.Unlock() f.steps = append(f.steps, "shutdown:"+itoaT(vmid)+":"+timeout.String()) return nil } func (f *fakePower) PowerOffHost(context.Context) error { f.mu.Lock() defer f.mu.Unlock() f.steps = append(f.steps, "poweroff") return nil } func (f *fakePower) got() []string { f.mu.Lock() defer f.mu.Unlock() return append([]string(nil), f.steps...) } func itoaT(n int) string { b, _ := json.Marshal(n); return string(b) } type fakeVzdump struct{ running bool } func (f fakeVzdump) Guests(context.Context) ([]proxmox.Guest, error) { return nil, nil } func (f fakeVzdump) Lock(context.Context, int) (string, bool, error) { return "", true, nil } func (f fakeVzdump) BackupRunning(context.Context, int) (bool, error) { return f.running, nil } func (f fakeVzdump) Unlock(context.Context, int) error { return nil } func (f fakeVzdump) Start(context.Context, int) error { return nil } func (f fakeVzdump) HasVzdumpSnapshot(context.Context, int) (bool, error) { return false, nil } func (f fakeVzdump) DeleteVzdumpSnapshot(context.Context, int) error { return nil } type powerRig struct { srv *Server h http.Handler power *fakePower state *poweroff.State report *[]string } func newPowerRig(t *testing.T, uptime time.Duration) *powerRig { t.Helper() srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil) srv.inFlight = &backup.InFlight{} srv.staleLock = fakeVzdump{} p := &fakePower{} st := poweroff.New(filepath.Join(t.TempDir(), "poweroff.json"), func() string { return "1700000000" }) var reports []string srv.SetHostPowerOff(HostPowerOffConfig{ State: st, Power: p, ReportNow: func(context.Context) bool { // The record must already exist when the hub is told — that is the point of telling it. if st.Current() == nil { reports = append(reports, "report-without-record") } else if len(p.got()) != 0 { reports = append(reports, "report-after-"+p.got()[0]) } else { reports = append(reports, "report") } return true }, Uptime: func() (time.Duration, bool) { return uptime, true }, Running: func() ([]int, error) { return []int{8200}, nil }, Now: func() time.Time { return testNow }, }) return &powerRig{srv: srv, h: srv.Handler(), power: p, state: st, report: &reports} } func (r *powerRig) waitSteps(t *testing.T, n int) []string { t.Helper() for i := 0; i < 200 && len(r.power.got()) < n; i++ { time.Sleep(time.Millisecond) } return r.power.got() } func reasonOf(t *testing.T, body []byte) string { t.Helper() var resp struct { Data struct { Reason string `json:"reason"` } `json:"data"` } if err := json.Unmarshal(body, &resp); err != nil { t.Fatalf("body %s: %v", body, err) } return resp.Data.Reason } // The whole path: 202, the record written, the hub told FIRST, then guest shutdown (120 s), then host off. func TestHostPowerOff_AcceptedOrder(t *testing.T) { r := newPowerRig(t, time.Hour) w := do(t, r.h, "POST", "/host/poweroff", "A", "") if w.Code != http.StatusAccepted { t.Fatalf("got %d want 202 (%s)", w.Code, w.Body.String()) } steps := r.waitSteps(t, 2) want := []string{"shutdown:8200:2m0s", "poweroff"} if len(steps) != 2 || steps[0] != want[0] || steps[1] != want[1] { t.Fatalf("steps = %v, want %v", steps, want) } // The heavy-operation gate stays taken: no backup can start between the checks and the power-off. if _, busy, ok := r.srv.inFlight.TryAcquire("backup:primary"); ok || busy != "host-poweroff" { t.Fatalf("after acceptance a backup could take the gate (ok=%v busy=%q)", ok, busy) } if len(*r.report) != 1 || (*r.report)[0] != "report" { t.Fatalf("hub report = %v, want one report carrying the record", *r.report) } if rec := r.state.Current(); rec == nil || rec.VMID != 8200 { t.Fatalf("record = %+v", rec) } } // Refused during a backup — the shared heavy-operation gate held by a backup tier. func TestHostPowerOff_RefusedDuringBackup(t *testing.T) { r := newPowerRig(t, time.Hour) release, _, ok := r.srv.inFlight.TryAcquire("backup:primary") if !ok { t.Fatal("could not hold the gate") } defer release() w := do(t, r.h, "POST", "/host/poweroff", "A", "") if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffBackup { t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffBackup) } time.Sleep(5 * time.Millisecond) if s := r.power.got(); len(s) != 0 { t.Fatalf("power steps ran during a backup: %v", s) } if r.state.Current() != nil { t.Fatal("a refused request must not use up the boot's one request") } // The same guard names a restore test and an OS step. release() for gate, want := range map[string]string{"restore-test": PowerOffRestoreTest, "os-pve-step": PowerOffOSUpdate} { rel, _, _ := r.srv.inFlight.TryAcquire(gate) w := do(t, r.h, "POST", "/host/poweroff", "A", "") rel() if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != want { t.Fatalf("%s: got %d %s, want 409 %s", gate, w.Code, w.Body.String(), want) } } } // A vzdump of the guest in flight (a Proxmox task, not the agent's own gate) refuses too. func TestHostPowerOff_RefusedDuringVzdump(t *testing.T) { r := newPowerRig(t, time.Hour) r.srv.staleLock = fakeVzdump{running: true} w := do(t, r.h, "POST", "/host/poweroff", "A", "") if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffBackup { t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffBackup) } } // One accepted request per boot: the second is refused and nothing runs twice. func TestHostPowerOff_SecondRequestSameBootRefused(t *testing.T) { r := newPowerRig(t, time.Hour) if w := do(t, r.h, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusAccepted { t.Fatalf("first: %d", w.Code) } r.waitSteps(t, 2) w := do(t, r.h, "POST", "/host/poweroff", "A", "") if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffAlreadyAsked { t.Fatalf("second: got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffAlreadyAsked) } time.Sleep(5 * time.Millisecond) if s := r.power.got(); len(s) != 2 { t.Fatalf("steps = %v, want exactly one run", s) } } // Refused in the first 10 minutes after boot; an unknown uptime refuses too (fail closed). func TestHostPowerOff_TooSoonAfterBoot(t *testing.T) { r := newPowerRig(t, 9*time.Minute) w := do(t, r.h, "POST", "/host/poweroff", "A", "") if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffTooSoon { t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffTooSoon) } r.srv.powerOff.Uptime = func() (time.Duration, bool) { return 0, false } w = do(t, r.h, "POST", "/host/poweroff", "A", "") if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffUnknown { t.Fatalf("unknown uptime: got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffUnknown) } } // Not configured → 503; another guest's vmid in the body → 403; no token → 401. func TestHostPowerOff_ScopeAndConfig(t *testing.T) { plain := newTestServer(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil) if w := do(t, plain, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusServiceUnavailable { t.Fatalf("unconfigured: %d", w.Code) } r := newPowerRig(t, time.Hour) if w := do(t, r.h, "POST", "/host/poweroff", "A", `{"vmid":9300}`); w.Code != http.StatusForbidden { t.Fatalf("cross-guest: %d", w.Code) } if w := do(t, r.h, "POST", "/host/poweroff", "", ""); w.Code != http.StatusUnauthorized { t.Fatalf("no token: %d", w.Code) } if r.state.Current() != nil || len(r.power.got()) != 0 { t.Fatal("a refused request changed state") } } // Another guest or VM on the host refuses: one controller must not switch off someone else's guest (security // review 2026-10-13; demo-hp runs a scratch guest and the Tester 1 VM). An unreadable process list refuses too. func TestHostPowerOff_RefusedWhileOtherGuestsRun(t *testing.T) { r := newPowerRig(t, time.Hour) r.srv.powerOff.Running = func() ([]int, error) { return []int{8200, 9202, 341}, nil } w := do(t, r.h, "POST", "/host/poweroff", "A", "") if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffOtherGuests { t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffOtherGuests) } r.srv.powerOff.Running = func() ([]int, error) { return nil, errors.New("no /proc") } w = do(t, r.h, "POST", "/host/poweroff", "A", "") if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffUnknown { t.Fatalf("unreadable: got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffUnknown) } if len(r.power.got()) != 0 || r.state.Current() != nil { t.Fatal("a refused request changed state") } } // Fail closed: an unwired vzdump reader refuses instead of skipping the check. func TestHostPowerOff_FailsClosedWithoutTaskReader(t *testing.T) { r := newPowerRig(t, time.Hour) r.srv.staleLock = nil w := do(t, r.h, "POST", "/host/poweroff", "A", "") if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffUnknown { t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffUnknown) } } // The /proc reader finds containers and VMs by their real command lines (as measured on demo-hp). func TestRunningGuestsFromProc(t *testing.T) { dir := t.TempDir() write := func(pid, cmd string) { _ = os.MkdirAll(filepath.Join(dir, pid), 0o755) _ = os.WriteFile(filepath.Join(dir, pid, "cmdline"), []byte(strings.ReplaceAll(cmd, " ", "\x00")+"\x00"), 0o644) } write("1927", "/usr/bin/lxc-start -F -n 9201") write("1900", "/usr/bin/lxc-start -F -n 9202") write("1823", "/usr/bin/kvm -id 341 -name night1004-tester1 -no-shutdown") write("974", "/usr/libexec/lxc/lxc-monitord --daemon") write("50", "/usr/bin/bash -c echo -n 5") _ = os.MkdirAll(filepath.Join(dir, "self"), 0o755) got, err := runningGuestsFromProc(dir) if err != nil { t.Fatal(err) } sort.Ints(got) if len(got) != 3 || got[0] != 341 || got[1] != 9201 || got[2] != 9202 { t.Fatalf("got %v, want [341 9201 9202]", got) } if _, err := runningGuestsFromProc(filepath.Join(dir, "missing")); err == nil { t.Fatal("an unreadable /proc must be an error") } } // switchVzdump is a vzdump reader the test can flip while the power-off runs. type switchVzdump struct { fakeVzdump on *atomic.Bool } func (f switchVzdump) BackupRunning(context.Context, int) (bool, error) { return f.on.Load(), nil } // A job that starts AFTER acceptance (security review 2026-10-13): the power-off waits; if the job is still running // when the wait ends, the power-off is cancelled — no guest shutdown, no host power-off, the gate given back. func TestHostPowerOff_JobStartedAfterAcceptance(t *testing.T) { // (a) it clears during the wait → the power-off goes ahead after it r := newPowerRig(t, time.Hour) var on atomic.Bool r.srv.staleLock = switchVzdump{on: &on} r.srv.powerOff.JobWait, r.srv.powerOff.JobPoll = 2*time.Second, 2*time.Millisecond r.srv.powerOff.ReportNow = func(context.Context) bool { on.Store(true); return true } // a vzdump starts now if w := do(t, r.h, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusAccepted { t.Fatalf("accept: %d %s", w.Code, w.Body.String()) } time.Sleep(30 * time.Millisecond) if s := r.power.got(); len(s) != 0 { t.Fatalf("a step ran while the vzdump ran: %v", s) } on.Store(false) if s := r.waitSteps(t, 2); len(s) != 2 { t.Fatalf("after the vzdump ended the steps = %v, want shutdown + poweroff", s) } // (b) it never clears → cancelled, nothing switched off, the heavy gate free again r2 := newPowerRig(t, time.Hour) var on2 atomic.Bool r2.srv.staleLock = switchVzdump{on: &on2} r2.srv.powerOff.JobWait, r2.srv.powerOff.JobPoll = 40*time.Millisecond, 2*time.Millisecond r2.srv.powerOff.ReportNow = func(context.Context) bool { on2.Store(true); return true } if w := do(t, r2.h, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusAccepted { t.Fatalf("accept: %d", w.Code) } time.Sleep(150 * time.Millisecond) if s := r2.power.got(); len(s) != 0 { t.Fatalf("cancelled power-off still ran: %v", s) } if busy := r2.srv.inFlight.Busy(); busy != "" { t.Fatalf("the heavy gate is still held after a cancel: %q", busy) } }