felhom-agent/internal/localapi/hostpoweroff_test.go

package localapi

import (
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
	"gitea.dooplex.hu/admin/felhom-agent/internal/poweroff"
	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// fakePower records the power-off steps in order.
type fakePower struct {
	mu    sync.Mutex
	steps []string
}

func (f *fakePower) ShutdownGuest(_ context.Context, vmid int, timeout time.Duration) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.steps = append(f.steps, "shutdown:"+itoaT(vmid)+":"+timeout.String())
	return nil
}
func (f *fakePower) PowerOffHost(context.Context) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.steps = append(f.steps, "poweroff")
	return nil
}
func (f *fakePower) got() []string {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]string(nil), f.steps...)
}

func itoaT(n int) string { b, _ := json.Marshal(n); return string(b) }

type fakeVzdump struct{ running bool }

func (f fakeVzdump) Guests(context.Context) ([]proxmox.Guest, error)      { return nil, nil }
func (f fakeVzdump) Lock(context.Context, int) (string, bool, error)      { return "", true, nil }
func (f fakeVzdump) BackupRunning(context.Context, int) (bool, error)     { return f.running, nil }
func (f fakeVzdump) Unlock(context.Context, int) error                    { return nil }
func (f fakeVzdump) Start(context.Context, int) error                     { return nil }
func (f fakeVzdump) HasVzdumpSnapshot(context.Context, int) (bool, error) { return false, nil }
func (f fakeVzdump) DeleteVzdumpSnapshot(context.Context, int) error      { return nil }

type powerRig struct {
	srv    *Server
	h      http.Handler
	power  *fakePower
	state  *poweroff.State
	report *[]string
}

func newPowerRig(t *testing.T, uptime time.Duration) *powerRig {
	t.Helper()
	srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
	srv.inFlight = &backup.InFlight{}
	srv.staleLock = fakeVzdump{}
	p := &fakePower{}
	st := poweroff.New(filepath.Join(t.TempDir(), "poweroff.json"), func() string { return "1700000000" })
	var reports []string
	srv.SetHostPowerOff(HostPowerOffConfig{
		State: st,
		Power: p,
		ReportNow: func(context.Context) bool {
			// The record must already exist when the hub is told — that is the point of telling it.
			if st.Current() == nil {
				reports = append(reports, "report-without-record")
			} else if len(p.got()) != 0 {
				reports = append(reports, "report-after-"+p.got()[0])
			} else {
				reports = append(reports, "report")
			}
			return true
		},
		Uptime:  func() (time.Duration, bool) { return uptime, true },
		Running: func() ([]int, error) { return []int{8200}, nil },
		Now:     func() time.Time { return testNow },
	})
	return &powerRig{srv: srv, h: srv.Handler(), power: p, state: st, report: &reports}
}

func (r *powerRig) waitSteps(t *testing.T, n int) []string {
	t.Helper()
	for i := 0; i < 200 && len(r.power.got()) < n; i++ {
		time.Sleep(time.Millisecond)
	}
	return r.power.got()
}

func reasonOf(t *testing.T, body []byte) string {
	t.Helper()
	var resp struct {
		Data struct {
			Reason string `json:"reason"`
		} `json:"data"`
	}
	if err := json.Unmarshal(body, &resp); err != nil {
		t.Fatalf("body %s: %v", body, err)
	}
	return resp.Data.Reason
}

// The whole path: 202, the record written, the hub told FIRST, then guest shutdown (120 s), then host off.
func TestHostPowerOff_AcceptedOrder(t *testing.T) {
	r := newPowerRig(t, time.Hour)
	w := do(t, r.h, "POST", "/host/poweroff", "A", "")
	if w.Code != http.StatusAccepted {
		t.Fatalf("got %d want 202 (%s)", w.Code, w.Body.String())
	}
	steps := r.waitSteps(t, 2)
	want := []string{"shutdown:8200:2m0s", "poweroff"}
	if len(steps) != 2 || steps[0] != want[0] || steps[1] != want[1] {
		t.Fatalf("steps = %v, want %v", steps, want)
	}
	// The heavy-operation gate stays taken: no backup can start between the checks and the power-off.
	if _, busy, ok := r.srv.inFlight.TryAcquire("backup:primary"); ok || busy != "host-poweroff" {
		t.Fatalf("after acceptance a backup could take the gate (ok=%v busy=%q)", ok, busy)
	}
	if len(*r.report) != 1 || (*r.report)[0] != "report" {
		t.Fatalf("hub report = %v, want one report carrying the record", *r.report)
	}
	if rec := r.state.Current(); rec == nil || rec.VMID != 8200 {
		t.Fatalf("record = %+v", rec)
	}
}

// Refused during a backup — the shared heavy-operation gate held by a backup tier.
func TestHostPowerOff_RefusedDuringBackup(t *testing.T) {
	r := newPowerRig(t, time.Hour)
	release, _, ok := r.srv.inFlight.TryAcquire("backup:primary")
	if !ok {
		t.Fatal("could not hold the gate")
	}
	defer release()
	w := do(t, r.h, "POST", "/host/poweroff", "A", "")
	if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffBackup {
		t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffBackup)
	}
	time.Sleep(5 * time.Millisecond)
	if s := r.power.got(); len(s) != 0 {
		t.Fatalf("power steps ran during a backup: %v", s)
	}
	if r.state.Current() != nil {
		t.Fatal("a refused request must not use up the boot's one request")
	}
	// The same guard names a restore test and an OS step.
	release()
	for gate, want := range map[string]string{"restore-test": PowerOffRestoreTest, "os-pve-step": PowerOffOSUpdate} {
		rel, _, _ := r.srv.inFlight.TryAcquire(gate)
		w := do(t, r.h, "POST", "/host/poweroff", "A", "")
		rel()
		if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != want {
			t.Fatalf("%s: got %d %s, want 409 %s", gate, w.Code, w.Body.String(), want)
		}
	}
}

// A vzdump of the guest in flight (a Proxmox task, not the agent's own gate) refuses too.
func TestHostPowerOff_RefusedDuringVzdump(t *testing.T) {
	r := newPowerRig(t, time.Hour)
	r.srv.staleLock = fakeVzdump{running: true}
	w := do(t, r.h, "POST", "/host/poweroff", "A", "")
	if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffBackup {
		t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffBackup)
	}
}

// One accepted request per boot: the second is refused and nothing runs twice.
func TestHostPowerOff_SecondRequestSameBootRefused(t *testing.T) {
	r := newPowerRig(t, time.Hour)
	if w := do(t, r.h, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusAccepted {
		t.Fatalf("first: %d", w.Code)
	}
	r.waitSteps(t, 2)
	w := do(t, r.h, "POST", "/host/poweroff", "A", "")
	if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffAlreadyAsked {
		t.Fatalf("second: got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffAlreadyAsked)
	}
	time.Sleep(5 * time.Millisecond)
	if s := r.power.got(); len(s) != 2 {
		t.Fatalf("steps = %v, want exactly one run", s)
	}
}

// Refused in the first 10 minutes after boot; an unknown uptime refuses too (fail closed).
func TestHostPowerOff_TooSoonAfterBoot(t *testing.T) {
	r := newPowerRig(t, 9*time.Minute)
	w := do(t, r.h, "POST", "/host/poweroff", "A", "")
	if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffTooSoon {
		t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffTooSoon)
	}
	r.srv.powerOff.Uptime = func() (time.Duration, bool) { return 0, false }
	w = do(t, r.h, "POST", "/host/poweroff", "A", "")
	if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffUnknown {
		t.Fatalf("unknown uptime: got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffUnknown)
	}
}

// Not configured → 503; another guest's vmid in the body → 403; no token → 401.
func TestHostPowerOff_ScopeAndConfig(t *testing.T) {
	plain := newTestServer(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
	if w := do(t, plain, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusServiceUnavailable {
		t.Fatalf("unconfigured: %d", w.Code)
	}
	r := newPowerRig(t, time.Hour)
	if w := do(t, r.h, "POST", "/host/poweroff", "A", `{"vmid":9300}`); w.Code != http.StatusForbidden {
		t.Fatalf("cross-guest: %d", w.Code)
	}
	if w := do(t, r.h, "POST", "/host/poweroff", "", ""); w.Code != http.StatusUnauthorized {
		t.Fatalf("no token: %d", w.Code)
	}
	if r.state.Current() != nil || len(r.power.got()) != 0 {
		t.Fatal("a refused request changed state")
	}
}

// Another guest or VM on the host refuses: one controller must not switch off someone else's guest (security
// review 2026-10-13; demo-hp runs a scratch guest and the Tester 1 VM). An unreadable process list refuses too.
func TestHostPowerOff_RefusedWhileOtherGuestsRun(t *testing.T) {
	r := newPowerRig(t, time.Hour)
	r.srv.powerOff.Running = func() ([]int, error) { return []int{8200, 9202, 341}, nil }
	w := do(t, r.h, "POST", "/host/poweroff", "A", "")
	if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffOtherGuests {
		t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffOtherGuests)
	}
	r.srv.powerOff.Running = func() ([]int, error) { return nil, errors.New("no /proc") }
	w = do(t, r.h, "POST", "/host/poweroff", "A", "")
	if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffUnknown {
		t.Fatalf("unreadable: got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffUnknown)
	}
	if len(r.power.got()) != 0 || r.state.Current() != nil {
		t.Fatal("a refused request changed state")
	}
}

// Fail closed: an unwired vzdump reader refuses instead of skipping the check.
func TestHostPowerOff_FailsClosedWithoutTaskReader(t *testing.T) {
	r := newPowerRig(t, time.Hour)
	r.srv.staleLock = nil
	w := do(t, r.h, "POST", "/host/poweroff", "A", "")
	if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffUnknown {
		t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffUnknown)
	}
}

// The /proc reader finds containers and VMs by their real command lines (as measured on demo-hp).
func TestRunningGuestsFromProc(t *testing.T) {
	dir := t.TempDir()
	write := func(pid, cmd string) {
		_ = os.MkdirAll(filepath.Join(dir, pid), 0o755)
		_ = os.WriteFile(filepath.Join(dir, pid, "cmdline"), []byte(strings.ReplaceAll(cmd, " ", "\x00")+"\x00"), 0o644)
	}
	write("1927", "/usr/bin/lxc-start -F -n 9201")
	write("1900", "/usr/bin/lxc-start -F -n 9202")
	write("1823", "/usr/bin/kvm -id 341 -name night1004-tester1 -no-shutdown")
	write("974", "/usr/libexec/lxc/lxc-monitord --daemon")
	write("50", "/usr/bin/bash -c echo -n 5")
	_ = os.MkdirAll(filepath.Join(dir, "self"), 0o755)
	got, err := runningGuestsFromProc(dir)
	if err != nil {
		t.Fatal(err)
	}
	sort.Ints(got)
	if len(got) != 3 || got[0] != 341 || got[1] != 9201 || got[2] != 9202 {
		t.Fatalf("got %v, want [341 9201 9202]", got)
	}
	if _, err := runningGuestsFromProc(filepath.Join(dir, "missing")); err == nil {
		t.Fatal("an unreadable /proc must be an error")
	}
}

// switchVzdump is a vzdump reader the test can flip while the power-off runs.
type switchVzdump struct {
	fakeVzdump
	on *atomic.Bool
}

func (f switchVzdump) BackupRunning(context.Context, int) (bool, error) { return f.on.Load(), nil }

// A job that starts AFTER acceptance (security review 2026-10-13): the power-off waits; if the job is still running
// when the wait ends, the power-off is cancelled — no guest shutdown, no host power-off, the gate given back.
func TestHostPowerOff_JobStartedAfterAcceptance(t *testing.T) {
	// (a) it clears during the wait → the power-off goes ahead after it
	r := newPowerRig(t, time.Hour)
	var on atomic.Bool
	r.srv.staleLock = switchVzdump{on: &on}
	r.srv.powerOff.JobWait, r.srv.powerOff.JobPoll = 2*time.Second, 2*time.Millisecond
	r.srv.powerOff.ReportNow = func(context.Context) bool { on.Store(true); return true } // a vzdump starts now
	if w := do(t, r.h, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusAccepted {
		t.Fatalf("accept: %d %s", w.Code, w.Body.String())
	}
	time.Sleep(30 * time.Millisecond)
	if s := r.power.got(); len(s) != 0 {
		t.Fatalf("a step ran while the vzdump ran: %v", s)
	}
	on.Store(false)
	if s := r.waitSteps(t, 2); len(s) != 2 {
		t.Fatalf("after the vzdump ended the steps = %v, want shutdown + poweroff", s)
	}

	// (b) it never clears → cancelled, nothing switched off, the heavy gate free again
	r2 := newPowerRig(t, time.Hour)
	var on2 atomic.Bool
	r2.srv.staleLock = switchVzdump{on: &on2}
	r2.srv.powerOff.JobWait, r2.srv.powerOff.JobPoll = 40*time.Millisecond, 2*time.Millisecond
	r2.srv.powerOff.ReportNow = func(context.Context) bool { on2.Store(true); return true }
	if w := do(t, r2.h, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusAccepted {
		t.Fatalf("accept: %d", w.Code)
	}
	time.Sleep(150 * time.Millisecond)
	if s := r2.power.got(); len(s) != 0 {
		t.Fatalf("cancelled power-off still ran: %v", s)
	}
	if busy := r2.srv.inFlight.Busy(); busy != "" {
		t.Fatalf("the heavy gate is still held after a cancel: %q", busy)
	}
}