From 3403c87c80649b8dae0abe5dfe1d40e8c3998985 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Tue, 13 Oct 2026 19:29:52 +0200 Subject: [PATCH] Host power-off for the household (R-998): POST /host/poweroff The guest's controller asks; the agent refuses in the first 10 min after boot, a second time in one boot (persisted, keyed by /proc/stat btime), and while a backup, restore test, OS step, controller update, drive format, NAS verify, escrow ceremony or vzdump runs (each a named 409 reason). On acceptance: the record first, a host report carrying deliberate_off accepted by the hub (bounded 60 s), then pct shutdown --timeout 120, then systemctl poweroff. Sudoers: two exact vectors (pct ^shutdown N --timeout N$, systemctl poweroff) in FELHOM_POWEROFF; nine injection cases. New TestEveryAliasIsGranted: a defined alias missing from the grant line passed the coverage check before. Every new test red-proofed. --- cmd/felhom-agent/main.go | 47 +++++ configs/felhom-agent.sudoers | 10 +- internal/capability/manifest.go | 2 + internal/capability/manifest_test.go | 36 ++++ internal/capability/r861_injection_test.go | 10 + internal/hub/collect.go | 11 + internal/hub/deliberate_off_test.go | 29 +++ internal/hub/loop.go | 10 + internal/hub/report.go | 11 + internal/localapi/guestbind.go | 21 ++ internal/localapi/hostpoweroff.go | 226 +++++++++++++++++++++ internal/localapi/hostpoweroff_test.go | 217 ++++++++++++++++++++ internal/localapi/server.go | 6 + internal/poweroff/state.go | 117 +++++++++++ internal/poweroff/state_test.go | 41 ++++ 15 files changed, 792 insertions(+), 2 deletions(-) create mode 100644 internal/hub/deliberate_off_test.go create mode 100644 internal/localapi/hostpoweroff.go create mode 100644 internal/localapi/hostpoweroff_test.go create mode 100644 internal/poweroff/state.go create mode 100644 internal/poweroff/state_test.go diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index a99071c..d2b8791 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -50,6 +50,7 @@ import ( "gitea.dooplex.hu/admin/felhom-agent/internal/pbs" "gitea.dooplex.hu/admin/felhom-agent/internal/pbsdr" "gitea.dooplex.hu/admin/felhom-agent/internal/poke" + "gitea.dooplex.hu/admin/felhom-agent/internal/poweroff" "gitea.dooplex.hu/admin/felhom-agent/internal/provision" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" "gitea.dooplex.hu/admin/felhom-agent/internal/reconcile" @@ -833,7 +834,25 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int // (The startup self-check log runs AFTER the pbsdr manager is wired below, so its snapshot // already carries the gated view — v0.86.0.) collector.SetCapabilityProber(probeAll) + // R-998 (doc 14): the household power-off record of THIS boot rides every report until the box is down. + powerOffState := poweroff.New(filepath.Join(cfg.OOB.WithDefaults().StateDir, "poweroff.json"), poweroff.ProcBtime) + collector.SetDeliberateOffSource(func() *hub.DeliberateOff { + if r := powerOffState.Current(); r != nil { + return &hub.DeliberateOff{RequestedAt: r.RequestedAt.UTC().Format(time.RFC3339)} + } + return nil + }) loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger) + // R-998: the power-off path waits until the hub ACCEPTED a report carrying the record. + powerOffAck := make(chan struct{}, 1) + loop.SetReportedHook(func(r *hub.HostReport) { + if r != nil && r.DeliberateOff != nil { + select { + case powerOffAck <- struct{}{}: + default: + } + } + }) interval := time.Duration(hcfg.PollSeconds) * time.Second // Heartbeat log-pull (v0.83.0): when the control envelope requests it, the NEXT // heartbeat carries the debug ring's tail (newest-kept, byte-capped in the loop). @@ -1180,6 +1199,34 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int } localSrv := buildLocalAPIServer(cfg, px, backupStore, heavyOps, observer, driveKnown, hostOps, gate, collector, client, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens) if localSrv != nil { + // R-998 (doc 14): POST /host/poweroff — guest shutdown, then host power-off; guards in hostpoweroff.go. + pwMode := proxmox.RunnerMode(cfg.Privileged.Mode) + if pwMode == "" { + pwMode = proxmox.RunnerSudo + } + localSrv.SetHostPowerOff(localapi.HostPowerOffConfig{ + State: powerOffState, + Power: localapi.NewGuestBinder(&proxmox.ExecRunner{Mode: pwMode, SudoPath: cfg.Privileged.SudoPath}, logger), + ReportNow: func(ctx context.Context) bool { + select { + case storageTrigger <- struct{}{}: + default: // a report is already queued; it runs after the record was written + } + select { + case <-powerOffAck: + return true + case <-ctx.Done(): + return false + } + }, + Uptime: func() (time.Duration, bool) { + b, err := strconv.ParseInt(poweroff.ProcBtime(), 10, 64) + if err != nil || b <= 0 { + return 0, false + } + return time.Since(time.Unix(b, 0)), true + }, + }) localSrv.SetAfterPrimaryBackup(func(ctx context.Context, vmid int) { // Let the controller finish bringing its apps back after the backup, then run (still under the gate). select { diff --git a/configs/felhom-agent.sudoers b/configs/felhom-agent.sudoers index b7ba9d7..8cb6ee1 100644 --- a/configs/felhom-agent.sudoers +++ b/configs/felhom-agent.sudoers @@ -80,7 +80,13 @@ Cmnd_Alias FELHOM_DNSMASQ = \ Cmnd_Alias FELHOM_GUESTHOOK = \ /usr/sbin/pct ^set [0-9]+ --hookscript local\:snippets/felhom-guest-hook\.sh$, \ /usr/sbin/pct ^set [0-9]+ --delete mp[0-9]+$, \ - /usr/sbin/pct ^reboot [0-9]+$ + /usr/sbin/pct ^reboot [0-9]+$, \ + /usr/sbin/pct ^shutdown [0-9]+ --timeout [0-9]+$ + +# Host power-off (R-998, doc 14): the household's „A gép kikapcsolása" — guest shutdown above, then this one fixed +# vector. Guarded agent-side (10 min after boot, one per boot, never during a backup/restore/update/drive job). +Cmnd_Alias FELHOM_POWEROFF = \ + /usr/bin/systemctl poweroff # Intermediary mount model (the drive hot-swap re-architecture). The agent keeps a SHARED host parent # /mnt/felhom-drives (self-bind + make-shared) and binds/unbinds each drive's felhom-data namespace UNDERNEATH it so the @@ -314,4 +320,4 @@ Cmnd_Alias FELHOM_GUESTNET = \ /usr/sbin/pct ^exec [0-9]+ -- pgrep -x dhclient$, \ /usr/sbin/pct ^exec [0-9]+ -- dhclient -pf /run/dhclient\.eth0\.pid -lf /var/lib/dhcp/dhclient\.eth0\.leases eth0$ -felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_FSTRIM, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_BACKUPTARGET, FELHOM_SELFHEAL, FELHOM_ESCROW, FELHOM_GUESTNET, FELHOM_SCRATCH_TEARDOWN, FELHOM_OSAPPLY +felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_FSTRIM, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_BACKUPTARGET, FELHOM_SELFHEAL, FELHOM_ESCROW, FELHOM_GUESTNET, FELHOM_SCRATCH_TEARDOWN, FELHOM_OSAPPLY, FELHOM_POWEROFF diff --git a/internal/capability/manifest.go b/internal/capability/manifest.go index b247d4a..3d9e0cc 100644 --- a/internal/capability/manifest.go +++ b/internal/capability/manifest.go @@ -115,6 +115,8 @@ var manifest = []Capability{ {"guesthook-register", "pre-start hook register", "/usr/sbin/pct", []string{"set", "9201", "--hookscript", "local:snippets/felhom-guest-hook.sh"}, false, ""}, {"guesthook-delete-mp", "dead mountpoint slot delete (C1 net)", "/usr/sbin/pct", []string{"set", "9201", "--delete", "mp0"}, false, ""}, {"guest-reboot", "enroll activate-binds reboot", "/usr/sbin/pct", []string{"reboot", "9201"}, false, ""}, + {"guest-shutdown", "household power-off: guest shutdown (R-998)", "/usr/sbin/pct", []string{"shutdown", "9201", "--timeout", "120"}, false, ""}, + {"host-poweroff", "household power-off: host power-off (R-998)", "/usr/bin/systemctl", []string{"poweroff"}, false, ""}, // ---- LAN split-horizon resolver (dnsmasq) ---- {"dnsmasq-install", "dnsmasq package install", "/usr/bin/apt-get", []string{"install", "-y", "-q", "dnsmasq"}, false, ""}, diff --git a/internal/capability/manifest_test.go b/internal/capability/manifest_test.go index c0edd3a..f910463 100644 --- a/internal/capability/manifest_test.go +++ b/internal/capability/manifest_test.go @@ -311,3 +311,39 @@ func TestWGCapabilityCriticality(t *testing.T) { } } } + +// TestEveryAliasIsGranted closes a gap found while adding FELHOM_POWEROFF (R-998): the coverage check above reads +// every Cmnd_Alias body whether or not the alias appears on the grant line, so a manifest command whose alias was +// never GRANTED still passed — and failed with „a password is required" on the box. Every defined alias must be +// named on the `felhom-agent ALL=(root) NOPASSWD:` line. Red-proof: drop FELHOM_POWEROFF from that line. +func TestEveryAliasIsGranted(t *testing.T) { + data, err := os.ReadFile(sudoersPath) + if err != nil { + t.Fatalf("read sudoers: %v", err) + } + var defined []string + granted := map[string]bool{} + grantLines := 0 + for _, ln := range strings.Split(string(data), "\n") { + tr := strings.TrimSpace(ln) + if strings.HasPrefix(tr, "Cmnd_Alias ") { + if f := strings.Fields(tr); len(f) >= 2 { + defined = append(defined, f[1]) + } + } + if strings.HasPrefix(tr, "felhom-agent ") && strings.Contains(tr, "NOPASSWD:") { + grantLines++ + for _, a := range strings.Split(tr[strings.Index(tr, "NOPASSWD:")+len("NOPASSWD:"):], ",") { + granted[strings.TrimSpace(a)] = true + } + } + } + if grantLines == 0 || len(defined) < 10 { + t.Fatalf("parsed %d grant line(s), %d aliases — parser broke", grantLines, len(defined)) + } + for _, a := range defined { + if !granted[a] { + t.Errorf("Cmnd_Alias %s is defined but never granted to felhom-agent", a) + } + } +} diff --git a/internal/capability/r861_injection_test.go b/internal/capability/r861_injection_test.go index c0b8022..561c685 100644 --- a/internal/capability/r861_injection_test.go +++ b/internal/capability/r861_injection_test.go @@ -53,6 +53,16 @@ var r861Injections = []string{ "/usr/sbin/pct exec 9201 -- tee /etc/felhom-controller-image", "/usr/local/sbin/felhom-priv-apply controller-image 9201 9202", "/usr/local/sbin/felhom-priv-apply controller-image 9201;id", + // R-998 (host power-off): the two new grants take exactly their fixed shape — no extra option, no other target + "/usr/sbin/pct shutdown 9201 --timeout 120 --forceStop 1", + "/usr/sbin/pct shutdown 9201 --forceStop 1 --timeout 120", + "/usr/sbin/pct shutdown 9201", + "/usr/sbin/pct shutdown 9201 --timeout -1", + "/usr/bin/systemctl poweroff --force", + "/usr/bin/systemctl poweroff -ff", + "/usr/bin/systemctl reboot", + "/usr/bin/systemctl halt", + "/usr/bin/systemctl kexec", } func TestSudoersRefusesTheR861Injections(t *testing.T) { diff --git a/internal/hub/collect.go b/internal/hub/collect.go index 4c185a4..4cf836c 100644 --- a/internal/hub/collect.go +++ b/internal/hub/collect.go @@ -123,6 +123,7 @@ type Collector struct { ctrlSup ControllerSupervisorReporter // R-523: in-guest controller supervisor (nil → stanza omitted) guestNet GuestNetReporter // R-54: per-guest network watchdog (nil → stanza omitted) diskTrim GuestDiskTrimReporter // R-444: weekly guest disk trim (nil → stanza omitted) + deliberateOff func() *DeliberateOff // R-998: household power-off record of this boot (nil → omitted) foreignKey ForeignKeyArchiveReporter // R-366 slice 2 (nil → omitted) selfUpdate SelfUpdateReporter // D1: agent self-update pending status (nil → false) mgmtPlane MgmtPlaneReporter // G1: management-plane health (nil → stanza omitted) @@ -255,6 +256,12 @@ func (c *Collector) SetGuestDiskTrimReporter(r GuestDiskTrimReporter) *Collector return c } +// SetDeliberateOffSource wires the R-998 power-off record (poweroff.State.Current, adapted). +func (c *Collector) SetDeliberateOffSource(f func() *DeliberateOff) *Collector { + c.deliberateOff = f + return c +} + // SelfUpdateReporter is the D1 seam the selfupdate commit-manager plugs into (same consumer-side // pattern — hub does not import selfupdate). nil (feature not wired) → pending=false on the report. type SelfUpdateReporter interface { @@ -431,6 +438,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { if c.oob != nil { report.OOB = c.oob.OOBStatus(ctx) } + // R-998: a household power-off accepted in this boot rides every report until the box is down. + if c.deliberateOff != nil { + report.DeliberateOff = c.deliberateOff() + } return report, nil } diff --git a/internal/hub/deliberate_off_test.go b/internal/hub/deliberate_off_test.go new file mode 100644 index 0000000..3279a7e --- /dev/null +++ b/internal/hub/deliberate_off_test.go @@ -0,0 +1,29 @@ +package hub + +import ( + "context" + "errors" + "testing" + "time" +) + +// R-998: the power-off path waits on SetReportedHook, so the hook must fire only for a report the hub +// ACCEPTED — a failed push must not let the box power off believing the hub knows. +func TestLoop_ReportedHookOnlyAfterAcceptedReport(t *testing.T) { + var cn, rn int32 + rep := &HostReport{DeliberateOff: &DeliberateOff{RequestedAt: "2026-10-13T18:00:00Z"}} + loop := NewLoop(&fakeCollector{report: rep, n: &cn}, + &fakeReporter{env: &ControlEnvelope{}, errSeq: []error{errors.New("hub 5xx")}, n: &rn}, + 900*time.Second, quietLogger()) + var seen []*HostReport + loop.SetReportedHook(func(r *HostReport) { seen = append(seen, r) }) + + loop.cycle(context.Background(), 900*time.Second) // the push fails + if len(seen) != 0 { + t.Fatalf("hook fired for a refused report: %d", len(seen)) + } + loop.cycle(context.Background(), 900*time.Second) // the push is accepted + if len(seen) != 1 || seen[0].DeliberateOff == nil || seen[0].DeliberateOff.RequestedAt != "2026-10-13T18:00:00Z" { + t.Fatalf("hook after an accepted report = %+v", seen) + } +} diff --git a/internal/hub/loop.go b/internal/hub/loop.go index 1c9435c..50f0457 100644 --- a/internal/hub/loop.go +++ b/internal/hub/loop.go @@ -66,8 +66,15 @@ type Loop struct { // retry, no duplicate shipping). Loop state is single-goroutine (cycle only). logTailSource func(maxBytes int) []string logTailPending bool + + // reported (R-998) is called after every report the hub ACCEPTED, with that report. The power-off + // path waits on it so the hub has the deliberate-off record before the box goes down. + reported func(r *HostReport) } +// SetReportedHook wires the R-998 after-report observer. Optional. +func (l *Loop) SetReportedHook(f func(r *HostReport)) { l.reported = f } + // logTailMaxBytes caps the heartbeat log tail (newest lines kept). const logTailMaxBytes = 128 * 1024 @@ -156,6 +163,9 @@ func (l *Loop) cycle(ctx context.Context, current time.Duration) time.Duration { l.logger.Warn("hub: report failed; keeping current interval", "err", err) return current } + if l.reported != nil { + l.reported(report) + } if report.LogTail != nil { // Transparency: the pull is visible in the box's own log (and thus in the ring). l.logger.Info("operator log pull served", "component", "agent", "lines", len(report.LogTail.Lines)) diff --git a/internal/hub/report.go b/internal/hub/report.go index aed7fec..efd9581 100644 --- a/internal/hub/report.go +++ b/internal/hub/report.go @@ -144,6 +144,12 @@ type HostReport struct { // in the steady state, so the cross-repo host-report golden stays byte-stable. LogTail *LogTail `json:"log_tail,omitempty"` + // DeliberateOff (R-998, doc 14): the household asked this box to power off, in THIS boot. Present on + // every report from the request until the box goes down; absent again after the next boot — that + // absence is how the hub learns the box is back. The hub holds its silence alarms while it is set + // (14 days). `omitempty`: absent in the steady state, so the cross-repo host-report golden stays stable. + DeliberateOff *DeliberateOff `json:"deliberate_off,omitempty"` + // OOB is the operator-access health stanza (TASK H1). It answers the operator's question — "can I // get into this box right now, and if not, why" — from the hub: felhom-sshd up + on which port, // locally reachable, the tunnel handshake age (the OOB path rides wg-felhom), whether the operator @@ -758,3 +764,8 @@ type ForeignKeyArchives struct { Oldest string `json:"oldest"` Newest string `json:"newest"` } + +// DeliberateOff is the household-requested power-off record (R-998). +type DeliberateOff struct { + RequestedAt string `json:"requested_at"` // RFC3339 UTC, the agent's clock +} diff --git a/internal/localapi/guestbind.go b/internal/localapi/guestbind.go index 6f1c94c..e4fdd4f 100644 --- a/internal/localapi/guestbind.go +++ b/internal/localapi/guestbind.go @@ -1,6 +1,7 @@ package localapi import ( + "time" "context" "fmt" "log/slog" @@ -104,6 +105,26 @@ func (b *GuestBinder) RebootGuest(ctx context.Context, vmid int) error { return nil } +// ShutdownGuest stops the guest cleanly with a time limit (R-998 host power-off, step 2). `pct shutdown` +// blocks until the guest is down or the limit passes; then it fails and the guest is still running. +func (b *GuestBinder) ShutdownGuest(ctx context.Context, vmid int, timeout time.Duration) error { + secs := int(timeout / time.Second) + if secs < 1 { + secs = 1 + } + b.logger.Warn("host-poweroff: shutting the guest down", "vmid", vmid, "timeout_s", secs) + if err := b.run(ctx, "pct", "shutdown", strconv.Itoa(vmid), "--timeout", strconv.Itoa(secs)); err != nil { + return fmt.Errorf("host-poweroff: pct shutdown %d: %w", vmid, err) + } + return nil +} + +// PowerOffHost powers the Proxmox host off (R-998, step 3). The one fixed vector `/usr/bin/systemctl poweroff`. +func (b *GuestBinder) PowerOffHost(ctx context.Context) error { + b.logger.Warn("host-poweroff: powering the host off") + return b.run(ctx, "/usr/bin/systemctl", "poweroff") +} + func (b *GuestBinder) run(ctx context.Context, name string, args ...string) error { _, stderr, err := b.runner.Run(ctx, name, args...) if err != nil { diff --git a/internal/localapi/hostpoweroff.go b/internal/localapi/hostpoweroff.go new file mode 100644 index 0000000..31a92a5 --- /dev/null +++ b/internal/localapi/hostpoweroff.go @@ -0,0 +1,226 @@ +package localapi + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/poweroff" +) + +// ── POST /host/poweroff (R-998, documentation/architecture/14-host-power-off.md) ────────────────────── +// +// The household's „A gép kikapcsolása": shut THIS guest down cleanly, then power the whole host off. The +// first local-API route whose effect reaches beyond the caller's own guest — a compromised controller can +// now switch the box off, and nobody can switch it on remotely. The mitigations are the limits below: +// +// - refused in the first 10 minutes after boot (no power-off loop); +// - at most ONE accepted request per boot (poweroff.State, persisted, keyed by the boot); +// - refused while a backup, restore test, OS update step, controller update, drive format, NAS verify +// or escrow ceremony runs, or a vzdump of the guest is in flight — each named in the 409. +// +// On acceptance the record is written FIRST, the hub is given a report carrying it (bounded wait), and +// only then: guest shutdown (time-limited), host power-off. Pinned by hostpoweroff_test.go. + +// HostPowerer is the privileged half: both run as fixed sudo vectors (configs/felhom-agent.sudoers). +type HostPowerer interface { + ShutdownGuest(ctx context.Context, vmid int, timeout time.Duration) error + PowerOffHost(ctx context.Context) error +} + +// HostPowerOffConfig wires the route. A nil config leaves it answering 503. +type HostPowerOffConfig struct { + State *poweroff.State + Power HostPowerer + ReportNow func(ctx context.Context) bool // push a report carrying the record; true = the hub accepted it + Uptime func() (time.Duration, bool) // time since host boot; false = unknown (refuse) + MinUptime time.Duration // default 10 min + GuestStop time.Duration // pct shutdown --timeout; default 120 s + ReportMax time.Duration // longest wait for the hub; default 60 s + Now func() time.Time +} + +// Refusal codes — the controller maps each to a household sentence. Stable wire values. +const ( + PowerOffTooSoon = "too_soon_after_boot" + PowerOffAlreadyAsked = "already_requested_this_boot" + PowerOffBackup = "backup_running" + PowerOffRestoreTest = "restore_test_running" + PowerOffOSUpdate = "os_update_running" + PowerOffCtrlUpdate = "controller_update_running" + PowerOffDrive = "drive_operation_running" + PowerOffEscrow = "escrow_ceremony_running" + PowerOffBusy = "host_busy" + PowerOffUnknown = "state_unknown" +) + +// SetHostPowerOff enables POST /host/poweroff. +func (s *Server) SetHostPowerOff(c HostPowerOffConfig) { + if c.MinUptime == 0 { + c.MinUptime = 10 * time.Minute + } + if c.GuestStop == 0 { + c.GuestStop = 120 * time.Second + } + if c.ReportMax == 0 { + c.ReportMax = 60 * time.Second + } + if c.Now == nil { + c.Now = time.Now + } + s.powerOff = &c +} + +type hostPowerOffRequest struct { + VMID int `json:"vmid"` +} + +func (s *Server) handleHostPowerOff(w http.ResponseWriter, r *http.Request, vmid int) { + c := s.powerOff + if c == nil || c.State == nil || c.Power == nil { + writeErr(w, http.StatusServiceUnavailable, "host power-off not configured on this host") + return + } + if r.ContentLength != 0 { + var req hostPowerOffRequest + if !decodeBody(w, r, &req) { + return + } + if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) { + return + } + } + // One decision at a time: two requests racing must not both pass the guards before either marks. + s.powerOffMu.Lock() + defer s.powerOffMu.Unlock() + if code, detail := s.powerOffBlocker(r.Context(), vmid); code != "" { + s.logger.Info("local-api: host power-off refused", "vmid", vmid, "reason", code, "detail", detail) + writeStatus(w, http.StatusConflict, false, map[string]any{"reason": code, "detail": detail}, "host power-off refused: "+code) + return + } + rec, err := c.State.Mark(c.Now(), vmid) + if err != nil { + code := PowerOffUnknown + if errors.Is(err, poweroff.ErrAlreadyThisBoot) { + code = PowerOffAlreadyAsked + } + s.logger.Warn("local-api: host power-off refused (record)", "vmid", vmid, "reason", code, "err", err) + writeStatus(w, http.StatusConflict, false, map[string]any{"reason": code, "detail": err.Error()}, "host power-off refused: "+code) + return + } + s.logger.Warn("local-api: HOST POWER-OFF accepted (household request)", "vmid", vmid, + "requested_at", rec.RequestedAt.Format(time.RFC3339)) + base := s.baseCtx + if base == nil { + base = context.Background() + } + go s.runHostPowerOff(base, vmid) + writeStatus(w, http.StatusAccepted, true, map[string]any{ + "vmid": vmid, "powering_off": true, "requested_at": rec.RequestedAt.Format(time.RFC3339), + }, "") +} + +// runHostPowerOff: hub first (bounded), then the guest (time-limited), then the host. A failed guest +// shutdown does not stop the power-off — the household asked for it, and Proxmox's own shutdown stops +// any guest still running (pve-guests) before the host goes down. +func (s *Server) runHostPowerOff(base context.Context, vmid int) { + c := s.powerOff + if c.ReportNow != nil { + ctx, cancel := context.WithTimeout(base, c.ReportMax) + ok := c.ReportNow(ctx) + cancel() + s.logger.Warn("host-poweroff: step 1/3 — hub told", "vmid", vmid, "hub_accepted", ok) + } + start := time.Now() + ctx, cancel := context.WithTimeout(base, c.GuestStop+60*time.Second) + err := c.Power.ShutdownGuest(ctx, vmid, c.GuestStop) + cancel() + if err != nil { + s.logger.Error("host-poweroff: step 2/3 — guest shutdown failed; powering off anyway", "vmid", vmid, + "took", time.Since(start).Round(time.Second), "err", err) + } else { + s.logger.Warn("host-poweroff: step 2/3 — guest stopped", "vmid", vmid, "took", time.Since(start).Round(time.Second)) + } + s.logger.Warn("host-poweroff: step 3/3 — powering the host off") + ctx, cancel = context.WithTimeout(base, 60*time.Second) + defer cancel() + if err := c.Power.PowerOffHost(ctx); err != nil { + s.logger.Error("host-poweroff: host power-off command failed", "err", err) + } +} + +// powerOffBlocker returns the first reason to refuse ("" = none) and a detail for the log/operator. +func (s *Server) powerOffBlocker(ctx context.Context, vmid int) (string, string) { + c := s.powerOff + if c.State.Current() != nil { + return PowerOffAlreadyAsked, "a power-off was already accepted in this boot" + } + if c.Uptime == nil { + return PowerOffUnknown, "uptime not wired" + } + up, ok := c.Uptime() + if !ok { + return PowerOffUnknown, "cannot read the host uptime" + } + if up < c.MinUptime { + return PowerOffTooSoon, "host up " + up.Round(time.Second).String() + ", minimum " + c.MinUptime.String() + } + if busy := s.inFlight.Busy(); busy != "" { + switch { + case strings.HasPrefix(busy, "backup:"): + return PowerOffBackup, busy + case busy == "restore-test": + return PowerOffRestoreTest, busy + case strings.HasPrefix(busy, "os-"): + return PowerOffOSUpdate, busy + default: + return PowerOffBusy, busy + } + } + s.jobsMu.Lock() + for k, j := range s.jobs { + if j != nil && backupInFlight(j.Phase) { + s.jobsMu.Unlock() + return PowerOffBackup, "backup job " + j.JobID + " on target " + k.target + } + } + s.jobsMu.Unlock() + s.swapMu.Lock() + for v, on := range s.swapInFlight { + if on { + s.swapMu.Unlock() + return PowerOffCtrlUpdate, "controller swap in flight for guest " + strconv.Itoa(v) + } + } + s.swapMu.Unlock() + if s.formatJobs != nil { + if j := s.formatJobs.get(); j != nil && j.Phase == formatPhaseRunning { + return PowerOffDrive, "drive format running" + } + } + s.netVerifyMu.Lock() + nv := s.netVerifyCur != nil && s.netVerifyCur.Phase == netVerifyPhaseRunning + s.netVerifyMu.Unlock() + if nv { + return PowerOffDrive, "network storage verify running" + } + s.escrowMu.Lock() + esc := s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning + s.escrowMu.Unlock() + if esc { + return PowerOffEscrow, "escrow ceremony running" + } + if s.staleLock != nil { + running, err := s.staleLock.BackupRunning(ctx, vmid) + if err != nil { + return PowerOffUnknown, "cannot list running Proxmox tasks: " + err.Error() + } + if running { + return PowerOffBackup, "vzdump running for guest " + strconv.Itoa(vmid) + } + } + return "", "" +} diff --git a/internal/localapi/hostpoweroff_test.go b/internal/localapi/hostpoweroff_test.go new file mode 100644 index 0000000..ed6c0c0 --- /dev/null +++ b/internal/localapi/hostpoweroff_test.go @@ -0,0 +1,217 @@ +package localapi + +import ( + "context" + "encoding/json" + "net/http" + "path/filepath" + "sync" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/backup" + "gitea.dooplex.hu/admin/felhom-agent/internal/poweroff" + "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" +) + +// fakePower records the power-off steps in order. +type fakePower struct { + mu sync.Mutex + steps []string +} + +func (f *fakePower) ShutdownGuest(_ context.Context, vmid int, timeout time.Duration) error { + f.mu.Lock() + defer f.mu.Unlock() + f.steps = append(f.steps, "shutdown:"+itoaT(vmid)+":"+timeout.String()) + return nil +} +func (f *fakePower) PowerOffHost(context.Context) error { + f.mu.Lock() + defer f.mu.Unlock() + f.steps = append(f.steps, "poweroff") + return nil +} +func (f *fakePower) got() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.steps...) +} + +func itoaT(n int) string { b, _ := json.Marshal(n); return string(b) } + +type fakeVzdump struct{ running bool } + +func (f fakeVzdump) Guests(context.Context) ([]proxmox.Guest, error) { return nil, nil } +func (f fakeVzdump) Lock(context.Context, int) (string, bool, error) { return "", true, nil } +func (f fakeVzdump) BackupRunning(context.Context, int) (bool, error) { return f.running, nil } +func (f fakeVzdump) Unlock(context.Context, int) error { return nil } +func (f fakeVzdump) Start(context.Context, int) error { return nil } +func (f fakeVzdump) HasVzdumpSnapshot(context.Context, int) (bool, error) { return false, nil } +func (f fakeVzdump) DeleteVzdumpSnapshot(context.Context, int) error { return nil } + +type powerRig struct { + srv *Server + h http.Handler + power *fakePower + state *poweroff.State + report *[]string +} + +func newPowerRig(t *testing.T, uptime time.Duration) *powerRig { + t.Helper() + srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil) + srv.inFlight = &backup.InFlight{} + p := &fakePower{} + st := poweroff.New(filepath.Join(t.TempDir(), "poweroff.json"), func() string { return "1700000000" }) + var reports []string + srv.SetHostPowerOff(HostPowerOffConfig{ + State: st, + Power: p, + ReportNow: func(context.Context) bool { + // The record must already exist when the hub is told — that is the point of telling it. + if st.Current() == nil { + reports = append(reports, "report-without-record") + } else if len(p.got()) != 0 { + reports = append(reports, "report-after-"+p.got()[0]) + } else { + reports = append(reports, "report") + } + return true + }, + Uptime: func() (time.Duration, bool) { return uptime, true }, + Now: func() time.Time { return testNow }, + }) + return &powerRig{srv: srv, h: srv.Handler(), power: p, state: st, report: &reports} +} + +func (r *powerRig) waitSteps(t *testing.T, n int) []string { + t.Helper() + for i := 0; i < 200 && len(r.power.got()) < n; i++ { + time.Sleep(time.Millisecond) + } + return r.power.got() +} + +func reasonOf(t *testing.T, body []byte) string { + t.Helper() + var resp struct { + Data struct { + Reason string `json:"reason"` + } `json:"data"` + } + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("body %s: %v", body, err) + } + return resp.Data.Reason +} + +// The whole path: 202, the record written, the hub told FIRST, then guest shutdown (120 s), then host off. +func TestHostPowerOff_AcceptedOrder(t *testing.T) { + r := newPowerRig(t, time.Hour) + w := do(t, r.h, "POST", "/host/poweroff", "A", "") + if w.Code != http.StatusAccepted { + t.Fatalf("got %d want 202 (%s)", w.Code, w.Body.String()) + } + steps := r.waitSteps(t, 2) + want := []string{"shutdown:8200:2m0s", "poweroff"} + if len(steps) != 2 || steps[0] != want[0] || steps[1] != want[1] { + t.Fatalf("steps = %v, want %v", steps, want) + } + if len(*r.report) != 1 || (*r.report)[0] != "report" { + t.Fatalf("hub report = %v, want one report carrying the record", *r.report) + } + if rec := r.state.Current(); rec == nil || rec.VMID != 8200 { + t.Fatalf("record = %+v", rec) + } +} + +// Refused during a backup — the shared heavy-operation gate held by a backup tier. +func TestHostPowerOff_RefusedDuringBackup(t *testing.T) { + r := newPowerRig(t, time.Hour) + release, _, ok := r.srv.inFlight.TryAcquire("backup:primary") + if !ok { + t.Fatal("could not hold the gate") + } + defer release() + w := do(t, r.h, "POST", "/host/poweroff", "A", "") + if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffBackup { + t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffBackup) + } + time.Sleep(5 * time.Millisecond) + if s := r.power.got(); len(s) != 0 { + t.Fatalf("power steps ran during a backup: %v", s) + } + if r.state.Current() != nil { + t.Fatal("a refused request must not use up the boot's one request") + } + // The same guard names a restore test and an OS step. + release() + for gate, want := range map[string]string{"restore-test": PowerOffRestoreTest, "os-pve-step": PowerOffOSUpdate} { + rel, _, _ := r.srv.inFlight.TryAcquire(gate) + w := do(t, r.h, "POST", "/host/poweroff", "A", "") + rel() + if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != want { + t.Fatalf("%s: got %d %s, want 409 %s", gate, w.Code, w.Body.String(), want) + } + } +} + +// A vzdump of the guest in flight (a Proxmox task, not the agent's own gate) refuses too. +func TestHostPowerOff_RefusedDuringVzdump(t *testing.T) { + r := newPowerRig(t, time.Hour) + r.srv.staleLock = fakeVzdump{running: true} + w := do(t, r.h, "POST", "/host/poweroff", "A", "") + if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffBackup { + t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffBackup) + } +} + +// One accepted request per boot: the second is refused and nothing runs twice. +func TestHostPowerOff_SecondRequestSameBootRefused(t *testing.T) { + r := newPowerRig(t, time.Hour) + if w := do(t, r.h, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusAccepted { + t.Fatalf("first: %d", w.Code) + } + r.waitSteps(t, 2) + w := do(t, r.h, "POST", "/host/poweroff", "A", "") + if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffAlreadyAsked { + t.Fatalf("second: got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffAlreadyAsked) + } + time.Sleep(5 * time.Millisecond) + if s := r.power.got(); len(s) != 2 { + t.Fatalf("steps = %v, want exactly one run", s) + } +} + +// Refused in the first 10 minutes after boot; an unknown uptime refuses too (fail closed). +func TestHostPowerOff_TooSoonAfterBoot(t *testing.T) { + r := newPowerRig(t, 9*time.Minute) + w := do(t, r.h, "POST", "/host/poweroff", "A", "") + if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffTooSoon { + t.Fatalf("got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffTooSoon) + } + r.srv.powerOff.Uptime = func() (time.Duration, bool) { return 0, false } + w = do(t, r.h, "POST", "/host/poweroff", "A", "") + if w.Code != http.StatusConflict || reasonOf(t, w.Body.Bytes()) != PowerOffUnknown { + t.Fatalf("unknown uptime: got %d %s, want 409 %s", w.Code, w.Body.String(), PowerOffUnknown) + } +} + +// Not configured → 503; another guest's vmid in the body → 403; no token → 401. +func TestHostPowerOff_ScopeAndConfig(t *testing.T) { + plain := newTestServer(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil) + if w := do(t, plain, "POST", "/host/poweroff", "A", ""); w.Code != http.StatusServiceUnavailable { + t.Fatalf("unconfigured: %d", w.Code) + } + r := newPowerRig(t, time.Hour) + if w := do(t, r.h, "POST", "/host/poweroff", "A", `{"vmid":9300}`); w.Code != http.StatusForbidden { + t.Fatalf("cross-guest: %d", w.Code) + } + if w := do(t, r.h, "POST", "/host/poweroff", "", ""); w.Code != http.StatusUnauthorized { + t.Fatalf("no token: %d", w.Code) + } + if r.state.Current() != nil || len(r.power.got()) != 0 { + t.Fatal("a refused request changed state") + } +} diff --git a/internal/localapi/server.go b/internal/localapi/server.go index 8819be8..73629eb 100644 --- a/internal/localapi/server.go +++ b/internal/localapi/server.go @@ -394,6 +394,10 @@ type Server struct { // detached pipeline runs through (tests inject; production defaults set in NewServer). netVerifyMu sync.Mutex netVerifyCur *netVerifyJob + + // R-998: POST /host/poweroff (nil = not configured → 503); one guard-and-mark decision at a time. + powerOff *HostPowerOffConfig + powerOffMu sync.Mutex // netTrigger performs the mount-waking directory read through the automount path (Q1). netTrigger func(where string) error // netMounted judges mount success from /proc/mounts — the §8 truth source (never readability). @@ -550,6 +554,8 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /disks/guest-attach", s.withGuest(s.handleDiskGuestAttach)) // Guest reboot (slice 10 P2 activation): user-triggered restart to activate pending drive binds. mux.HandleFunc("POST /guest/reboot", s.withGuest(s.handleGuestReboot)) + // Host power-off (R-998, doc 14): the household's „A gép kikapcsolása". Guarded in hostpoweroff.go. + mux.HandleFunc("POST /host/poweroff", s.withGuest(s.handleHostPowerOff)) // Guest RAM resize (v0.90.0, R-24): read current allocation + bounds, and apply a bounded resize // (live cgroup apply, no reboot). Self-scoped; the agent enforces every bound fresh per request. mux.HandleFunc("GET /guest/memory", s.withGuest(s.handleGuestMemory)) diff --git a/internal/poweroff/state.go b/internal/poweroff/state.go new file mode 100644 index 0000000..0c467bb --- /dev/null +++ b/internal/poweroff/state.go @@ -0,0 +1,117 @@ +// Package poweroff keeps the box's record of a household-requested power-off (R-998, doc 14). +// +// One small JSON file in the agent's state dir. A record belongs to the boot it was made in (the +// kernel's btime from /proc/stat), so: +// - "at most one accepted request per boot": Mark refuses when this boot already has a record; +// - the host report carries the record only while the box is still in that boot (Current), and +// the next boot's reports carry nothing — that absence is how the hub learns the box is back. +// +// Pinned by state_test.go. +package poweroff + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// ErrAlreadyThisBoot is Mark's refusal: a power-off was already accepted in the current boot. +var ErrAlreadyThisBoot = errors.New("a power-off was already accepted in this boot") + +// ErrBootUnknown is Mark's refusal when the boot cannot be identified — fail closed: without it the +// per-boot limit could not hold. +var ErrBootUnknown = errors.New("cannot read the host's boot time") + +// Record is one accepted request. +type Record struct { + RequestedAt time.Time `json:"requested_at"` + BootBtime string `json:"boot_btime"` // /proc/stat btime of the boot it was accepted in + VMID int `json:"vmid"` // the guest whose controller asked +} + +// State is the persisted record. +type State struct { + path string + btime func() string + mu sync.Mutex +} + +// New returns the state at path. btime returns the current boot's /proc/stat btime ("" = unknown). +func New(path string, btime func() string) *State { + if btime == nil { + btime = ProcBtime + } + return &State{path: path, btime: btime} +} + +// Current is the record made in THIS boot, or nil (none, another boot's, unreadable). +func (s *State) Current() *Record { + if s == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + r := s.load() + b := s.btime() + if r == nil || b == "" || r.BootBtime != b { + return nil + } + return r +} + +// Mark records an accepted request for this boot. It refuses a second one in the same boot. +func (s *State) Mark(now time.Time, vmid int) (*Record, error) { + s.mu.Lock() + defer s.mu.Unlock() + b := s.btime() + if b == "" { + return nil, ErrBootUnknown + } + if r := s.load(); r != nil && r.BootBtime == b { + return nil, ErrAlreadyThisBoot + } + r := &Record{RequestedAt: now.UTC().Truncate(time.Second), BootBtime: b, VMID: vmid} + data, _ := json.Marshal(r) + if err := os.MkdirAll(filepath.Dir(s.path), 0o750); err != nil { + return nil, fmt.Errorf("poweroff: state dir: %w", err) + } + tmp := s.path + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return nil, fmt.Errorf("poweroff: write: %w", err) + } + if err := os.Rename(tmp, s.path); err != nil { + return nil, fmt.Errorf("poweroff: rename: %w", err) + } + return r, nil +} + +func (s *State) load() *Record { + data, err := os.ReadFile(s.path) + if err != nil { + return nil + } + var r Record + if json.Unmarshal(data, &r) != nil || r.BootBtime == "" { + return nil + } + return &r +} + +// ProcBtime reads the boot time (seconds since the epoch, as text) from /proc/stat; "" when unreadable. +func ProcBtime() string { + data, err := os.ReadFile("/proc/stat") + if err != nil { + return "" + } + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "btime ") { + return strings.TrimSpace(strings.TrimPrefix(line, "btime ")) + } + } + return "" +} diff --git a/internal/poweroff/state_test.go b/internal/poweroff/state_test.go new file mode 100644 index 0000000..57cbada --- /dev/null +++ b/internal/poweroff/state_test.go @@ -0,0 +1,41 @@ +package poweroff + +import ( + "errors" + "path/filepath" + "testing" + "time" +) + +// One accepted request per boot; the record is "current" only in the boot it was made in. +func TestOnePerBootAndCurrentFollowsTheBoot(t *testing.T) { + boot := "1000" + s := New(filepath.Join(t.TempDir(), "poweroff.json"), func() string { return boot }) + if s.Current() != nil { + t.Fatal("no record yet, Current must be nil") + } + now := time.Date(2026, 10, 13, 18, 0, 0, 0, time.UTC) + if _, err := s.Mark(now, 9201); err != nil { + t.Fatalf("first Mark: %v", err) + } + if r := s.Current(); r == nil || !r.RequestedAt.Equal(now) || r.VMID != 9201 { + t.Fatalf("Current after Mark = %+v", r) + } + if _, err := s.Mark(now.Add(time.Minute), 9201); !errors.Is(err, ErrAlreadyThisBoot) { + t.Fatalf("second Mark in the same boot: err = %v, want ErrAlreadyThisBoot", err) + } + boot = "2000" // the machine was switched on again + if s.Current() != nil { + t.Fatal("a record from the previous boot must not be current") + } + if _, err := s.Mark(now.Add(time.Hour), 9201); err != nil { + t.Fatalf("Mark in a new boot: %v", err) + } +} + +func TestUnknownBootFailsClosed(t *testing.T) { + s := New(filepath.Join(t.TempDir(), "p.json"), func() string { return "" }) + if _, err := s.Mark(time.Now(), 1); !errors.Is(err, ErrBootUnknown) { + t.Fatalf("err = %v, want ErrBootUnknown", err) + } +}