From 6f14b66191593f509680b400195adbd8052f40ad Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Fri, 26 Jun 2026 21:26:12 +0200 Subject: [PATCH] =?UTF-8?q?v0.42.0:=20agentic=20controller=20update=20?= =?UTF-8?q?=E2=80=94=20in-guest=20image=20swap=20+=20rollback=20(Phase=201?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New local-API POST /controller/swap (+ GET /controller/swap/status), withGuest- scoped: the agent records the previous image, confirms the target is present, rewrites /etc/felhom-controller-image, restarts felhom-controller-bootstrap.service, verifies the new controller is healthy (docker inspect, <=90s), and ROLLS BACK to the previous image if not. Single-flight per guest; strict image-ref validation; crash-safety state file. GuestBinder.GuestExec is the pct-exec seam. --selftest=controller-swap exercises it directly. Tests: happy/rollback-on-unhealthy(+red-proof)/image-absent/no-healthcheck/ bad-image-400/single-flight-409. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01TtXesNa2LGbMmE4DNL6SE7 --- CHANGELOG.md | 26 ++ cmd/felhom-agent/main.go | 47 ++- internal/localapi/controllerswap.go | 354 +++++++++++++++++++++++ internal/localapi/controllerswap_test.go | 213 ++++++++++++++ internal/localapi/guestbind.go | 14 + internal/localapi/server.go | 24 +- 6 files changed, 672 insertions(+), 6 deletions(-) create mode 100644 internal/localapi/controllerswap.go create mode 100644 internal/localapi/controllerswap_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7301e77..3615722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,32 @@ All notable changes to **felhom-agent** are recorded here. Update on every code change that gets pushed. +## v0.42.0 — agentic controller update: in-guest image swap + rollback (Phase 1) (2026-06-26) + +The host agent now owns the in-guest controller image **swap** — the new-architecture replacement for +the controller's dead in-container `docker compose` self-update. The controller pre-pulls the target +image (shared docker socket, its own registry token) then asks the agent to swap; the agent — external +to the controller container, so it survives the controller being killed mid-swap — does the rest and +**rolls back** if the new controller doesn't come up healthy. + +- **New local-API routes** (`internal/localapi/controllerswap.go`, token-scoped via `withGuest`): + - `POST /controller/swap {image}` → **202** `{status:"swapping", previous_image, target_image}`, then + async: record previous (crash-safety state file `/var/lib/felhom-agent/controller-swap-.json`) + → confirm the target image is present in the guest (else abort, **no swap**) → write + `/etc/felhom-controller-image` → `systemctl restart felhom-controller-bootstrap.service` → poll the + new controller to **healthy** (`docker inspect`, ≤90s) → **roll back** to the previous image + restart + if it doesn't (the guest is never left without a controller). Single-flight per guest (409 if busy). + Image ref is strict-validated (`gitea.dooplex.hu/admin/felhom-controller:`) before any action. + - `GET /controller/swap/status` → `{state: swapping|done|failed, current, previous, target, error}`. +- **`GuestBinder.GuestExec`** (`internal/localapi/guestbind.go`): the one `pct exec` seam the swap + composes over (cat/inspect/write/restart), reusing the fenced root runner. +- **`--selftest=controller-swap -vmid -image `**: exercise the primitive directly (the target image + must already be pulled in the guest). +- Wired `ControllerSwap: guestBinder` into the local-API server (`cmd/felhom-agent/main.go`). +- Tests (`controllerswap_test.go`): happy swap, **rollback-on-unhealthy** (+ companion red-proof: + dropping the rollback leaves the guest on the bad image and fails the test), image-absent no-swap, + no-healthcheck-running, bad-image 400, single-flight 409. + ## v0.41.0 — provisioned customer guests auto-start after a host reboot (`onboot:1`) (2026-06-24) **F3 fix.** The provision back-half now sets **`onboot:1`** on the customer guest, so after a host diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index c7d3539..dd3eac5 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -44,7 +44,7 @@ import ( // version is the agent version. Overridable at build time with // -ldflags "-X main.version="; defaults to the in-repo CHANGELOG version. -var version = "0.41.0" +var version = "0.42.0" // runGuestHook is the PVE pre-start hook body (`felhom-agent guest-hook `). On the // pre-start phase it creates placeholder dirs for any absent bind-mount source so the guest always boots @@ -104,6 +104,7 @@ func main() { keyDest string idBundlePath string directivePath string + swapImage string showVersion bool ) flag.StringVar(&cfgPath, "config", envOr("FELHOM_AGENT_CONFIG", "/etc/felhom-agent/agent.json"), "path to the agent config file (JSON)") @@ -130,6 +131,7 @@ func main() { flag.StringVar(&directivePath, "directive", "", "for --selftest=escrow-create: a JSON file with the non-secret DR directive (pbs repo/ns, expected fingerprint, tunnel id)") flag.StringVar(&custID, "customer-id", "", "for --selftest=provision: the customer id — the hub config-pull target, baked into the guest's bootstrap") flag.StringVar(&hubPassword, "hub-password", "", "for --selftest=provision: the customer's hub RETRIEVAL PASSPHRASE (SECRET) — baked into bootstrap.json so the controller pulls its config (and the customer-scoped hub key) from the hub. The customer must already exist in the hub.") + flag.StringVar(&swapImage, "image", "", "for --selftest=controller-swap: the target controller image ref (gitea.dooplex.hu/admin/felhom-controller:) — must already be pulled in the guest") flag.StringVar(&custDomain, "customer-domain", "", "for --selftest=provision: customer domain (accepted; used by bring-up only — NOT baked into v2 bootstrap, the hub provides it)") flag.StringVar(&custName, "customer-name", "", "for --selftest=provision: customer display name (accepted; NOT baked into v2 bootstrap)") flag.StringVar(&custEmail, "customer-email", "", "for --selftest=provision: customer email (accepted; NOT baked into v2 bootstrap)") @@ -189,6 +191,8 @@ func main() { os.Exit(runSelftestEscrowConsume(context.Background(), logger, blobPath, expectedFP, keyDest)) case "identity-consume": os.Exit(runSelftestIdentityConsume(context.Background(), logger, blobPath, keyDest)) + case "controller-swap": + os.Exit(runSelftestControllerSwap(context.Background(), cfg, logger, vmid, swapImage)) } } @@ -712,8 +716,9 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St Disks: hostOps, DiskGate: storageGateAdapter{gate: gate, hostID: cfg.Hub.HostID}, Guests2: px, - GuestAttach: guestBinder, // slice 10 P2: bind enrolled data drives into the guest - Intent: intent, // slice 10 P3: record enroll/eject intent for self-heal + GuestAttach: guestBinder, // slice 10 P2: bind enrolled data drives into the guest + ControllerSwap: guestBinder, // Phase 1: agentic controller update — in-guest image swap + Intent: intent, // slice 10 P3: record enroll/eject intent for self-heal GuestBinds: guestBinds, // F9: per-guest bind record for the startup re-assert FormatJobs: formatJobs, // F20-BUG3: detached-format job record + restart recovery @@ -798,6 +803,42 @@ func buildVerifier(cfg config.Config, logger *slog.Logger) (reconcile.OpVerifier return authz.New(signers, store, cfg.Hub.HostID), store, nil } +// runSelftestControllerSwap exercises the agentic controller-update swap primitive directly (Phase 1): +// it swaps guest -vmid's controller to -image, verifies it comes up healthy, and ROLLS BACK if not. +// The target image must already be pulled in the guest (the controller pre-pulls it in the real flow). +func runSelftestControllerSwap(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int, image string) int { + if vmid <= 0 { + fmt.Fprintln(os.Stderr, "selftest=controller-swap: -vmid is required") + return 1 + } + if !localapi.ValidControllerImage(image) { + fmt.Fprintln(os.Stderr, "selftest=controller-swap: -image must be gitea.dooplex.hu/admin/felhom-controller:") + return 1 + } + gaMode := proxmox.RunnerMode(cfg.Privileged.Mode) + if gaMode == "" { + gaMode = proxmox.RunnerSudo + } + binder := localapi.NewGuestBinder(&proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, logger) + swapper := localapi.NewControllerSwapper(binder, "/var/lib/felhom-agent", logger) + + cur, _ := swapper.CurrentImage(ctx, vmid) + fmt.Printf("=== felhom-agent %s selftest=controller-swap (vmid=%d) ===\n", version, vmid) + fmt.Printf(" current image: %s\n target image: %s\n", cur, image) + + st := swapper.Swap(ctx, vmid, image) + fmt.Printf(" --- result ---\n state=%s current=%s previous=%s\n", st.State, st.Current, st.Previous) + if st.Error != "" { + fmt.Printf(" error: %s\n", st.Error) + } + if st.State == "done" { + fmt.Println("=== selftest=controller-swap OK (swapped + healthy) ===") + return 0 + } + fmt.Println("=== selftest=controller-swap FAILED (rolled back if previous existed) ===") + return 1 +} + // runSelftestHub validates hub config, does ONE collect + report, and prints the // report it would send plus the envelope it got back. func runSelftestHub(ctx context.Context, cfg config.Config, logger *slog.Logger) int { diff --git a/internal/localapi/controllerswap.go b/internal/localapi/controllerswap.go new file mode 100644 index 0000000..31b1be4 --- /dev/null +++ b/internal/localapi/controllerswap.go @@ -0,0 +1,354 @@ +package localapi + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// ---- agentic controller update (Phase 1): the agent owns the in-guest controller image SWAP ------- +// +// The in-guest controller pre-PULLS the target image (its own registry token, shared docker socket), +// then asks the agent to swap. The agent — external to the controller container, so it survives the +// controller being killed mid-swap — rewrites /etc/felhom-controller-image and restarts the golden's +// felhom-controller-bootstrap.service (docker rm -f + docker run), verifies the new controller comes +// up healthy, and ROLLS BACK to the previous image if it does not. Latest-only is enforced caller-side +// (the controller resolves latest via queryRegistry); here we only accept a strict controller image ref. + +const ( + controllerImageFile = "/etc/felhom-controller-image" + bootstrapUnit = "felhom-controller-bootstrap.service" + controllerContainer = "felhom-controller" +) + +// controllerImageRe is the fail-safe gate: ONLY our controller repo + a 3-part semver tag may be run. +// A malformed/hostile body can't make the agent docker-run an arbitrary image. +var controllerImageRe = regexp.MustCompile(`^gitea\.dooplex\.hu/admin/felhom-controller:[0-9]+\.[0-9]+\.[0-9]+$`) + +// ValidControllerImage reports whether ref is an acceptable swap target (strict). +func ValidControllerImage(ref string) bool { return controllerImageRe.MatchString(ref) } + +// GuestExecutor runs a command inside a guest (pct exec) and returns stdout. Satisfied by *GuestBinder; +// faked in tests. The single seam the swap composes over (no hand-rolled pct). +type GuestExecutor interface { + GuestExec(ctx context.Context, vmid int, args ...string) (string, error) +} + +// ControllerSwapState is the durable record of a swap (crash-safety + status). Written before the swap +// so a mid-swap agent crash leaves the previous image recoverable. +type ControllerSwapState struct { + VMID int `json:"vmid"` + State string `json:"state"` // swapping | done | failed + Previous string `json:"previous_image"` + Target string `json:"target_image"` + Current string `json:"current_image"` // image the file currently names + Error string `json:"error,omitempty"` + UpdatedAt string `json:"updated_at"` +} + +// ControllerSwapper performs the swap over a GuestExecutor. Stateless except the on-disk state file and +// the (test-tunable) verify timing. +type ControllerSwapper struct { + exec GuestExecutor + stateDir string + logger *slog.Logger + verifyTimeout time.Duration + verifyInterval time.Duration +} + +// NewControllerSwapper builds a swapper. stateDir holds controller-swap-.json (default +// /var/lib/felhom-agent when empty). +func NewControllerSwapper(exec GuestExecutor, stateDir string, logger *slog.Logger) *ControllerSwapper { + if stateDir == "" { + stateDir = "/var/lib/felhom-agent" + } + if logger == nil { + logger = slog.Default() + } + return &ControllerSwapper{ + exec: exec, + stateDir: stateDir, + logger: logger, + verifyTimeout: 90 * time.Second, + verifyInterval: 3 * time.Second, + } +} + +func (c *ControllerSwapper) statePath(vmid int) string { + return filepath.Join(c.stateDir, fmt.Sprintf("controller-swap-%d.json", vmid)) +} + +func (c *ControllerSwapper) saveState(st *ControllerSwapState) { + st.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + if err := os.MkdirAll(c.stateDir, 0o755); err != nil { + c.logger.Error("controller-swap: mkdir state dir", "err", err) + return + } + b, _ := json.MarshalIndent(st, "", " ") + tmp := c.statePath(st.VMID) + ".tmp" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + c.logger.Error("controller-swap: write state", "err", err) + return + } + if err := os.Rename(tmp, c.statePath(st.VMID)); err != nil { + c.logger.Error("controller-swap: rename state", "err", err) + } +} + +// LoadState returns the recorded swap state for a guest, or nil if none. +func (c *ControllerSwapper) LoadState(vmid int) (*ControllerSwapState, error) { + b, err := os.ReadFile(c.statePath(vmid)) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var st ControllerSwapState + if err := json.Unmarshal(b, &st); err != nil { + return nil, err + } + return &st, nil +} + +// CurrentImage reads /etc/felhom-controller-image in the guest. +func (c *ControllerSwapper) CurrentImage(ctx context.Context, vmid int) (string, error) { + out, err := c.exec.GuestExec(ctx, vmid, "cat", controllerImageFile) + if err != nil { + return "", err + } + return strings.TrimSpace(out), nil +} + +func (c *ControllerSwapper) imagePresent(ctx context.Context, vmid int, image string) bool { + _, err := c.exec.GuestExec(ctx, vmid, "docker", "image", "inspect", image) + return err == nil +} + +func (c *ControllerSwapper) writeImage(ctx context.Context, vmid int, image string) error { + // image is strict-validated (controllerImageRe) before we ever get here, so this single-quoted + // interpolation cannot smuggle shell metacharacters. + cmd := fmt.Sprintf("printf '%%s\\n' '%s' > %s", image, controllerImageFile) + _, err := c.exec.GuestExec(ctx, vmid, "bash", "-c", cmd) + return err +} + +func (c *ControllerSwapper) restartBootstrap(ctx context.Context, vmid int) error { + _, err := c.exec.GuestExec(ctx, vmid, "systemctl", "restart", bootstrapUnit) + return err +} + +// controllerHealthy reports running + (healthy or no healthcheck) + the running image == want. +// Returns (ok, stillStarting): stillStarting=true means keep polling. +func (c *ControllerSwapper) controllerHealthy(ctx context.Context, vmid int, want string) (ok, starting bool) { + out, err := c.exec.GuestExec(ctx, vmid, "docker", "inspect", "-f", + "{{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{.Config.Image}}", controllerContainer) + if err != nil { + return false, true // container not there yet (rm -f window) → keep polling + } + f := strings.SplitN(strings.TrimSpace(out), "|", 3) + if len(f) != 3 { + return false, true + } + running, health, image := f[0] == "true", f[1], f[2] + if !running { + return false, true + } + if image != want { + return false, true // bootstrap may not have re-run yet + } + switch health { + case "healthy", "none": + return true, false + case "starting": + return false, true + default: // unhealthy + return false, true + } +} + +func (c *ControllerSwapper) verify(ctx context.Context, vmid int, want string) bool { + deadline := time.Now().Add(c.verifyTimeout) + for { + if ok, _ := c.controllerHealthy(ctx, vmid, want); ok { + return true + } + if time.Now().After(deadline) || ctx.Err() != nil { + return false + } + select { + case <-ctx.Done(): + return false + case <-time.After(c.verifyInterval): + } + } +} + +// Swap performs the full swap+verify+rollback. Returns the final state. It is synchronous (the HTTP +// handler runs it in a goroutine after a 202). Idempotency/single-flight is the caller's. +func (c *ControllerSwapper) Swap(ctx context.Context, vmid int, target string) *ControllerSwapState { + st := &ControllerSwapState{VMID: vmid, State: "swapping", Target: target} + prev, err := c.CurrentImage(ctx, vmid) + if err != nil { + c.logger.Warn("controller-swap: could not read current image (continuing)", "vmid", vmid, "err", err) + } + st.Previous, st.Current = prev, prev + c.saveState(st) // crash-safety: previous recorded BEFORE any mutation + + // The controller pre-pulled the image; refuse to swap to an absent image (would brick the guest). + if !c.imagePresent(ctx, vmid, target) { + st.State = "failed" + st.Error = "target image not present in guest (controller did not pre-pull it)" + c.saveState(st) + c.logger.Error("controller-swap: target image absent — no swap", "vmid", vmid, "target", target) + return st + } + + if err := c.writeImage(ctx, vmid, target); err != nil { + st.State = "failed" + st.Error = "write image file: " + err.Error() + c.saveState(st) + return st + } + st.Current = target + c.saveState(st) + c.logger.Info("controller-swap: image file written, restarting bootstrap", "vmid", vmid, "target", target) + if err := c.restartBootstrap(ctx, vmid); err != nil { + c.logger.Error("controller-swap: bootstrap restart failed — rolling back", "vmid", vmid, "err", err) + return c.rollback(ctx, st, "bootstrap restart failed: "+err.Error()) + } + + if c.verify(ctx, vmid, target) { + st.State = "done" + st.Error = "" + c.saveState(st) + c.logger.Info("controller-swap: new controller healthy", "vmid", vmid, "target", target) + return st + } + return c.rollback(ctx, st, "new controller did not become healthy within timeout") +} + +// rollback reverts /etc/felhom-controller-image to the previous image and restarts the unit so the +// guest is NEVER left with no controller. Best-effort verify of the previous coming back. +func (c *ControllerSwapper) rollback(ctx context.Context, st *ControllerSwapState, reason string) *ControllerSwapState { + st.State = "failed" + st.Error = reason + if st.Previous == "" { + c.logger.Error("controller-swap: cannot roll back — no previous image recorded", "vmid", st.VMID) + c.saveState(st) + return st + } + c.logger.Warn("controller-swap: rolling back", "vmid", st.VMID, "previous", st.Previous, "reason", reason) + if err := c.writeImage(ctx, st.VMID, st.Previous); err != nil { + st.Error = reason + "; ALSO rollback write failed: " + err.Error() + c.saveState(st) + return st + } + st.Current = st.Previous + if err := c.restartBootstrap(ctx, st.VMID); err != nil { + st.Error = reason + "; ALSO rollback restart failed: " + err.Error() + c.saveState(st) + return st + } + if c.verify(ctx, st.VMID, st.Previous) { + c.logger.Info("controller-swap: rolled back to previous, controller healthy", "vmid", st.VMID, "previous", st.Previous) + } else { + c.logger.Error("controller-swap: rolled back but previous controller not confirmed healthy", "vmid", st.VMID) + } + c.saveState(st) + return st +} + +// ---- HTTP handlers ---------------------------------------------------------------------------------- + +// handleControllerSwap is POST /controller/swap {image}. Auth+scoping via withGuest. Responds 202 then +// swaps async (single-flight per guest). The controller will be killed by the swap, so the agent owns +// the rest (verify + rollback) independently. +func (s *Server) handleControllerSwap(w http.ResponseWriter, r *http.Request, vmid int) { + if s.swap == nil { + writeErr(w, http.StatusServiceUnavailable, "controller-swap not configured on this host") + return + } + var req struct { + Image string `json:"image"` + } + if !decodeBody(w, r, &req) { + return + } + target := strings.TrimSpace(req.Image) + if !ValidControllerImage(target) { + writeErr(w, http.StatusBadRequest, "invalid controller image ref (must be gitea.dooplex.hu/admin/felhom-controller:)") + return + } + + s.swapMu.Lock() + if s.swapInFlight[vmid] { + s.swapMu.Unlock() + writeErr(w, http.StatusConflict, "a controller swap is already in progress for this guest") + return + } + s.swapInFlight[vmid] = true + s.swapMu.Unlock() + + // Read the previous image for the immediate response (best-effort, short ctx). + pctx, pcancel := context.WithTimeout(context.Background(), 10*time.Second) + prev, _ := s.swap.CurrentImage(pctx, vmid) + pcancel() + + base := s.baseCtx + if base == nil { + base = context.Background() + } + go func() { + defer func() { + s.swapMu.Lock() + delete(s.swapInFlight, vmid) + s.swapMu.Unlock() + }() + ctx, cancel := context.WithTimeout(base, 5*time.Minute) + defer cancel() + s.swap.Swap(ctx, vmid, target) + }() + + s.logger.Warn("local-api: controller swap requested", "vmid", vmid, "target", target, "previous", prev) + writeStatus(w, http.StatusAccepted, true, map[string]any{ + "status": "swapping", "previous_image": prev, "target_image": target, + }, "") +} + +// handleControllerSwapStatus is GET /controller/swap/status — the swap outcome for the (new) controller +// and the test to read. +func (s *Server) handleControllerSwapStatus(w http.ResponseWriter, r *http.Request, vmid int) { + if s.swap == nil { + writeErr(w, http.StatusServiceUnavailable, "controller-swap not configured on this host") + return + } + s.swapMu.Lock() + inflight := s.swapInFlight[vmid] + s.swapMu.Unlock() + + st, err := s.swap.LoadState(vmid) + if err != nil { + writeErr(w, http.StatusInternalServerError, "read swap state: "+err.Error()) + return + } + if st == nil { + writeStatus(w, http.StatusOK, true, map[string]any{"state": "none", "in_flight": inflight}, "") + return + } + writeStatus(w, http.StatusOK, true, map[string]any{ + "state": st.State, + "in_flight": inflight, + "current": st.Current, + "previous": st.Previous, + "target": st.Target, + "error": st.Error, + }, "") +} diff --git a/internal/localapi/controllerswap_test.go b/internal/localapi/controllerswap_test.go new file mode 100644 index 0000000..2583901 --- /dev/null +++ b/internal/localapi/controllerswap_test.go @@ -0,0 +1,213 @@ +package localapi + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http/httptest" + "regexp" + "strings" + "sync" + "testing" + "time" +) + +func discardLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } + +// fakeGuestExec simulates `pct exec` into a guest for the swap primitive. It models the image file, +// which images are present (pulled), which come up healthy, and the running container's image. +type fakeGuestExec struct { + mu sync.Mutex + calls [][]string + imageFile string // /etc/felhom-controller-image content + present map[string]bool // images pulled into the guest + good map[string]bool // images that report healthy when running + containerImg string // image the running container currently has + failRestart bool + noHealthBlock bool // if set, .State.Health is absent ("none") +} + +var writeImgRe = regexp.MustCompile(`'([^']+)' >`) + +func (f *fakeGuestExec) GuestExec(_ context.Context, _ int, args ...string) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, args) + switch { + case len(args) >= 2 && args[0] == "cat" && args[1] == controllerImageFile: + return f.imageFile + "\n", nil + case len(args) >= 4 && args[0] == "docker" && args[1] == "image" && args[2] == "inspect": + if f.present[args[3]] { + return args[3], nil + } + return "", fmt.Errorf("no such image: %s", args[3]) + case len(args) >= 3 && args[0] == "bash" && args[1] == "-c": + m := writeImgRe.FindStringSubmatch(args[2]) + if m == nil { + return "", fmt.Errorf("fake: unparseable write cmd %q", args[2]) + } + f.imageFile = m[1] + return "", nil + case len(args) >= 3 && args[0] == "systemctl" && args[1] == "restart": + if f.failRestart { + return "", fmt.Errorf("fake: systemctl restart failed") + } + f.containerImg = f.imageFile // bootstrap re-ran: container now runs the file's image + return "", nil + case len(args) >= 2 && args[0] == "docker" && args[1] == "inspect": + if f.containerImg == "" { + return "", fmt.Errorf("no such container") + } + health := "healthy" + if f.noHealthBlock { + health = "none" + } else if !f.good[f.containerImg] { + health = "unhealthy" + } + return fmt.Sprintf("true|%s|%s", health, f.containerImg), nil + } + return "", fmt.Errorf("fake: unexpected exec %v", args) +} + +func (f *fakeGuestExec) wrote(image string) bool { + f.mu.Lock() + defer f.mu.Unlock() + for _, c := range f.calls { + if len(c) >= 3 && c[0] == "bash" && strings.Contains(c[2], "'"+image+"'") { + return true + } + } + return false +} + +func newTestSwapper(t *testing.T, fe *fakeGuestExec) *ControllerSwapper { + t.Helper() + s := NewControllerSwapper(fe, t.TempDir(), nil) + s.verifyTimeout = 200 * time.Millisecond + s.verifyInterval = 5 * time.Millisecond + return s +} + +const ( + prevImg = "gitea.dooplex.hu/admin/felhom-controller:0.77.0" + newImg = "gitea.dooplex.hu/admin/felhom-controller:0.84.0" +) + +func TestControllerSwap_Happy(t *testing.T) { + fe := &fakeGuestExec{ + imageFile: prevImg, + present: map[string]bool{newImg: true}, + good: map[string]bool{newImg: true, prevImg: true}, + } + s := newTestSwapper(t, fe) + st := s.Swap(context.Background(), 9201, newImg) + + if st.State != "done" { + t.Fatalf("state = %q (err=%q), want done", st.State, st.Error) + } + if fe.imageFile != newImg { + t.Errorf("image file = %q, want %q", fe.imageFile, newImg) + } + if st.Previous != prevImg { + t.Errorf("previous = %q, want %q", st.Previous, prevImg) + } + if !fe.wrote(newImg) { + t.Errorf("expected the new image to be written") + } + // state file persisted + got, _ := s.LoadState(9201) + if got == nil || got.State != "done" { + t.Errorf("state file = %+v, want done", got) + } +} + +// The load-bearing failure path: an unhealthy target must roll back to the previous image. +func TestControllerSwap_RollbackOnUnhealthy(t *testing.T) { + fe := &fakeGuestExec{ + imageFile: prevImg, + present: map[string]bool{newImg: true}, + good: map[string]bool{prevImg: true}, // newImg present but NEVER healthy + } + s := newTestSwapper(t, fe) + st := s.Swap(context.Background(), 9201, newImg) + + if st.State != "failed" { + t.Fatalf("state = %q, want failed", st.State) + } + if fe.imageFile != prevImg { + t.Errorf("image file = %q after rollback, want previous %q (guest left on bad image!)", fe.imageFile, prevImg) + } + if fe.containerImg != prevImg { + t.Errorf("running container = %q after rollback, want %q", fe.containerImg, prevImg) + } + // restart called at least twice (swap + rollback) + restarts := 0 + for _, c := range fe.calls { + if len(c) >= 2 && c[0] == "systemctl" && c[1] == "restart" { + restarts++ + } + } + if restarts < 2 { + t.Errorf("systemctl restart called %d times, want ≥2 (swap + rollback)", restarts) + } +} + +func TestControllerSwap_ImageAbsent_NoSwap(t *testing.T) { + fe := &fakeGuestExec{ + imageFile: prevImg, + present: map[string]bool{}, // target NOT pulled + good: map[string]bool{prevImg: true}, + } + s := newTestSwapper(t, fe) + st := s.Swap(context.Background(), 9201, newImg) + + if st.State != "failed" { + t.Fatalf("state = %q, want failed", st.State) + } + if fe.imageFile != prevImg { + t.Errorf("image file = %q, want unchanged %q (no swap on absent image)", fe.imageFile, prevImg) + } + if fe.wrote(newImg) { + t.Errorf("must NOT write the image file when the target image is absent") + } +} + +func TestControllerSwap_HealthyWithNoHealthcheck(t *testing.T) { + fe := &fakeGuestExec{ + imageFile: prevImg, + present: map[string]bool{newImg: true}, + noHealthBlock: true, // image defines no HEALTHCHECK → running is enough + } + s := newTestSwapper(t, fe) + st := s.Swap(context.Background(), 9201, newImg) + if st.State != "done" { + t.Fatalf("state = %q, want done (running + no healthcheck)", st.State) + } +} + +func TestControllerSwapHandler_BadImage(t *testing.T) { + fe := &fakeGuestExec{present: map[string]bool{}} + s := &Server{swap: newTestSwapper(t, fe), swapInFlight: map[int]bool{}, logger: discardLogger()} + body := `{"image":"docker.io/evil/runme:latest"}` + req := httptest.NewRequest("POST", "/controller/swap", strings.NewReader(body)) + rr := httptest.NewRecorder() + s.handleControllerSwap(rr, req, 9201) + if rr.Code != 400 { + t.Fatalf("status = %d, want 400 for a non-controller image", rr.Code) + } + if len(fe.calls) != 0 { + t.Errorf("no guest exec should run for a rejected image, got %v", fe.calls) + } +} + +func TestControllerSwapHandler_SingleFlight(t *testing.T) { + fe := &fakeGuestExec{present: map[string]bool{}} + s := &Server{swap: newTestSwapper(t, fe), swapInFlight: map[int]bool{9201: true}, logger: discardLogger()} + req := httptest.NewRequest("POST", "/controller/swap", strings.NewReader(`{"image":"`+newImg+`"}`)) + rr := httptest.NewRecorder() + s.handleControllerSwap(rr, req, 9201) + if rr.Code != 409 { + t.Fatalf("status = %d, want 409 (swap already in progress)", rr.Code) + } +} diff --git a/internal/localapi/guestbind.go b/internal/localapi/guestbind.go index cffd6ef..b0e88fc 100644 --- a/internal/localapi/guestbind.go +++ b/internal/localapi/guestbind.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "strconv" + "strings" "sync" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" @@ -110,3 +111,16 @@ func (b *GuestBinder) run(ctx context.Context, name string, args ...string) erro } return nil } + +// GuestExec runs a command INSIDE the guest via `pct exec -- ` and returns its stdout. +// It is the single guest-exec seam the controller-swap primitive composes over (cat the image file, +// docker image-inspect, write the file, systemctl restart, docker inspect health) — reusing the same +// fenced root runner as the bind ops rather than hand-rolling pct. The guest must be running. +func (b *GuestBinder) GuestExec(ctx context.Context, vmid int, args ...string) (string, error) { + pctArgs := append([]string{"exec", strconv.Itoa(vmid), "--"}, args...) + out, stderr, err := b.runner.Run(ctx, "pct", pctArgs...) + if err != nil { + return string(out), fmt.Errorf("pct exec %d %v: %w: %s", vmid, args, err, strings.TrimSpace(string(stderr))) + } + return string(out), nil +} diff --git a/internal/localapi/server.go b/internal/localapi/server.go index 295e065..8b835b0 100644 --- a/internal/localapi/server.go +++ b/internal/localapi/server.go @@ -84,6 +84,11 @@ type Options struct { // GuestAttach binds an enrolled user-data drive's felhom-data namespace into the guest (slice 10 // P2, Model A). OPTIONAL — when nil, POST /disks/guest-attach reports "not configured". GuestAttach GuestAttacher + // ControllerSwap runs guest commands (pct exec) for the agentic controller-update swap (Phase 1). + // OPTIONAL — when nil, POST /controller/swap reports "not configured". Satisfied by *GuestBinder. + ControllerSwap GuestExecutor + // ControllerSwapStateDir holds the per-guest swap state file (crash-safety). "" → /var/lib/felhom-agent. + ControllerSwapStateDir string // Intent records drive enroll/eject intent for the self-heal watchdog (slice 10 P3). OPTIONAL — // when nil, no intent is recorded (self-heal runs ungated). Intent IntentRecorder @@ -178,6 +183,11 @@ type Server struct { jobsMu sync.Mutex jobs map[int]*backupJob // per-guest backup job state (slice 8B) + // agentic controller update (Phase 1): the swapper + per-guest single-flight gate. + swap *ControllerSwapper + swapMu sync.Mutex + swapInFlight map[int]bool + baseCtx context.Context // for fire-and-forget backups; set in Run } @@ -215,12 +225,16 @@ func NewServer(o Options) (*Server, error) { guestBinds: o.GuestBinds, formatJobs: o.FormatJobs, host: o.HostReader, - hostMetrics: o.HostMetrics, - hostID: o.HostID, - jobs: map[int]*backupJob{}, + hostMetrics: o.HostMetrics, + hostID: o.HostID, + jobs: map[int]*backupJob{}, + swapInFlight: map[int]bool{}, } s.reresolveWipe = s.reresolveDurableForWipe s.deviceDurableID = storage.DeviceDurableID + if o.ControllerSwap != nil { + s.swap = NewControllerSwapper(o.ControllerSwap, o.ControllerSwapStateDir, o.Logger) + } return s, nil } @@ -248,6 +262,10 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /disks/guest-attach", s.withGuest(s.handleDiskGuestAttach)) // Guest reboot (slice 10 P2 activation): user-triggered restart to activate pending drive binds. mux.HandleFunc("POST /guest/reboot", s.withGuest(s.handleGuestReboot)) + + // agentic controller update (Phase 1): in-guest image swap + rollback, owned by the agent. + mux.HandleFunc("POST /controller/swap", s.withGuest(s.handleControllerSwap)) + mux.HandleFunc("GET /controller/swap/status", s.withGuest(s.handleControllerSwapStatus)) return mux }