package localapi import ( "context" "encoding/json" "fmt" "io" "log/slog" "net/http" "os" "path/filepath" "regexp" "strconv" "strings" "time" ) // ---- agentic controller update (Phase 1): the agent owns the in-guest controller image SWAP ------- // // The in-guest controller pre-PULLS the target image (its own registry token, shared docker socket), // then asks the agent to swap. The agent — external to the controller container, so it survives the // controller being killed mid-swap — rewrites /etc/felhom-controller-image and restarts the golden's // felhom-controller-bootstrap.service (docker rm -f + docker run), verifies the new controller comes // up healthy, and ROLLS BACK to the previous image if it does not. Latest-only is enforced caller-side // (the controller resolves latest via queryRegistry); here we only accept a strict controller image ref. const ( controllerImageFile = "/etc/felhom-controller-image" bootstrapUnit = "felhom-controller-bootstrap.service" controllerContainer = "felhom-controller" ) // controllerImageRe is the fail-safe gate: ONLY our controller repo + a 3-part semver tag may be run. // A malformed/hostile body can't make the agent docker-run an arbitrary image. var controllerImageRe = regexp.MustCompile(`^gitea\.dooplex\.hu/admin/felhom-controller:[0-9]+\.[0-9]+\.[0-9]+$`) // ValidControllerImage reports whether ref is an acceptable swap target (strict). func ValidControllerImage(ref string) bool { return controllerImageRe.MatchString(ref) } // GuestExecutor runs a command inside a guest (pct exec) and returns stdout. Satisfied by *GuestBinder; // faked in tests. The single seam the swap composes over (no hand-rolled pct). type GuestExecutor interface { GuestExec(ctx context.Context, vmid int, args ...string) (string, error) // GuestExecStdin is GuestExec with the command's stdin fed from stdin — the swap write pipes the // image ref into an in-guest `tee` (no shell vector). GuestExecStdin(ctx context.Context, vmid int, stdin io.Reader, args ...string) (string, error) } // ControllerSwapState is the durable record of a swap (crash-safety + status). Written before the swap // so a mid-swap agent crash leaves the previous image recoverable. type ControllerSwapState struct { VMID int `json:"vmid"` State string `json:"state"` // swapping | done | failed Previous string `json:"previous_image"` Target string `json:"target_image"` Current string `json:"current_image"` // image the file currently names Error string `json:"error,omitempty"` UpdatedAt string `json:"updated_at"` } // ControllerSwapper performs the swap over a GuestExecutor. Stateless except the on-disk state file and // the (test-tunable) verify timing. type ControllerSwapper struct { exec GuestExecutor stateDir string logger *slog.Logger verifyTimeout time.Duration verifyInterval time.Duration verifyDwell int // F1: consecutive ok polls required to accept a NO-healthcheck image (a real // healthcheck passes without dwell). Catches a slow crash-loop that hasn't bumped RestartCount yet. } // NewControllerSwapper builds a swapper. stateDir holds controller-swap-.json (default // /var/lib/felhom-agent when empty). func NewControllerSwapper(exec GuestExecutor, stateDir string, logger *slog.Logger) *ControllerSwapper { if stateDir == "" { stateDir = "/var/lib/felhom-agent" } if logger == nil { logger = slog.Default() } return &ControllerSwapper{ exec: exec, stateDir: stateDir, logger: logger, verifyTimeout: 90 * time.Second, verifyInterval: 3 * time.Second, verifyDwell: 3, } } func (c *ControllerSwapper) statePath(vmid int) string { return filepath.Join(c.stateDir, fmt.Sprintf("controller-swap-%d.json", vmid)) } func (c *ControllerSwapper) saveState(st *ControllerSwapState) { st.UpdatedAt = time.Now().UTC().Format(time.RFC3339) if err := os.MkdirAll(c.stateDir, 0o755); err != nil { c.logger.Error("controller-swap: mkdir state dir", "err", err) return } b, _ := json.MarshalIndent(st, "", " ") tmp := c.statePath(st.VMID) + ".tmp" if err := os.WriteFile(tmp, b, 0o600); err != nil { c.logger.Error("controller-swap: write state", "err", err) return } if err := os.Rename(tmp, c.statePath(st.VMID)); err != nil { c.logger.Error("controller-swap: rename state", "err", err) } } // LoadState returns the recorded swap state for a guest, or nil if none. func (c *ControllerSwapper) LoadState(vmid int) (*ControllerSwapState, error) { b, err := os.ReadFile(c.statePath(vmid)) if os.IsNotExist(err) { return nil, nil } if err != nil { return nil, err } var st ControllerSwapState if err := json.Unmarshal(b, &st); err != nil { return nil, err } return &st, nil } // CurrentImage reads /etc/felhom-controller-image in the guest. func (c *ControllerSwapper) CurrentImage(ctx context.Context, vmid int) (string, error) { out, err := c.exec.GuestExec(ctx, vmid, "cat", controllerImageFile) if err != nil { return "", err } return strings.TrimSpace(out), nil } func (c *ControllerSwapper) imagePresent(ctx context.Context, vmid int, image string) bool { _, err := c.exec.GuestExec(ctx, vmid, "docker", "image", "inspect", image) return err == nil } func (c *ControllerSwapper) writeImage(ctx context.Context, vmid int, image string) error { // Non-root path: pipe the image ref into an in-guest `tee` over stdin — no shell, no // interpolation, no `bash -c` (the only swap vector that would have needed an arbitrary-exec // grant). The trailing "\n" makes the on-disk bytes byte-identical to the golden's // `printf '%s\n'`; the bootstrap reads `IMAGE=$(cat …)` so the newline is stripped on read // (spike SPIKE-controllerswap-narrow-grants-2026-06-29). image is strict-validated // (controllerImageRe) upstream in Swap; defence-in-depth, the stdin path can't smuggle anyway. _, err := c.exec.GuestExecStdin(ctx, vmid, strings.NewReader(image+"\n"), "tee", controllerImageFile) return err } func (c *ControllerSwapper) restartBootstrap(ctx context.Context, vmid int) error { _, err := c.exec.GuestExec(ctx, vmid, "systemctl", "restart", bootstrapUnit) return err } // controllerHealthy is a pure point-in-time predicate: running + RestartCount==0 + (healthy OR no // healthcheck) + the running image == want. Returns (ok, stillStarting, needsDwell): // - stillStarting=true → keep polling (container absent, not-yet-running, wrong image, unhealthy, or // already-restarted). // - needsDwell=true → ok BUT the image has NO healthcheck, so the caller must confirm it stays ok for // verifyDwell consecutive polls before trusting it (F1: a no-healthcheck crash-loop can flicker // Running for one instant). A real `healthy` result is trusted immediately (Docker already gated it). // // RestartCount>0 means the process has already crashed+restarted → not stably up, regardless of // healthcheck presence (the F1 hole: alpine flickered Running between restarts and passed). func (c *ControllerSwapper) controllerHealthy(ctx context.Context, vmid int, want string) (ok, starting, needsDwell bool) { out, err := c.exec.GuestExec(ctx, vmid, "docker", "inspect", "-f", "{{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{.Config.Image}}|{{.RestartCount}}", controllerContainer) if err != nil { return false, true, false // container not there yet (rm -f window) → keep polling } f := strings.SplitN(strings.TrimSpace(out), "|", 4) if len(f) != 4 { return false, true, false } running, health, image, restartStr := f[0] == "true", f[1], f[2], f[3] if !running { return false, true, false } if image != want { return false, true, false // bootstrap may not have re-run yet } if rc, err := strconv.Atoi(strings.TrimSpace(restartStr)); err == nil && rc > 0 { return false, true, false // already crash-restarted → not stably up } switch health { case "healthy": return true, false, false // Docker gated it → trust immediately case "none": return true, false, true // no healthcheck → ok, but require the dwell case "starting": return false, true, false default: // unhealthy return false, true, false } } func (c *ControllerSwapper) verify(ctx context.Context, vmid int, want string) bool { deadline := time.Now().Add(c.verifyTimeout) consecutiveOK := 0 dwell := c.verifyDwell if dwell < 1 { dwell = 1 } for { ok, _, needsDwell := c.controllerHealthy(ctx, vmid, want) switch { case ok && !needsDwell: return true // real healthcheck passed → done case ok: // no-healthcheck image: require verifyDwell consecutive ok polls consecutiveOK++ if consecutiveOK >= dwell { return true } default: consecutiveOK = 0 // any not-ok resets the dwell (a crash between polls) } if time.Now().After(deadline) || ctx.Err() != nil { return false } select { case <-ctx.Done(): return false case <-time.After(c.verifyInterval): } } } // Swap performs the full swap+verify+rollback. Returns the final state. It is synchronous (the HTTP // handler runs it in a goroutine after a 202). Idempotency/single-flight is the caller's. func (c *ControllerSwapper) Swap(ctx context.Context, vmid int, target string) *ControllerSwapState { st := &ControllerSwapState{VMID: vmid, State: "swapping", Target: target} prev, err := c.CurrentImage(ctx, vmid) if err != nil { c.logger.Warn("controller-swap: could not read current image (continuing)", "vmid", vmid, "err", err) } st.Previous, st.Current = prev, prev c.saveState(st) // crash-safety: previous recorded BEFORE any mutation // The controller pre-pulled the image; refuse to swap to an absent image (would brick the guest). if !c.imagePresent(ctx, vmid, target) { st.State = "failed" st.Error = "target image not present in guest (controller did not pre-pull it)" c.saveState(st) c.logger.Error("controller-swap: target image absent — no swap", "vmid", vmid, "target", target) return st } if err := c.writeImage(ctx, vmid, target); err != nil { st.State = "failed" st.Error = "write image file: " + err.Error() c.saveState(st) return st } st.Current = target c.saveState(st) c.logger.Info("controller-swap: image file written, restarting bootstrap", "vmid", vmid, "target", target) if err := c.restartBootstrap(ctx, vmid); err != nil { c.logger.Error("controller-swap: bootstrap restart failed — rolling back", "vmid", vmid, "err", err) return c.rollback(ctx, st, "bootstrap restart failed: "+err.Error()) } if c.verify(ctx, vmid, target) { st.State = "done" st.Error = "" c.saveState(st) c.logger.Info("controller-swap: new controller healthy", "vmid", vmid, "target", target) return st } return c.rollback(ctx, st, "new controller did not become healthy within timeout") } // rollback reverts /etc/felhom-controller-image to the previous image and restarts the unit so the // guest is NEVER left with no controller. Best-effort verify of the previous coming back. func (c *ControllerSwapper) rollback(ctx context.Context, st *ControllerSwapState, reason string) *ControllerSwapState { st.State = "failed" st.Error = reason if st.Previous == "" { c.logger.Error("controller-swap: cannot roll back — no previous image recorded", "vmid", st.VMID) c.saveState(st) return st } c.logger.Warn("controller-swap: rolling back", "vmid", st.VMID, "previous", st.Previous, "reason", reason) if err := c.writeImage(ctx, st.VMID, st.Previous); err != nil { st.Error = reason + "; ALSO rollback write failed: " + err.Error() c.saveState(st) return st } st.Current = st.Previous if err := c.restartBootstrap(ctx, st.VMID); err != nil { st.Error = reason + "; ALSO rollback restart failed: " + err.Error() c.saveState(st) return st } if c.verify(ctx, st.VMID, st.Previous) { c.logger.Info("controller-swap: rolled back to previous, controller healthy", "vmid", st.VMID, "previous", st.Previous) } else { c.logger.Error("controller-swap: rolled back but previous controller not confirmed healthy", "vmid", st.VMID) } c.saveState(st) return st } // ---- HTTP handlers ---------------------------------------------------------------------------------- // handleControllerSwap is POST /controller/swap {image}. Auth+scoping via withGuest. Responds 202 then // swaps async (single-flight per guest). The controller will be killed by the swap, so the agent owns // the rest (verify + rollback) independently. func (s *Server) handleControllerSwap(w http.ResponseWriter, r *http.Request, vmid int) { if s.swap == nil { writeErr(w, http.StatusServiceUnavailable, "controller-swap not configured on this host") return } var req struct { Image string `json:"image"` } if !decodeBody(w, r, &req) { return } target := strings.TrimSpace(req.Image) if !ValidControllerImage(target) { writeErr(w, http.StatusBadRequest, "invalid controller image ref (must be gitea.dooplex.hu/admin/felhom-controller:)") return } s.swapMu.Lock() if s.swapInFlight[vmid] { s.swapMu.Unlock() writeErr(w, http.StatusConflict, "a controller swap is already in progress for this guest") return } s.swapInFlight[vmid] = true s.swapMu.Unlock() // Read the previous image for the immediate response (best-effort, short ctx). pctx, pcancel := context.WithTimeout(context.Background(), 10*time.Second) prev, _ := s.swap.CurrentImage(pctx, vmid) pcancel() base := s.baseCtx if base == nil { base = context.Background() } go func() { defer func() { s.swapMu.Lock() delete(s.swapInFlight, vmid) s.swapMu.Unlock() }() ctx, cancel := context.WithTimeout(base, 5*time.Minute) defer cancel() s.swap.Swap(ctx, vmid, target) }() s.logger.Warn("local-api: controller swap requested", "vmid", vmid, "target", target, "previous", prev) writeStatus(w, http.StatusAccepted, true, map[string]any{ "status": "swapping", "previous_image": prev, "target_image": target, }, "") } // handleControllerSwapStatus is GET /controller/swap/status — the swap outcome for the (new) controller // and the test to read. func (s *Server) handleControllerSwapStatus(w http.ResponseWriter, r *http.Request, vmid int) { if s.swap == nil { writeErr(w, http.StatusServiceUnavailable, "controller-swap not configured on this host") return } s.swapMu.Lock() inflight := s.swapInFlight[vmid] s.swapMu.Unlock() st, err := s.swap.LoadState(vmid) if err != nil { writeErr(w, http.StatusInternalServerError, "read swap state: "+err.Error()) return } if st == nil { writeStatus(w, http.StatusOK, true, map[string]any{"state": "none", "in_flight": inflight}, "") return } writeStatus(w, http.StatusOK, true, map[string]any{ "state": st.State, "in_flight": inflight, "current": st.Current, "previous": st.Previous, "target": st.Target, "error": st.Error, }, "") }