felhom-agent/internal/localapi/controllerswap.go

package localapi

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// ---- agentic controller update (Phase 1): the agent owns the in-guest controller image SWAP -------
//
// The in-guest controller pre-PULLS the target image (its own registry token, shared docker socket),
// then asks the agent to swap. The agent — external to the controller container, so it survives the
// controller being killed mid-swap — rewrites /etc/felhom-controller-image and restarts the golden's
// felhom-controller-bootstrap.service (docker rm -f + docker run), verifies the new controller comes
// up healthy, and ROLLS BACK to the previous image if it does not. Latest-only is enforced caller-side
// (the controller resolves latest via queryRegistry); here we only accept a strict controller image ref.

const (
	controllerImageFile = "/etc/felhom-controller-image"
	bootstrapUnit       = "felhom-controller-bootstrap.service"
	controllerContainer = "felhom-controller"
)

// controllerImageRe is the fail-safe gate: ONLY our controller repo + a 3-part semver tag may be run.
// A malformed/hostile body can't make the agent docker-run an arbitrary image.
var controllerImageRe = regexp.MustCompile(`^gitea\.dooplex\.hu/admin/felhom-controller:[0-9]+\.[0-9]+\.[0-9]+$`)

// ValidControllerImage reports whether ref is an acceptable swap target (strict).
func ValidControllerImage(ref string) bool { return controllerImageRe.MatchString(ref) }

// GuestExecutor runs a command inside a guest (pct exec) and returns stdout. Satisfied by *GuestBinder;
// faked in tests. The single seam the swap composes over (no hand-rolled pct).
type GuestExecutor interface {
	GuestExec(ctx context.Context, vmid int, args ...string) (string, error)
	// GuestExecStdin is GuestExec with the command's stdin fed from stdin — the swap write pipes the
	// image ref into an in-guest `tee` (no shell vector).
	GuestExecStdin(ctx context.Context, vmid int, stdin io.Reader, args ...string) (string, error)
}

// ControllerSwapState is the durable record of a swap (crash-safety + status). Written before the swap
// so a mid-swap agent crash leaves the previous image recoverable.
type ControllerSwapState struct {
	VMID      int    `json:"vmid"`
	State     string `json:"state"` // swapping | done | failed
	Previous  string `json:"previous_image"`
	Target    string `json:"target_image"`
	Current   string `json:"current_image"` // image the file currently names
	Error     string `json:"error,omitempty"`
	UpdatedAt string `json:"updated_at"`
}

// ControllerSwapper performs the swap over a GuestExecutor. Stateless except the on-disk state file and
// the (test-tunable) verify timing.
type ControllerSwapper struct {
	exec           GuestExecutor
	stateDir       string
	logger         *slog.Logger
	verifyTimeout  time.Duration
	verifyInterval time.Duration
	verifyDwell    int // F1: consecutive ok polls required to accept a NO-healthcheck image (a real
	// healthcheck passes without dwell). Catches a slow crash-loop that hasn't bumped RestartCount yet.
}

// NewControllerSwapper builds a swapper. stateDir holds controller-swap-<vmid>.json (default
// /var/lib/felhom-agent when empty).
func NewControllerSwapper(exec GuestExecutor, stateDir string, logger *slog.Logger) *ControllerSwapper {
	if stateDir == "" {
		stateDir = "/var/lib/felhom-agent"
	}
	if logger == nil {
		logger = slog.Default()
	}
	return &ControllerSwapper{
		exec:           exec,
		stateDir:       stateDir,
		logger:         logger,
		verifyTimeout:  90 * time.Second,
		verifyInterval: 3 * time.Second,
		verifyDwell:    3,
	}
}

func (c *ControllerSwapper) statePath(vmid int) string {
	return filepath.Join(c.stateDir, fmt.Sprintf("controller-swap-%d.json", vmid))
}

func (c *ControllerSwapper) saveState(st *ControllerSwapState) {
	st.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
	if err := os.MkdirAll(c.stateDir, 0o755); err != nil {
		c.logger.Error("controller-swap: mkdir state dir", "err", err)
		return
	}
	b, _ := json.MarshalIndent(st, "", "  ")
	tmp := c.statePath(st.VMID) + ".tmp"
	if err := os.WriteFile(tmp, b, 0o600); err != nil {
		c.logger.Error("controller-swap: write state", "err", err)
		return
	}
	if err := os.Rename(tmp, c.statePath(st.VMID)); err != nil {
		c.logger.Error("controller-swap: rename state", "err", err)
	}
}

// LoadState returns the recorded swap state for a guest, or nil if none.
func (c *ControllerSwapper) LoadState(vmid int) (*ControllerSwapState, error) {
	b, err := os.ReadFile(c.statePath(vmid))
	if os.IsNotExist(err) {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}
	var st ControllerSwapState
	if err := json.Unmarshal(b, &st); err != nil {
		return nil, err
	}
	return &st, nil
}

// CurrentImage reads /etc/felhom-controller-image in the guest.
func (c *ControllerSwapper) CurrentImage(ctx context.Context, vmid int) (string, error) {
	out, err := c.exec.GuestExec(ctx, vmid, "cat", controllerImageFile)
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(out), nil
}

func (c *ControllerSwapper) imagePresent(ctx context.Context, vmid int, image string) bool {
	_, err := c.exec.GuestExec(ctx, vmid, "docker", "image", "inspect", image)
	return err == nil
}

func (c *ControllerSwapper) writeImage(ctx context.Context, vmid int, image string) error {
	// Non-root path: pipe the image ref into an in-guest `tee` over stdin — no shell, no
	// interpolation, no `bash -c` (the only swap vector that would have needed an arbitrary-exec
	// grant). The trailing "\n" makes the on-disk bytes byte-identical to the golden's
	// `printf '%s\n'`; the bootstrap reads `IMAGE=$(cat …)` so the newline is stripped on read
	// (spike SPIKE-controllerswap-narrow-grants-2026-06-29). image is strict-validated
	// (controllerImageRe) upstream in Swap; defence-in-depth, the stdin path can't smuggle anyway.
	_, err := c.exec.GuestExecStdin(ctx, vmid, strings.NewReader(image+"\n"), "tee", controllerImageFile)
	return err
}

func (c *ControllerSwapper) restartBootstrap(ctx context.Context, vmid int) error {
	_, err := c.exec.GuestExec(ctx, vmid, "systemctl", "restart", bootstrapUnit)
	return err
}

// controllerHealthy is a pure point-in-time predicate: running + RestartCount==0 + (healthy OR no
// healthcheck) + the running image == want. Returns (ok, stillStarting, needsDwell):
//   - stillStarting=true → keep polling (container absent, not-yet-running, wrong image, unhealthy, or
//     already-restarted).
//   - needsDwell=true → ok BUT the image has NO healthcheck, so the caller must confirm it stays ok for
//     verifyDwell consecutive polls before trusting it (F1: a no-healthcheck crash-loop can flicker
//     Running for one instant). A real `healthy` result is trusted immediately (Docker already gated it).
//
// RestartCount>0 means the process has already crashed+restarted → not stably up, regardless of
// healthcheck presence (the F1 hole: alpine flickered Running between restarts and passed).
func (c *ControllerSwapper) controllerHealthy(ctx context.Context, vmid int, want string) (ok, starting, needsDwell bool) {
	out, err := c.exec.GuestExec(ctx, vmid, "docker", "inspect", "-f",
		"{{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{.Config.Image}}|{{.RestartCount}}", controllerContainer)
	if err != nil {
		return false, true, false // container not there yet (rm -f window) → keep polling
	}
	f := strings.SplitN(strings.TrimSpace(out), "|", 4)
	if len(f) != 4 {
		return false, true, false
	}
	running, health, image, restartStr := f[0] == "true", f[1], f[2], f[3]
	if !running {
		return false, true, false
	}
	if image != want {
		return false, true, false // bootstrap may not have re-run yet
	}
	if rc, err := strconv.Atoi(strings.TrimSpace(restartStr)); err == nil && rc > 0 {
		return false, true, false // already crash-restarted → not stably up
	}
	switch health {
	case "healthy":
		return true, false, false // Docker gated it → trust immediately
	case "none":
		return true, false, true // no healthcheck → ok, but require the dwell
	case "starting":
		return false, true, false
	default: // unhealthy
		return false, true, false
	}
}

func (c *ControllerSwapper) verify(ctx context.Context, vmid int, want string) bool {
	deadline := time.Now().Add(c.verifyTimeout)
	consecutiveOK := 0
	dwell := c.verifyDwell
	if dwell < 1 {
		dwell = 1
	}
	for {
		ok, _, needsDwell := c.controllerHealthy(ctx, vmid, want)
		switch {
		case ok && !needsDwell:
			return true // real healthcheck passed → done
		case ok: // no-healthcheck image: require verifyDwell consecutive ok polls
			consecutiveOK++
			if consecutiveOK >= dwell {
				return true
			}
		default:
			consecutiveOK = 0 // any not-ok resets the dwell (a crash between polls)
		}
		if time.Now().After(deadline) || ctx.Err() != nil {
			return false
		}
		select {
		case <-ctx.Done():
			return false
		case <-time.After(c.verifyInterval):
		}
	}
}

// Swap performs the full swap+verify+rollback. Returns the final state. It is synchronous (the HTTP
// handler runs it in a goroutine after a 202). Idempotency/single-flight is the caller's.
func (c *ControllerSwapper) Swap(ctx context.Context, vmid int, target string) *ControllerSwapState {
	st := &ControllerSwapState{VMID: vmid, State: "swapping", Target: target}
	prev, err := c.CurrentImage(ctx, vmid)
	if err != nil {
		c.logger.Warn("controller-swap: could not read current image (continuing)", "vmid", vmid, "err", err)
	}
	st.Previous, st.Current = prev, prev
	c.saveState(st) // crash-safety: previous recorded BEFORE any mutation

	// The controller pre-pulled the image; refuse to swap to an absent image (would brick the guest).
	c.logger.Debug("controller-swap: pre-pull verify", "vmid", vmid, "target", target, "previous", prev)
	if !c.imagePresent(ctx, vmid, target) {
		st.State = "failed"
		st.Error = "target image not present in guest (controller did not pre-pull it)"
		c.saveState(st)
		c.logger.Error("controller-swap: target image absent — no swap", "vmid", vmid, "target", target)
		return st
	}

	if err := c.writeImage(ctx, vmid, target); err != nil {
		st.State = "failed"
		st.Error = "write image file: " + err.Error()
		c.saveState(st)
		return st
	}
	st.Current = target
	c.saveState(st)
	c.logger.Info("controller-swap: image file written, restarting bootstrap", "vmid", vmid, "target", target)
	if err := c.restartBootstrap(ctx, vmid); err != nil {
		c.logger.Error("controller-swap: bootstrap restart failed — rolling back", "vmid", vmid, "err", err)
		return c.rollback(ctx, st, "bootstrap restart failed: "+err.Error())
	}

	if c.verify(ctx, vmid, target) {
		st.State = "done"
		st.Error = ""
		c.saveState(st)
		c.logger.Info("controller-swap: new controller healthy", "vmid", vmid, "target", target)
		return st
	}
	c.logger.Warn("controller-swap: health verdict negative — rolling back", "vmid", vmid, "target", target)
	return c.rollback(ctx, st, "new controller did not become healthy within timeout")
}

// rollback reverts /etc/felhom-controller-image to the previous image and restarts the unit so the
// guest is NEVER left with no controller. Best-effort verify of the previous coming back.
func (c *ControllerSwapper) rollback(ctx context.Context, st *ControllerSwapState, reason string) *ControllerSwapState {
	st.State = "failed"
	st.Error = reason
	if st.Previous == "" {
		c.logger.Error("controller-swap: cannot roll back — no previous image recorded", "vmid", st.VMID)
		c.saveState(st)
		return st
	}
	c.logger.Warn("controller-swap: rolling back", "vmid", st.VMID, "previous", st.Previous, "reason", reason)
	if err := c.writeImage(ctx, st.VMID, st.Previous); err != nil {
		st.Error = reason + "; ALSO rollback write failed: " + err.Error()
		c.saveState(st)
		return st
	}
	st.Current = st.Previous
	if err := c.restartBootstrap(ctx, st.VMID); err != nil {
		st.Error = reason + "; ALSO rollback restart failed: " + err.Error()
		c.saveState(st)
		return st
	}
	if c.verify(ctx, st.VMID, st.Previous) {
		c.logger.Info("controller-swap: rolled back to previous, controller healthy", "vmid", st.VMID, "previous", st.Previous)
	} else {
		c.logger.Error("controller-swap: rolled back but previous controller not confirmed healthy", "vmid", st.VMID)
	}
	c.saveState(st)
	return st
}

// ---- HTTP handlers ----------------------------------------------------------------------------------

// handleControllerSwap is POST /controller/swap {image}. Auth+scoping via withGuest. Responds 202 then
// swaps async (single-flight per guest). The controller will be killed by the swap, so the agent owns
// the rest (verify + rollback) independently.
func (s *Server) handleControllerSwap(w http.ResponseWriter, r *http.Request, vmid int) {
	if s.swap == nil {
		writeErr(w, http.StatusServiceUnavailable, "controller-swap not configured on this host")
		return
	}
	var req struct {
		Image string `json:"image"`
	}
	if !decodeBody(w, r, &req) {
		return
	}
	target := strings.TrimSpace(req.Image)
	if !ValidControllerImage(target) {
		writeErr(w, http.StatusBadRequest, "invalid controller image ref (must be gitea.dooplex.hu/admin/felhom-controller:<semver>)")
		return
	}

	s.swapMu.Lock()
	if s.swapInFlight[vmid] {
		s.swapMu.Unlock()
		writeErr(w, http.StatusConflict, "a controller swap is already in progress for this guest")
		return
	}
	s.swapInFlight[vmid] = true
	s.swapMu.Unlock()

	// Read the previous image for the immediate response (best-effort, short ctx).
	pctx, pcancel := context.WithTimeout(context.Background(), 10*time.Second)
	prev, _ := s.swap.CurrentImage(pctx, vmid)
	pcancel()

	base := s.baseCtx
	if base == nil {
		base = context.Background()
	}
	go func() {
		defer func() {
			s.swapMu.Lock()
			delete(s.swapInFlight, vmid)
			s.swapMu.Unlock()
		}()
		ctx, cancel := context.WithTimeout(base, 5*time.Minute)
		defer cancel()
		s.swap.Swap(ctx, vmid, target)
	}()

	s.logger.Warn("local-api: controller swap requested", "vmid", vmid, "target", target, "previous", prev)
	writeStatus(w, http.StatusAccepted, true, map[string]any{
		"status": "swapping", "previous_image": prev, "target_image": target,
	}, "")
}

// handleControllerSwapStatus is GET /controller/swap/status — the swap outcome for the (new) controller
// and the test to read.
func (s *Server) handleControllerSwapStatus(w http.ResponseWriter, r *http.Request, vmid int) {
	if s.swap == nil {
		writeErr(w, http.StatusServiceUnavailable, "controller-swap not configured on this host")
		return
	}
	s.swapMu.Lock()
	inflight := s.swapInFlight[vmid]
	s.swapMu.Unlock()

	st, err := s.swap.LoadState(vmid)
	if err != nil {
		writeErr(w, http.StatusInternalServerError, "read swap state: "+err.Error())
		return
	}
	if st == nil {
		writeStatus(w, http.StatusOK, true, map[string]any{"state": "none", "in_flight": inflight}, "")
		return
	}
	writeStatus(w, http.StatusOK, true, map[string]any{
		"state":     st.State,
		"in_flight": inflight,
		"current":   st.Current,
		"previous":  st.Previous,
		"target":    st.Target,
		"error":     st.Error,
	}, "")
}