v0.42.0: agentic controller update — in-guest image swap + rollback (Phase 1)
New local-API POST /controller/swap (+ GET /controller/swap/status), withGuest- scoped: the agent records the previous image, confirms the target is present, rewrites /etc/felhom-controller-image, restarts felhom-controller-bootstrap.service, verifies the new controller is healthy (docker inspect, <=90s), and ROLLS BACK to the previous image if not. Single-flight per guest; strict image-ref validation; crash-safety state file. GuestBinder.GuestExec is the pct-exec seam. --selftest=controller-swap exercises it directly. Tests: happy/rollback-on-unhealthy(+red-proof)/image-absent/no-healthcheck/ bad-image-400/single-flight-409. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TtXesNa2LGbMmE4DNL6SE7
This commit is contained in:
@@ -0,0 +1,354 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---- agentic controller update (Phase 1): the agent owns the in-guest controller image SWAP -------
|
||||
//
|
||||
// The in-guest controller pre-PULLS the target image (its own registry token, shared docker socket),
|
||||
// then asks the agent to swap. The agent — external to the controller container, so it survives the
|
||||
// controller being killed mid-swap — rewrites /etc/felhom-controller-image and restarts the golden's
|
||||
// felhom-controller-bootstrap.service (docker rm -f + docker run), verifies the new controller comes
|
||||
// up healthy, and ROLLS BACK to the previous image if it does not. Latest-only is enforced caller-side
|
||||
// (the controller resolves latest via queryRegistry); here we only accept a strict controller image ref.
|
||||
|
||||
const (
|
||||
controllerImageFile = "/etc/felhom-controller-image"
|
||||
bootstrapUnit = "felhom-controller-bootstrap.service"
|
||||
controllerContainer = "felhom-controller"
|
||||
)
|
||||
|
||||
// controllerImageRe is the fail-safe gate: ONLY our controller repo + a 3-part semver tag may be run.
|
||||
// A malformed/hostile body can't make the agent docker-run an arbitrary image.
|
||||
var controllerImageRe = regexp.MustCompile(`^gitea\.dooplex\.hu/admin/felhom-controller:[0-9]+\.[0-9]+\.[0-9]+$`)
|
||||
|
||||
// ValidControllerImage reports whether ref is an acceptable swap target (strict).
|
||||
func ValidControllerImage(ref string) bool { return controllerImageRe.MatchString(ref) }
|
||||
|
||||
// GuestExecutor runs a command inside a guest (pct exec) and returns stdout. Satisfied by *GuestBinder;
|
||||
// faked in tests. The single seam the swap composes over (no hand-rolled pct).
|
||||
type GuestExecutor interface {
|
||||
GuestExec(ctx context.Context, vmid int, args ...string) (string, error)
|
||||
}
|
||||
|
||||
// ControllerSwapState is the durable record of a swap (crash-safety + status). Written before the swap
|
||||
// so a mid-swap agent crash leaves the previous image recoverable.
|
||||
type ControllerSwapState struct {
|
||||
VMID int `json:"vmid"`
|
||||
State string `json:"state"` // swapping | done | failed
|
||||
Previous string `json:"previous_image"`
|
||||
Target string `json:"target_image"`
|
||||
Current string `json:"current_image"` // image the file currently names
|
||||
Error string `json:"error,omitempty"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ControllerSwapper performs the swap over a GuestExecutor. Stateless except the on-disk state file and
|
||||
// the (test-tunable) verify timing.
|
||||
type ControllerSwapper struct {
|
||||
exec GuestExecutor
|
||||
stateDir string
|
||||
logger *slog.Logger
|
||||
verifyTimeout time.Duration
|
||||
verifyInterval time.Duration
|
||||
}
|
||||
|
||||
// NewControllerSwapper builds a swapper. stateDir holds controller-swap-<vmid>.json (default
|
||||
// /var/lib/felhom-agent when empty).
|
||||
func NewControllerSwapper(exec GuestExecutor, stateDir string, logger *slog.Logger) *ControllerSwapper {
|
||||
if stateDir == "" {
|
||||
stateDir = "/var/lib/felhom-agent"
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &ControllerSwapper{
|
||||
exec: exec,
|
||||
stateDir: stateDir,
|
||||
logger: logger,
|
||||
verifyTimeout: 90 * time.Second,
|
||||
verifyInterval: 3 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ControllerSwapper) statePath(vmid int) string {
|
||||
return filepath.Join(c.stateDir, fmt.Sprintf("controller-swap-%d.json", vmid))
|
||||
}
|
||||
|
||||
func (c *ControllerSwapper) saveState(st *ControllerSwapState) {
|
||||
st.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
if err := os.MkdirAll(c.stateDir, 0o755); err != nil {
|
||||
c.logger.Error("controller-swap: mkdir state dir", "err", err)
|
||||
return
|
||||
}
|
||||
b, _ := json.MarshalIndent(st, "", " ")
|
||||
tmp := c.statePath(st.VMID) + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
c.logger.Error("controller-swap: write state", "err", err)
|
||||
return
|
||||
}
|
||||
if err := os.Rename(tmp, c.statePath(st.VMID)); err != nil {
|
||||
c.logger.Error("controller-swap: rename state", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// LoadState returns the recorded swap state for a guest, or nil if none.
|
||||
func (c *ControllerSwapper) LoadState(vmid int) (*ControllerSwapState, error) {
|
||||
b, err := os.ReadFile(c.statePath(vmid))
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var st ControllerSwapState
|
||||
if err := json.Unmarshal(b, &st); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &st, nil
|
||||
}
|
||||
|
||||
// CurrentImage reads /etc/felhom-controller-image in the guest.
|
||||
func (c *ControllerSwapper) CurrentImage(ctx context.Context, vmid int) (string, error) {
|
||||
out, err := c.exec.GuestExec(ctx, vmid, "cat", controllerImageFile)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(out), nil
|
||||
}
|
||||
|
||||
func (c *ControllerSwapper) imagePresent(ctx context.Context, vmid int, image string) bool {
|
||||
_, err := c.exec.GuestExec(ctx, vmid, "docker", "image", "inspect", image)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *ControllerSwapper) writeImage(ctx context.Context, vmid int, image string) error {
|
||||
// image is strict-validated (controllerImageRe) before we ever get here, so this single-quoted
|
||||
// interpolation cannot smuggle shell metacharacters.
|
||||
cmd := fmt.Sprintf("printf '%%s\\n' '%s' > %s", image, controllerImageFile)
|
||||
_, err := c.exec.GuestExec(ctx, vmid, "bash", "-c", cmd)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *ControllerSwapper) restartBootstrap(ctx context.Context, vmid int) error {
|
||||
_, err := c.exec.GuestExec(ctx, vmid, "systemctl", "restart", bootstrapUnit)
|
||||
return err
|
||||
}
|
||||
|
||||
// controllerHealthy reports running + (healthy or no healthcheck) + the running image == want.
|
||||
// Returns (ok, stillStarting): stillStarting=true means keep polling.
|
||||
func (c *ControllerSwapper) controllerHealthy(ctx context.Context, vmid int, want string) (ok, starting bool) {
|
||||
out, err := c.exec.GuestExec(ctx, vmid, "docker", "inspect", "-f",
|
||||
"{{.State.Running}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}|{{.Config.Image}}", controllerContainer)
|
||||
if err != nil {
|
||||
return false, true // container not there yet (rm -f window) → keep polling
|
||||
}
|
||||
f := strings.SplitN(strings.TrimSpace(out), "|", 3)
|
||||
if len(f) != 3 {
|
||||
return false, true
|
||||
}
|
||||
running, health, image := f[0] == "true", f[1], f[2]
|
||||
if !running {
|
||||
return false, true
|
||||
}
|
||||
if image != want {
|
||||
return false, true // bootstrap may not have re-run yet
|
||||
}
|
||||
switch health {
|
||||
case "healthy", "none":
|
||||
return true, false
|
||||
case "starting":
|
||||
return false, true
|
||||
default: // unhealthy
|
||||
return false, true
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ControllerSwapper) verify(ctx context.Context, vmid int, want string) bool {
|
||||
deadline := time.Now().Add(c.verifyTimeout)
|
||||
for {
|
||||
if ok, _ := c.controllerHealthy(ctx, vmid, want); ok {
|
||||
return true
|
||||
}
|
||||
if time.Now().After(deadline) || ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-time.After(c.verifyInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Swap performs the full swap+verify+rollback. Returns the final state. It is synchronous (the HTTP
|
||||
// handler runs it in a goroutine after a 202). Idempotency/single-flight is the caller's.
|
||||
func (c *ControllerSwapper) Swap(ctx context.Context, vmid int, target string) *ControllerSwapState {
|
||||
st := &ControllerSwapState{VMID: vmid, State: "swapping", Target: target}
|
||||
prev, err := c.CurrentImage(ctx, vmid)
|
||||
if err != nil {
|
||||
c.logger.Warn("controller-swap: could not read current image (continuing)", "vmid", vmid, "err", err)
|
||||
}
|
||||
st.Previous, st.Current = prev, prev
|
||||
c.saveState(st) // crash-safety: previous recorded BEFORE any mutation
|
||||
|
||||
// The controller pre-pulled the image; refuse to swap to an absent image (would brick the guest).
|
||||
if !c.imagePresent(ctx, vmid, target) {
|
||||
st.State = "failed"
|
||||
st.Error = "target image not present in guest (controller did not pre-pull it)"
|
||||
c.saveState(st)
|
||||
c.logger.Error("controller-swap: target image absent — no swap", "vmid", vmid, "target", target)
|
||||
return st
|
||||
}
|
||||
|
||||
if err := c.writeImage(ctx, vmid, target); err != nil {
|
||||
st.State = "failed"
|
||||
st.Error = "write image file: " + err.Error()
|
||||
c.saveState(st)
|
||||
return st
|
||||
}
|
||||
st.Current = target
|
||||
c.saveState(st)
|
||||
c.logger.Info("controller-swap: image file written, restarting bootstrap", "vmid", vmid, "target", target)
|
||||
if err := c.restartBootstrap(ctx, vmid); err != nil {
|
||||
c.logger.Error("controller-swap: bootstrap restart failed — rolling back", "vmid", vmid, "err", err)
|
||||
return c.rollback(ctx, st, "bootstrap restart failed: "+err.Error())
|
||||
}
|
||||
|
||||
if c.verify(ctx, vmid, target) {
|
||||
st.State = "done"
|
||||
st.Error = ""
|
||||
c.saveState(st)
|
||||
c.logger.Info("controller-swap: new controller healthy", "vmid", vmid, "target", target)
|
||||
return st
|
||||
}
|
||||
return c.rollback(ctx, st, "new controller did not become healthy within timeout")
|
||||
}
|
||||
|
||||
// rollback reverts /etc/felhom-controller-image to the previous image and restarts the unit so the
|
||||
// guest is NEVER left with no controller. Best-effort verify of the previous coming back.
|
||||
func (c *ControllerSwapper) rollback(ctx context.Context, st *ControllerSwapState, reason string) *ControllerSwapState {
|
||||
st.State = "failed"
|
||||
st.Error = reason
|
||||
if st.Previous == "" {
|
||||
c.logger.Error("controller-swap: cannot roll back — no previous image recorded", "vmid", st.VMID)
|
||||
c.saveState(st)
|
||||
return st
|
||||
}
|
||||
c.logger.Warn("controller-swap: rolling back", "vmid", st.VMID, "previous", st.Previous, "reason", reason)
|
||||
if err := c.writeImage(ctx, st.VMID, st.Previous); err != nil {
|
||||
st.Error = reason + "; ALSO rollback write failed: " + err.Error()
|
||||
c.saveState(st)
|
||||
return st
|
||||
}
|
||||
st.Current = st.Previous
|
||||
if err := c.restartBootstrap(ctx, st.VMID); err != nil {
|
||||
st.Error = reason + "; ALSO rollback restart failed: " + err.Error()
|
||||
c.saveState(st)
|
||||
return st
|
||||
}
|
||||
if c.verify(ctx, st.VMID, st.Previous) {
|
||||
c.logger.Info("controller-swap: rolled back to previous, controller healthy", "vmid", st.VMID, "previous", st.Previous)
|
||||
} else {
|
||||
c.logger.Error("controller-swap: rolled back but previous controller not confirmed healthy", "vmid", st.VMID)
|
||||
}
|
||||
c.saveState(st)
|
||||
return st
|
||||
}
|
||||
|
||||
// ---- HTTP handlers ----------------------------------------------------------------------------------
|
||||
|
||||
// handleControllerSwap is POST /controller/swap {image}. Auth+scoping via withGuest. Responds 202 then
|
||||
// swaps async (single-flight per guest). The controller will be killed by the swap, so the agent owns
|
||||
// the rest (verify + rollback) independently.
|
||||
func (s *Server) handleControllerSwap(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.swap == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "controller-swap not configured on this host")
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Image string `json:"image"`
|
||||
}
|
||||
if !decodeBody(w, r, &req) {
|
||||
return
|
||||
}
|
||||
target := strings.TrimSpace(req.Image)
|
||||
if !ValidControllerImage(target) {
|
||||
writeErr(w, http.StatusBadRequest, "invalid controller image ref (must be gitea.dooplex.hu/admin/felhom-controller:<semver>)")
|
||||
return
|
||||
}
|
||||
|
||||
s.swapMu.Lock()
|
||||
if s.swapInFlight[vmid] {
|
||||
s.swapMu.Unlock()
|
||||
writeErr(w, http.StatusConflict, "a controller swap is already in progress for this guest")
|
||||
return
|
||||
}
|
||||
s.swapInFlight[vmid] = true
|
||||
s.swapMu.Unlock()
|
||||
|
||||
// Read the previous image for the immediate response (best-effort, short ctx).
|
||||
pctx, pcancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
prev, _ := s.swap.CurrentImage(pctx, vmid)
|
||||
pcancel()
|
||||
|
||||
base := s.baseCtx
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
go func() {
|
||||
defer func() {
|
||||
s.swapMu.Lock()
|
||||
delete(s.swapInFlight, vmid)
|
||||
s.swapMu.Unlock()
|
||||
}()
|
||||
ctx, cancel := context.WithTimeout(base, 5*time.Minute)
|
||||
defer cancel()
|
||||
s.swap.Swap(ctx, vmid, target)
|
||||
}()
|
||||
|
||||
s.logger.Warn("local-api: controller swap requested", "vmid", vmid, "target", target, "previous", prev)
|
||||
writeStatus(w, http.StatusAccepted, true, map[string]any{
|
||||
"status": "swapping", "previous_image": prev, "target_image": target,
|
||||
}, "")
|
||||
}
|
||||
|
||||
// handleControllerSwapStatus is GET /controller/swap/status — the swap outcome for the (new) controller
|
||||
// and the test to read.
|
||||
func (s *Server) handleControllerSwapStatus(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.swap == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "controller-swap not configured on this host")
|
||||
return
|
||||
}
|
||||
s.swapMu.Lock()
|
||||
inflight := s.swapInFlight[vmid]
|
||||
s.swapMu.Unlock()
|
||||
|
||||
st, err := s.swap.LoadState(vmid)
|
||||
if err != nil {
|
||||
writeErr(w, http.StatusInternalServerError, "read swap state: "+err.Error())
|
||||
return
|
||||
}
|
||||
if st == nil {
|
||||
writeStatus(w, http.StatusOK, true, map[string]any{"state": "none", "in_flight": inflight}, "")
|
||||
return
|
||||
}
|
||||
writeStatus(w, http.StatusOK, true, map[string]any{
|
||||
"state": st.State,
|
||||
"in_flight": inflight,
|
||||
"current": st.Current,
|
||||
"previous": st.Previous,
|
||||
"target": st.Target,
|
||||
"error": st.Error,
|
||||
}, "")
|
||||
}
|
||||
Reference in New Issue
Block a user