v0.42.0: agentic controller update — in-guest image swap + rollback (Phase 1)
New local-API POST /controller/swap (+ GET /controller/swap/status), withGuest- scoped: the agent records the previous image, confirms the target is present, rewrites /etc/felhom-controller-image, restarts felhom-controller-bootstrap.service, verifies the new controller is healthy (docker inspect, <=90s), and ROLLS BACK to the previous image if not. Single-flight per guest; strict image-ref validation; crash-safety state file. GuestBinder.GuestExec is the pct-exec seam. --selftest=controller-swap exercises it directly. Tests: happy/rollback-on-unhealthy(+red-proof)/image-absent/no-healthcheck/ bad-image-400/single-flight-409. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TtXesNa2LGbMmE4DNL6SE7
This commit is contained in:
@@ -0,0 +1,213 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func discardLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }
|
||||
|
||||
// fakeGuestExec simulates `pct exec` into a guest for the swap primitive. It models the image file,
|
||||
// which images are present (pulled), which come up healthy, and the running container's image.
|
||||
type fakeGuestExec struct {
|
||||
mu sync.Mutex
|
||||
calls [][]string
|
||||
imageFile string // /etc/felhom-controller-image content
|
||||
present map[string]bool // images pulled into the guest
|
||||
good map[string]bool // images that report healthy when running
|
||||
containerImg string // image the running container currently has
|
||||
failRestart bool
|
||||
noHealthBlock bool // if set, .State.Health is absent ("none")
|
||||
}
|
||||
|
||||
var writeImgRe = regexp.MustCompile(`'([^']+)' >`)
|
||||
|
||||
func (f *fakeGuestExec) GuestExec(_ context.Context, _ int, args ...string) (string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.calls = append(f.calls, args)
|
||||
switch {
|
||||
case len(args) >= 2 && args[0] == "cat" && args[1] == controllerImageFile:
|
||||
return f.imageFile + "\n", nil
|
||||
case len(args) >= 4 && args[0] == "docker" && args[1] == "image" && args[2] == "inspect":
|
||||
if f.present[args[3]] {
|
||||
return args[3], nil
|
||||
}
|
||||
return "", fmt.Errorf("no such image: %s", args[3])
|
||||
case len(args) >= 3 && args[0] == "bash" && args[1] == "-c":
|
||||
m := writeImgRe.FindStringSubmatch(args[2])
|
||||
if m == nil {
|
||||
return "", fmt.Errorf("fake: unparseable write cmd %q", args[2])
|
||||
}
|
||||
f.imageFile = m[1]
|
||||
return "", nil
|
||||
case len(args) >= 3 && args[0] == "systemctl" && args[1] == "restart":
|
||||
if f.failRestart {
|
||||
return "", fmt.Errorf("fake: systemctl restart failed")
|
||||
}
|
||||
f.containerImg = f.imageFile // bootstrap re-ran: container now runs the file's image
|
||||
return "", nil
|
||||
case len(args) >= 2 && args[0] == "docker" && args[1] == "inspect":
|
||||
if f.containerImg == "" {
|
||||
return "", fmt.Errorf("no such container")
|
||||
}
|
||||
health := "healthy"
|
||||
if f.noHealthBlock {
|
||||
health = "none"
|
||||
} else if !f.good[f.containerImg] {
|
||||
health = "unhealthy"
|
||||
}
|
||||
return fmt.Sprintf("true|%s|%s", health, f.containerImg), nil
|
||||
}
|
||||
return "", fmt.Errorf("fake: unexpected exec %v", args)
|
||||
}
|
||||
|
||||
func (f *fakeGuestExec) wrote(image string) bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, c := range f.calls {
|
||||
if len(c) >= 3 && c[0] == "bash" && strings.Contains(c[2], "'"+image+"'") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func newTestSwapper(t *testing.T, fe *fakeGuestExec) *ControllerSwapper {
|
||||
t.Helper()
|
||||
s := NewControllerSwapper(fe, t.TempDir(), nil)
|
||||
s.verifyTimeout = 200 * time.Millisecond
|
||||
s.verifyInterval = 5 * time.Millisecond
|
||||
return s
|
||||
}
|
||||
|
||||
const (
|
||||
prevImg = "gitea.dooplex.hu/admin/felhom-controller:0.77.0"
|
||||
newImg = "gitea.dooplex.hu/admin/felhom-controller:0.84.0"
|
||||
)
|
||||
|
||||
func TestControllerSwap_Happy(t *testing.T) {
|
||||
fe := &fakeGuestExec{
|
||||
imageFile: prevImg,
|
||||
present: map[string]bool{newImg: true},
|
||||
good: map[string]bool{newImg: true, prevImg: true},
|
||||
}
|
||||
s := newTestSwapper(t, fe)
|
||||
st := s.Swap(context.Background(), 9201, newImg)
|
||||
|
||||
if st.State != "done" {
|
||||
t.Fatalf("state = %q (err=%q), want done", st.State, st.Error)
|
||||
}
|
||||
if fe.imageFile != newImg {
|
||||
t.Errorf("image file = %q, want %q", fe.imageFile, newImg)
|
||||
}
|
||||
if st.Previous != prevImg {
|
||||
t.Errorf("previous = %q, want %q", st.Previous, prevImg)
|
||||
}
|
||||
if !fe.wrote(newImg) {
|
||||
t.Errorf("expected the new image to be written")
|
||||
}
|
||||
// state file persisted
|
||||
got, _ := s.LoadState(9201)
|
||||
if got == nil || got.State != "done" {
|
||||
t.Errorf("state file = %+v, want done", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The load-bearing failure path: an unhealthy target must roll back to the previous image.
|
||||
func TestControllerSwap_RollbackOnUnhealthy(t *testing.T) {
|
||||
fe := &fakeGuestExec{
|
||||
imageFile: prevImg,
|
||||
present: map[string]bool{newImg: true},
|
||||
good: map[string]bool{prevImg: true}, // newImg present but NEVER healthy
|
||||
}
|
||||
s := newTestSwapper(t, fe)
|
||||
st := s.Swap(context.Background(), 9201, newImg)
|
||||
|
||||
if st.State != "failed" {
|
||||
t.Fatalf("state = %q, want failed", st.State)
|
||||
}
|
||||
if fe.imageFile != prevImg {
|
||||
t.Errorf("image file = %q after rollback, want previous %q (guest left on bad image!)", fe.imageFile, prevImg)
|
||||
}
|
||||
if fe.containerImg != prevImg {
|
||||
t.Errorf("running container = %q after rollback, want %q", fe.containerImg, prevImg)
|
||||
}
|
||||
// restart called at least twice (swap + rollback)
|
||||
restarts := 0
|
||||
for _, c := range fe.calls {
|
||||
if len(c) >= 2 && c[0] == "systemctl" && c[1] == "restart" {
|
||||
restarts++
|
||||
}
|
||||
}
|
||||
if restarts < 2 {
|
||||
t.Errorf("systemctl restart called %d times, want ≥2 (swap + rollback)", restarts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControllerSwap_ImageAbsent_NoSwap(t *testing.T) {
|
||||
fe := &fakeGuestExec{
|
||||
imageFile: prevImg,
|
||||
present: map[string]bool{}, // target NOT pulled
|
||||
good: map[string]bool{prevImg: true},
|
||||
}
|
||||
s := newTestSwapper(t, fe)
|
||||
st := s.Swap(context.Background(), 9201, newImg)
|
||||
|
||||
if st.State != "failed" {
|
||||
t.Fatalf("state = %q, want failed", st.State)
|
||||
}
|
||||
if fe.imageFile != prevImg {
|
||||
t.Errorf("image file = %q, want unchanged %q (no swap on absent image)", fe.imageFile, prevImg)
|
||||
}
|
||||
if fe.wrote(newImg) {
|
||||
t.Errorf("must NOT write the image file when the target image is absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestControllerSwap_HealthyWithNoHealthcheck(t *testing.T) {
|
||||
fe := &fakeGuestExec{
|
||||
imageFile: prevImg,
|
||||
present: map[string]bool{newImg: true},
|
||||
noHealthBlock: true, // image defines no HEALTHCHECK → running is enough
|
||||
}
|
||||
s := newTestSwapper(t, fe)
|
||||
st := s.Swap(context.Background(), 9201, newImg)
|
||||
if st.State != "done" {
|
||||
t.Fatalf("state = %q, want done (running + no healthcheck)", st.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControllerSwapHandler_BadImage(t *testing.T) {
|
||||
fe := &fakeGuestExec{present: map[string]bool{}}
|
||||
s := &Server{swap: newTestSwapper(t, fe), swapInFlight: map[int]bool{}, logger: discardLogger()}
|
||||
body := `{"image":"docker.io/evil/runme:latest"}`
|
||||
req := httptest.NewRequest("POST", "/controller/swap", strings.NewReader(body))
|
||||
rr := httptest.NewRecorder()
|
||||
s.handleControllerSwap(rr, req, 9201)
|
||||
if rr.Code != 400 {
|
||||
t.Fatalf("status = %d, want 400 for a non-controller image", rr.Code)
|
||||
}
|
||||
if len(fe.calls) != 0 {
|
||||
t.Errorf("no guest exec should run for a rejected image, got %v", fe.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestControllerSwapHandler_SingleFlight(t *testing.T) {
|
||||
fe := &fakeGuestExec{present: map[string]bool{}}
|
||||
s := &Server{swap: newTestSwapper(t, fe), swapInFlight: map[int]bool{9201: true}, logger: discardLogger()}
|
||||
req := httptest.NewRequest("POST", "/controller/swap", strings.NewReader(`{"image":"`+newImg+`"}`))
|
||||
rr := httptest.NewRecorder()
|
||||
s.handleControllerSwap(rr, req, 9201)
|
||||
if rr.Code != 409 {
|
||||
t.Fatalf("status = %d, want 409 (swap already in progress)", rr.Code)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user