controllerswap: F1 verify hardening — reject crash-looping no-healthcheck image v0.47.0

controllerHealthy reads RestartCount (running&&rc>0 -> not ok) + signals needsDwell for no-healthcheck;
verify requires verifyDwell(=3) consecutive ok polls for a no-healthcheck image (real healthcheck
trusted immediately). Closes the F1 hole (alpine crash-loop passed the point-in-time check). Red-proof
+ dwell + real-image tests. No sudoers/orchestration change.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Pg8ANF97SEeKYSN5Jxw3qJ
This commit is contained in:
2026-06-29 22:45:41 +02:00
parent bb548e3c5a
commit 3844df7c23
8 changed files with 218 additions and 57 deletions
+93 -1
View File
@@ -26,6 +26,11 @@ type fakeGuestExec struct {
teeStdin []string // raw bytes piped into each `tee` write (the swap's write vector)
failRestart bool
noHealthBlock bool // if set, .State.Health is absent ("none")
restartCount int // F1: .RestartCount reported by docker inspect (a crash-looper has >0)
// runningSeq, when non-nil, drives .State.Running per docker-inspect poll (last value repeats) —
// e.g. {true, false} models a container that flickers Running then crashes. nil → always running.
runningSeq []bool
inspectN int
}
func (f *fakeGuestExec) GuestExec(_ context.Context, _ int, args ...string) (string, error) {
@@ -50,13 +55,22 @@ func (f *fakeGuestExec) GuestExec(_ context.Context, _ int, args ...string) (str
if f.containerImg == "" {
return "", fmt.Errorf("no such container")
}
running := true
if f.runningSeq != nil {
i := f.inspectN
if i >= len(f.runningSeq) {
i = len(f.runningSeq) - 1
}
running = f.runningSeq[i]
f.inspectN++
}
health := "healthy"
if f.noHealthBlock {
health = "none"
} else if !f.good[f.containerImg] {
health = "unhealthy"
}
return fmt.Sprintf("true|%s|%s", health, f.containerImg), nil
return fmt.Sprintf("%t|%s|%s|%d", running, health, f.containerImg, f.restartCount), nil
}
return "", fmt.Errorf("fake: unexpected exec %v", args)
}
@@ -267,3 +281,81 @@ func TestControllerSwapHandler_SingleFlight(t *testing.T) {
t.Fatalf("status = %d, want 409 (swap already in progress)", rr.Code)
}
}
// inspectScript is a minimal GuestExecutor for verify()-level F1 tests: it returns scripted
// `docker inspect` outputs (one per poll; the last repeats) and no-ops everything else.
type inspectScript struct {
outs []string
i int
}
func (s *inspectScript) GuestExec(_ context.Context, _ int, args ...string) (string, error) {
if len(args) >= 2 && args[0] == "docker" && args[1] == "inspect" {
j := s.i
if j >= len(s.outs) {
j = len(s.outs) - 1
}
s.i++
return s.outs[j], nil
}
return "", nil
}
func (s *inspectScript) GuestExecStdin(_ context.Context, _ int, _ io.Reader, _ ...string) (string, error) {
return "", nil
}
func fastSwapper(exec GuestExecutor) *ControllerSwapper {
s := NewControllerSwapper(exec, "", discardLogger())
s.verifyTimeout = 100 * time.Millisecond
s.verifyInterval = 4 * time.Millisecond
return s
}
// F1 RED-PROOF: a no-healthcheck image with RestartCount>0 (a crash-looper) must NOT verify ok →
// verify() returns false → the swap's existing rollback path runs. Companion: the same image with
// RestartCount=0 and dwell=1 DOES verify — proving the RestartCount check is what blocks the crasher
// (the old `running + none → ok` shape had no such guard, the F1 hole found live).
func TestF1_Verify_CrashLoopRestartCountBlocks(t *testing.T) {
s := fastSwapper(&inspectScript{outs: []string{"true|none|" + newImg + "|2"}})
if s.verify(context.Background(), 9201, newImg) {
t.Fatal("RestartCount>0 (crash-looping no-healthcheck) must NOT verify ok")
}
// controllerHealthy is a pure predicate: rc>0 → (false, starting).
ok, starting, _ := s.controllerHealthy(context.Background(), 9201, newImg)
if ok || !starting {
t.Fatalf("controllerHealthy(rc=2) = (ok=%v,starting=%v), want (false,true)", ok, starting)
}
// Companion: rc=0 + dwell=1 → verifies (shows the rc check, not something else, blocks the crasher).
s2 := fastSwapper(&inspectScript{outs: []string{"true|none|" + newImg + "|0"}})
s2.verifyDwell = 1
if !s2.verify(context.Background(), 9201, newImg) {
t.Fatal("companion: no-healthcheck rc=0 dwell=1 should verify")
}
}
// F1 dwell: a single ok poll then a crash (Running=false) must NOT verify with the dwell. Companion:
// dwell=1 accepts the single ok (the false-positive the dwell removes).
func TestF1_Verify_DwellSingleOkThenCrash(t *testing.T) {
seq := []string{"true|none|" + newImg + "|0", "false|none|" + newImg + "|0"}
s := fastSwapper(&inspectScript{outs: seq}) // verifyDwell=3 (default)
if s.verify(context.Background(), 9201, newImg) {
t.Fatal("a single ok then crash must NOT verify with the dwell")
}
s2 := fastSwapper(&inspectScript{outs: seq})
s2.verifyDwell = 1
if !s2.verify(context.Background(), 9201, newImg) {
t.Fatal("companion: dwell=1 accepts the single ok")
}
}
// A real healthy image verifies promptly (no dwell) — the real controller path is unaffected.
func TestF1_Verify_RealHealthcheckPassesPromptly(t *testing.T) {
s := fastSwapper(&inspectScript{outs: []string{"true|healthy|" + newImg + "|0"}})
if !s.verify(context.Background(), 9201, newImg) {
t.Fatal("a real healthy image must verify promptly")
}
ok, _, needsDwell := s.controllerHealthy(context.Background(), 9201, newImg)
if !ok || needsDwell {
t.Fatalf("controllerHealthy(healthy) = (ok=%v,needsDwell=%v), want (true,false)", ok, needsDwell)
}
}