controllerswap: F1 verify hardening — reject crash-looping no-healthcheck image v0.47.0
controllerHealthy reads RestartCount (running&&rc>0 -> not ok) + signals needsDwell for no-healthcheck; verify requires verifyDwell(=3) consecutive ok polls for a no-healthcheck image (real healthcheck trusted immediately). Closes the F1 hole (alpine crash-loop passed the point-in-time check). Red-proof + dwell + real-image tests. No sudoers/orchestration change. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Pg8ANF97SEeKYSN5Jxw3qJ
This commit is contained in:
@@ -26,6 +26,11 @@ type fakeGuestExec struct {
|
||||
teeStdin []string // raw bytes piped into each `tee` write (the swap's write vector)
|
||||
failRestart bool
|
||||
noHealthBlock bool // if set, .State.Health is absent ("none")
|
||||
restartCount int // F1: .RestartCount reported by docker inspect (a crash-looper has >0)
|
||||
// runningSeq, when non-nil, drives .State.Running per docker-inspect poll (last value repeats) —
|
||||
// e.g. {true, false} models a container that flickers Running then crashes. nil → always running.
|
||||
runningSeq []bool
|
||||
inspectN int
|
||||
}
|
||||
|
||||
func (f *fakeGuestExec) GuestExec(_ context.Context, _ int, args ...string) (string, error) {
|
||||
@@ -50,13 +55,22 @@ func (f *fakeGuestExec) GuestExec(_ context.Context, _ int, args ...string) (str
|
||||
if f.containerImg == "" {
|
||||
return "", fmt.Errorf("no such container")
|
||||
}
|
||||
running := true
|
||||
if f.runningSeq != nil {
|
||||
i := f.inspectN
|
||||
if i >= len(f.runningSeq) {
|
||||
i = len(f.runningSeq) - 1
|
||||
}
|
||||
running = f.runningSeq[i]
|
||||
f.inspectN++
|
||||
}
|
||||
health := "healthy"
|
||||
if f.noHealthBlock {
|
||||
health = "none"
|
||||
} else if !f.good[f.containerImg] {
|
||||
health = "unhealthy"
|
||||
}
|
||||
return fmt.Sprintf("true|%s|%s", health, f.containerImg), nil
|
||||
return fmt.Sprintf("%t|%s|%s|%d", running, health, f.containerImg, f.restartCount), nil
|
||||
}
|
||||
return "", fmt.Errorf("fake: unexpected exec %v", args)
|
||||
}
|
||||
@@ -267,3 +281,81 @@ func TestControllerSwapHandler_SingleFlight(t *testing.T) {
|
||||
t.Fatalf("status = %d, want 409 (swap already in progress)", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// inspectScript is a minimal GuestExecutor for verify()-level F1 tests: it returns scripted
|
||||
// `docker inspect` outputs (one per poll; the last repeats) and no-ops everything else.
|
||||
type inspectScript struct {
|
||||
outs []string
|
||||
i int
|
||||
}
|
||||
|
||||
func (s *inspectScript) GuestExec(_ context.Context, _ int, args ...string) (string, error) {
|
||||
if len(args) >= 2 && args[0] == "docker" && args[1] == "inspect" {
|
||||
j := s.i
|
||||
if j >= len(s.outs) {
|
||||
j = len(s.outs) - 1
|
||||
}
|
||||
s.i++
|
||||
return s.outs[j], nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
func (s *inspectScript) GuestExecStdin(_ context.Context, _ int, _ io.Reader, _ ...string) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func fastSwapper(exec GuestExecutor) *ControllerSwapper {
|
||||
s := NewControllerSwapper(exec, "", discardLogger())
|
||||
s.verifyTimeout = 100 * time.Millisecond
|
||||
s.verifyInterval = 4 * time.Millisecond
|
||||
return s
|
||||
}
|
||||
|
||||
// F1 RED-PROOF: a no-healthcheck image with RestartCount>0 (a crash-looper) must NOT verify ok →
|
||||
// verify() returns false → the swap's existing rollback path runs. Companion: the same image with
|
||||
// RestartCount=0 and dwell=1 DOES verify — proving the RestartCount check is what blocks the crasher
|
||||
// (the old `running + none → ok` shape had no such guard, the F1 hole found live).
|
||||
func TestF1_Verify_CrashLoopRestartCountBlocks(t *testing.T) {
|
||||
s := fastSwapper(&inspectScript{outs: []string{"true|none|" + newImg + "|2"}})
|
||||
if s.verify(context.Background(), 9201, newImg) {
|
||||
t.Fatal("RestartCount>0 (crash-looping no-healthcheck) must NOT verify ok")
|
||||
}
|
||||
// controllerHealthy is a pure predicate: rc>0 → (false, starting).
|
||||
ok, starting, _ := s.controllerHealthy(context.Background(), 9201, newImg)
|
||||
if ok || !starting {
|
||||
t.Fatalf("controllerHealthy(rc=2) = (ok=%v,starting=%v), want (false,true)", ok, starting)
|
||||
}
|
||||
// Companion: rc=0 + dwell=1 → verifies (shows the rc check, not something else, blocks the crasher).
|
||||
s2 := fastSwapper(&inspectScript{outs: []string{"true|none|" + newImg + "|0"}})
|
||||
s2.verifyDwell = 1
|
||||
if !s2.verify(context.Background(), 9201, newImg) {
|
||||
t.Fatal("companion: no-healthcheck rc=0 dwell=1 should verify")
|
||||
}
|
||||
}
|
||||
|
||||
// F1 dwell: a single ok poll then a crash (Running=false) must NOT verify with the dwell. Companion:
|
||||
// dwell=1 accepts the single ok (the false-positive the dwell removes).
|
||||
func TestF1_Verify_DwellSingleOkThenCrash(t *testing.T) {
|
||||
seq := []string{"true|none|" + newImg + "|0", "false|none|" + newImg + "|0"}
|
||||
s := fastSwapper(&inspectScript{outs: seq}) // verifyDwell=3 (default)
|
||||
if s.verify(context.Background(), 9201, newImg) {
|
||||
t.Fatal("a single ok then crash must NOT verify with the dwell")
|
||||
}
|
||||
s2 := fastSwapper(&inspectScript{outs: seq})
|
||||
s2.verifyDwell = 1
|
||||
if !s2.verify(context.Background(), 9201, newImg) {
|
||||
t.Fatal("companion: dwell=1 accepts the single ok")
|
||||
}
|
||||
}
|
||||
|
||||
// A real healthy image verifies promptly (no dwell) — the real controller path is unaffected.
|
||||
func TestF1_Verify_RealHealthcheckPassesPromptly(t *testing.T) {
|
||||
s := fastSwapper(&inspectScript{outs: []string{"true|healthy|" + newImg + "|0"}})
|
||||
if !s.verify(context.Background(), 9201, newImg) {
|
||||
t.Fatal("a real healthy image must verify promptly")
|
||||
}
|
||||
ok, _, needsDwell := s.controllerHealthy(context.Background(), 9201, newImg)
|
||||
if !ok || needsDwell {
|
||||
t.Fatalf("controllerHealthy(healthy) = (ok=%v,needsDwell=%v), want (true,false)", ok, needsDwell)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user