285dd1032f
R-51: aggregateState's mixed branch returned StateRunning ("partial"), so a stack whose
MAIN container was dead behind live helpers alerted on nothing — immich-server sat Exited
for 18 h, 100 % unreachable, no banner and no app_start_failed (audit F4). New
StateDegraded: a DOWN member whose docker restart policy is always/unless-stopped is a
fault (degraded, a down state); no/on-failure is a finished one-shot and stays benign; an
unreadable policy fails CLOSED. The unhealthy/restarting/paused/unknown exclusions are
byte-identical — folding unhealthy into down is the flapping fix-3 avoided.
R-52: new internal/bootrecon — one bounded start-once sweep at startup (2 attempts, 30 s
apart) for apps an interrupted boot left behind, inside the 90 s boot grace so a success
is silent and a failure still alerts. A zero-container stack is NEVER touched: the UI's
Stop is compose down, so a deliberate stop survives a reboot.
Both features carry a production-path wiring test (the v0.154.0 / v0.91.0 inert-seam
class). The main() assertion is an AST walk, not strings.Contains — the substring version
passed its own red-proof, because a commented-out call still contains the string.
Red-proofs run and restored: mix branch reverted -> "running" on the immich fixture;
boot hook commented out -> wiring test fails; zero-container gate dropped -> the
user-stopped app gets started.
NOTE: controller/cmd/controller/ is matched by .gitignore's `controller` entry, so new
files there need `git add -f` (and ripgrep silently skips main.go without --no-ignore).
123 lines
4.3 KiB
Go
123 lines
4.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"go/ast"
|
|
"go/parser"
|
|
"go/token"
|
|
"io"
|
|
"log"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/bootrecon"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
|
)
|
|
|
|
// §9 rule 6 — the seam-discipline test. Two inert-seam defects shipped in the two days before this
|
|
// task (controller v0.154.0 and agent v0.91.0), both the same shape: the component was correct, its
|
|
// unit tests injected the seam directly, and the PRODUCTION CALLER was never made. Everything was
|
|
// green and the feature did nothing. So R-52 gets its wiring asserted from package main, not only
|
|
// from internal/bootrecon.
|
|
|
|
// TestRunBootReconcile_InvokesTheSweep pins the function main() actually calls: after the settle
|
|
// window it runs the sweep exactly once, with the manager it was handed.
|
|
func TestRunBootReconcile_InvokesTheSweep(t *testing.T) {
|
|
orig := bootReconcileFn
|
|
t.Cleanup(func() { bootReconcileFn = orig })
|
|
origSettle := bootReconcileSettle
|
|
t.Cleanup(func() { bootReconcileSettle = origSettle })
|
|
bootReconcileSettle = time.Millisecond
|
|
|
|
calls := 0
|
|
var gotMgr bootrecon.StackProvider
|
|
bootReconcileFn = func(_ context.Context, mgr bootrecon.StackProvider, _ *log.Logger) bootrecon.Result {
|
|
calls++
|
|
gotMgr = mgr
|
|
return bootrecon.Result{}
|
|
}
|
|
|
|
fake := &wiringStacks{}
|
|
runBootReconcile(context.Background(), fake, log.New(io.Discard, "", 0))
|
|
|
|
if calls != 1 {
|
|
t.Fatalf("the boot sweep ran %d times, want exactly 1 (start-once, never a loop)", calls)
|
|
}
|
|
if gotMgr != bootrecon.StackProvider(fake) {
|
|
t.Fatalf("the sweep was handed %v, want the stack manager main() owns", gotMgr)
|
|
}
|
|
}
|
|
|
|
// A controller shutting down during its own settle window must not start anything.
|
|
func TestRunBootReconcile_CancelledDuringSettleDoesNothing(t *testing.T) {
|
|
orig := bootReconcileFn
|
|
t.Cleanup(func() { bootReconcileFn = orig })
|
|
|
|
calls := 0
|
|
bootReconcileFn = func(context.Context, bootrecon.StackProvider, *log.Logger) bootrecon.Result {
|
|
calls++
|
|
return bootrecon.Result{}
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
runBootReconcile(ctx, &wiringStacks{}, log.New(io.Discard, "", 0))
|
|
|
|
if calls != 0 {
|
|
t.Fatalf("the sweep ran %d times on a cancelled context, want 0", calls)
|
|
}
|
|
}
|
|
|
|
// The call site itself. A function-variable test can only prove the function is correct — it cannot
|
|
// prove main() calls it, which is exactly the hole both inert-seam defects fell through. This walks
|
|
// main.go's AST for a `go runBootReconcile(...)` inside func main(); delete or comment out that line
|
|
// and this fails, where every behavioural test above would still pass.
|
|
//
|
|
// It is an AST walk and not a strings.Contains for a reason found while red-proofing it: a
|
|
// commented-out call still satisfies a substring match, so the text version passed the very
|
|
// red-proof it existed to fail. Comments are not code.
|
|
func TestMainWiresBootReconcile(t *testing.T) {
|
|
fset := token.NewFileSet()
|
|
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
|
if err != nil {
|
|
t.Fatalf("parse main.go: %v", err)
|
|
}
|
|
|
|
found := false
|
|
for _, decl := range f.Decls {
|
|
fn, ok := decl.(*ast.FuncDecl)
|
|
if !ok || fn.Name.Name != "main" || fn.Body == nil {
|
|
continue
|
|
}
|
|
ast.Inspect(fn.Body, func(n ast.Node) bool {
|
|
gostmt, ok := n.(*ast.GoStmt)
|
|
if !ok {
|
|
return true
|
|
}
|
|
if ident, ok := gostmt.Call.Fun.(*ast.Ident); ok && ident.Name == "runBootReconcile" {
|
|
found = true
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
if !found {
|
|
t.Fatal("func main() no longer starts the R-52 boot reconciliation with `go runBootReconcile(...)` " +
|
|
"— the sweep is inert (the v0.154.0 / v0.91.0 defect class: a correct component nobody calls)")
|
|
}
|
|
}
|
|
|
|
// The settle window must stay inside the dead-app boot grace, or a successful recovery would alert.
|
|
func TestBootReconcileFitsInsideTheBootGrace(t *testing.T) {
|
|
worst := bootReconcileSettle + time.Duration(bootrecon.DefaultAttempts-1)*bootrecon.DefaultRetryDelay
|
|
if worst >= deadAppBootGrace {
|
|
t.Fatalf("worst-case sweep %s does not fit inside the %s boot grace — a successful "+
|
|
"recovery would fire app_start_failed", worst, deadAppBootGrace)
|
|
}
|
|
}
|
|
|
|
type wiringStacks struct{}
|
|
|
|
func (w *wiringStacks) GetStacks() []stacks.Stack { return nil }
|
|
func (w *wiringStacks) StartStack(string) error { return nil }
|
|
func (w *wiringStacks) RefreshStatus() error { return nil }
|