admin
5adae4dad9
gates / gates (push) Successful in 9s
Found live on 9201: neither sched.Daily nor sched.Every fires on registration, so a box booting with a filesystem already over the line would stay silent for up to 24h — the R-100 shape, and the same gap the hub's own checkers avoid by leaving already-breached keys unseeded at init. The watcher now runs once 90s after startup as well. Safe because the check is edge-triggered against persisted state: an already-warned filesystem stays silent. The delay lets mounts settle so a drive still returning reads as unreadable and is skipped rather than warned about. Pinned by an AST assertion — the schedule registration alone no longer satisfies the test.
496 lines
19 KiB
Go
496 lines
19 KiB
Go
package main
|
|
|
|
import (
|
|
"go/ast"
|
|
"go/parser"
|
|
"go/token"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// R-166 §10 seam discipline — the recovery and the backfill are seams, and a seam that is never
|
|
// called is the defect class this project has shipped four times: a correct component, green unit
|
|
// tests that inject it directly, and no production caller.
|
|
//
|
|
// These walk main.go's AST. NOT strings.Contains — the sibling bootrecon test records the reason at
|
|
// first hand: a commented-out call still satisfies a substring match, so the text version passed the
|
|
// very red-proof it existed to fail. Comments are not code.
|
|
|
|
// mainBody returns func main()'s body from main.go, parsed.
|
|
func mainBody(t *testing.T) *ast.BlockStmt {
|
|
t.Helper()
|
|
fset := token.NewFileSet()
|
|
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
|
if err != nil {
|
|
t.Fatalf("parse main.go: %v", err)
|
|
}
|
|
for _, decl := range f.Decls {
|
|
if fn, ok := decl.(*ast.FuncDecl); ok && fn.Name.Name == "main" && fn.Body != nil {
|
|
return fn.Body
|
|
}
|
|
}
|
|
t.Fatal("func main() not found in main.go")
|
|
return nil
|
|
}
|
|
|
|
// callsInMain returns, in source order, the names of every call in func main() whose function
|
|
// expression is `x.Sel(...)` or `Sel(...)` — enough to identify the wiring calls by name.
|
|
func callsInMain(t *testing.T, body *ast.BlockStmt) []string {
|
|
t.Helper()
|
|
var names []string
|
|
ast.Inspect(body, func(n ast.Node) bool {
|
|
call, ok := n.(*ast.CallExpr)
|
|
if !ok {
|
|
return true
|
|
}
|
|
switch fun := call.Fun.(type) {
|
|
case *ast.SelectorExpr:
|
|
names = append(names, fun.Sel.Name)
|
|
case *ast.Ident:
|
|
names = append(names, fun.Name)
|
|
}
|
|
return true
|
|
})
|
|
return names
|
|
}
|
|
|
|
func indexOfCall(names []string, want string) int {
|
|
for i, n := range names {
|
|
if n == want {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
|
|
// TestMainWiresAppStopRecovery is the Group-I seam test. Comment out the `appStopGuard.Recover()`
|
|
// line in main.go and this fails, where every behavioural test in internal/backup still passes.
|
|
func TestMainWiresAppStopRecovery(t *testing.T) {
|
|
names := callsInMain(t, mainBody(t))
|
|
|
|
if indexOfCall(names, "NewAppStopGuard") < 0 {
|
|
t.Fatal("func main() no longer builds the R-166 app-stop guard — nothing writes or reads the marker")
|
|
}
|
|
if indexOfCall(names, "SetStarter") < 0 {
|
|
t.Fatal("func main() no longer calls SetStarter on the app-stop guard — Recover would find the " +
|
|
"marker and be unable to start anything, leaving every interrupted app down")
|
|
}
|
|
if indexOfCall(names, "Recover") < 0 {
|
|
t.Fatal("func main() no longer calls Recover() on the app-stop guard — apps left stopped by an " +
|
|
"interrupted backup stay down forever (the R-166 defect, un-fixed)")
|
|
}
|
|
if indexOfCall(names, "SetAppStopGuard") < 0 {
|
|
t.Fatal("func main() no longer hands the recovered guard to the backup manager — the manager " +
|
|
"would build a SECOND guard over the same file, i.e. one file with two owners")
|
|
}
|
|
if indexOfCall(names, "SetStopGuard") < 0 {
|
|
t.Fatal("func main() no longer wires the exporter's stop guard — the .fab export path would be " +
|
|
"the one uncovered stop-and-restart site, which is how a reader concludes the class is handled")
|
|
}
|
|
}
|
|
|
|
// TestMainWiresDesiredStateBackfill pins the Part-1.5 call.
|
|
func TestMainWiresDesiredStateBackfill(t *testing.T) {
|
|
if indexOfCall(callsInMain(t, mainBody(t)), "BackfillDesiredState") < 0 {
|
|
t.Fatal("func main() no longer calls BackfillDesiredState — every existing app would stay on " +
|
|
"legacy inference until someone pressed a button on it")
|
|
}
|
|
}
|
|
|
|
// TestAppStopRecoveryPrecedesTheBootReconciler is §8.4's ORDERING requirement, and it is the reason
|
|
// the recovery returns its result instead of pushing it through a notifier seam.
|
|
//
|
|
// The recovery must COMPLETE — not merely be reached — before `go runBootReconcile(...)` is
|
|
// launched. If the boot reconciler ran first it would see an app the marker already explains, list
|
|
// it as an unexplained boot orphan, and one fault would be reported as two.
|
|
func TestAppStopRecoveryPrecedesTheBootReconciler(t *testing.T) {
|
|
names := callsInMain(t, mainBody(t))
|
|
|
|
recover := indexOfCall(names, "Recover")
|
|
bootrecon := indexOfCall(names, "runBootReconcile")
|
|
backfill := indexOfCall(names, "BackfillDesiredState")
|
|
|
|
if recover < 0 || bootrecon < 0 || backfill < 0 {
|
|
t.Fatalf("missing a call: Recover=%d runBootReconcile=%d BackfillDesiredState=%d", recover, bootrecon, backfill)
|
|
}
|
|
if recover >= bootrecon {
|
|
t.Fatal("the app-stop Recover no longer runs BEFORE the boot reconciler is launched — an app " +
|
|
"the marker explains would also be reported as an unexplained boot orphan (§8.4)")
|
|
}
|
|
if backfill >= bootrecon {
|
|
t.Fatal("the desired-state backfill no longer runs BEFORE the boot reconciler — the reconciler " +
|
|
"would decide from intent the backfill had not yet written")
|
|
}
|
|
if recover >= backfill {
|
|
t.Fatal("the backfill no longer runs AFTER the app-stop recovery — an app the recovery just " +
|
|
"restarted would still read as down and be left unrecorded")
|
|
}
|
|
}
|
|
|
|
// TestMainReportsTheInterruptedOperation pins §2.4: the recovery's outcome reaches the operator.
|
|
//
|
|
// The reporting call is deliberately far from the recovery (the notifier does not exist yet at
|
|
// recovery time), which is exactly the distance across which a wiring gets dropped.
|
|
func TestMainReportsTheInterruptedOperation(t *testing.T) {
|
|
body := mainBody(t)
|
|
names := callsInMain(t, body)
|
|
|
|
if indexOfCall(names, "NotifyBackupFailed") < 0 {
|
|
t.Fatal("func main() no longer reports an interrupted app-data operation to the operator — the " +
|
|
"controller died mid-backup and nobody is told (§2.4)")
|
|
}
|
|
// It must be guarded, not unconditional: a box with nothing to recover must not email an operator
|
|
// on every single boot.
|
|
//
|
|
// R-174 STRENGTHENED THIS. `!= nil` alone is no longer sufficient, because Recover now returns a
|
|
// non-nil result for a recovery that merely REFUSED starts (an absent data drive) — the drive
|
|
// gate working as designed. `NotifyBackupFailed` sends `backup_failed`, which is customer-enabled
|
|
// by default (settings.DefaultEnabledEvents), so a nil-only guard would email the customer
|
|
// "A biztonsági mentés sikertelen!" about an app nothing is wrong with. The guard must consult
|
|
// Alarming().
|
|
guardedByNil, guardedByAlarming := false, false
|
|
ast.Inspect(body, func(n ast.Node) bool {
|
|
ifst, ok := n.(*ast.IfStmt)
|
|
if !ok || ifst.Cond == nil {
|
|
return true
|
|
}
|
|
carries := false
|
|
for _, name := range callsInMain(t, ifst.Body) {
|
|
if name == "NotifyBackupFailed" {
|
|
carries = true
|
|
}
|
|
}
|
|
if !carries {
|
|
return true
|
|
}
|
|
// Walk the whole condition: it may be `a != nil && a.Alarming()`.
|
|
ast.Inspect(ifst.Cond, func(c ast.Node) bool {
|
|
switch e := c.(type) {
|
|
case *ast.BinaryExpr:
|
|
if x, ok := e.X.(*ast.Ident); ok && x.Name == "appStopRecovery" && e.Op == token.NEQ {
|
|
guardedByNil = true
|
|
}
|
|
case *ast.CallExpr:
|
|
if sel, ok := e.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Alarming" {
|
|
if x, ok := sel.X.(*ast.Ident); ok && x.Name == "appStopRecovery" {
|
|
guardedByAlarming = true
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
return true
|
|
})
|
|
if !guardedByNil {
|
|
t.Fatal("the interrupted-operation alert is not guarded by `appStopRecovery != nil` — every " +
|
|
"healthy boot would page the operator about a backup that was never interrupted")
|
|
}
|
|
if !guardedByAlarming {
|
|
t.Fatal("the interrupted-operation alert is not guarded by appStopRecovery.Alarming() — a " +
|
|
"recovery that only REFUSED starts (drive absent) would be reported through " +
|
|
"NotifyBackupFailed, a customer-enabled event type, telling the customer their backup " +
|
|
"failed when the drive gate was simply doing its job (R-174)")
|
|
}
|
|
}
|
|
|
|
// --- R-171 seam: the boot drive gate must be WIRED in production -------------------------------
|
|
|
|
// TestMainWiresBootDriveGate is the Group-H seam test. An unwired drive gate is not a crash — it is
|
|
// SILENTLY the pre-v0.190.0 behaviour, which started apps onto absent drives (observed live,
|
|
// audits/DIAG-bootrecon-drive-absent-2026-08-02.md). Every behavioural test in internal/bootrecon
|
|
// still passes with the wiring gone, which is exactly the hole this walks the AST to close.
|
|
//
|
|
// AST, not strings.Contains: a commented-out call still contains the string — the distinction that
|
|
// made a previous version of this project's own seam test pass its red-proof (2026-07-21).
|
|
func TestMainWiresBootDriveGate(t *testing.T) {
|
|
fset := token.NewFileSet()
|
|
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
|
if err != nil {
|
|
t.Fatalf("parse main.go: %v", err)
|
|
}
|
|
|
|
// (a) the settings handle the gate reads is assigned somewhere in main().
|
|
assigned := false
|
|
for _, name := range assignedIdentsIn(mainBody(t)) {
|
|
if name == "bootDriveSettings" {
|
|
assigned = true
|
|
}
|
|
}
|
|
if !assigned {
|
|
t.Fatal("func main() no longer assigns bootDriveSettings — the boot drive gate would read a " +
|
|
"nil settings handle and could not see a disconnected drive")
|
|
}
|
|
|
|
// (b) SetDriveGate is actually called where the reconciler is constructed.
|
|
called := false
|
|
ast.Inspect(f, func(n ast.Node) bool {
|
|
call, ok := n.(*ast.CallExpr)
|
|
if !ok {
|
|
return true
|
|
}
|
|
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "SetDriveGate" {
|
|
called = true
|
|
}
|
|
return true
|
|
})
|
|
if !called {
|
|
t.Fatal("main.go no longer calls SetDriveGate on the boot reconciler — the sweep would start " +
|
|
"apps whose data drive is absent (R-171, a regression observed live on 2026-08-02)")
|
|
}
|
|
}
|
|
|
|
// --- R-174 seam: the app-stop guard's starter must be GATED in production -----------------------
|
|
|
|
// TestMainWiresGatedAppStopStarter pins Part 0's production wiring. `SetStarter(stackMgr)` — the raw
|
|
// manager, which is what shipped in v0.189.0 — compiles, passes every behavioural test in
|
|
// internal/backup (they inject their own gating starter), and silently starts apps onto absent
|
|
// drives at boot. The ONLY thing that distinguishes the fixed wiring from the broken one is the
|
|
// argument at the call site, so that is what this reads.
|
|
//
|
|
// AST, not strings.Contains: a commented-out call still contains the string.
|
|
func TestMainWiresGatedAppStopStarter(t *testing.T) {
|
|
body := mainBody(t)
|
|
|
|
var arg ast.Expr
|
|
found := false
|
|
ast.Inspect(body, func(n ast.Node) bool {
|
|
call, ok := n.(*ast.CallExpr)
|
|
if !ok {
|
|
return true
|
|
}
|
|
sel, ok := call.Fun.(*ast.SelectorExpr)
|
|
if !ok || sel.Sel.Name != "SetStarter" || len(call.Args) != 1 {
|
|
return true
|
|
}
|
|
// Only the app-stop guard's SetStarter, not some other type's.
|
|
if x, ok := sel.X.(*ast.Ident); !ok || x.Name != "appStopGuard" {
|
|
return true
|
|
}
|
|
arg, found = call.Args[0], true
|
|
return false
|
|
})
|
|
if !found {
|
|
t.Fatal("func main() no longer calls appStopGuard.SetStarter — Recover would find the marker " +
|
|
"and be unable to start anything")
|
|
}
|
|
|
|
// The argument must be a gatedAppStopStarter composite literal. A bare identifier (`stackMgr`)
|
|
// is precisely the v0.189.0 defect.
|
|
lit, ok := arg.(*ast.CompositeLit)
|
|
if !ok {
|
|
t.Fatalf("appStopGuard.SetStarter is wired with %T, not a gatedAppStopStarter literal — an "+
|
|
"un-gated starter restarts apps onto MISSING drives at boot (R-174, the R-171 defect one "+
|
|
"path over)", arg)
|
|
}
|
|
id, ok := lit.Type.(*ast.Ident)
|
|
if !ok || id.Name != "gatedAppStopStarter" {
|
|
t.Fatalf("appStopGuard.SetStarter is wired with a %v literal, want gatedAppStopStarter", lit.Type)
|
|
}
|
|
|
|
// And that gate must be a driveStartGate — the SAME predicate the boot sweep uses, so the two
|
|
// cannot disagree about whether an app's drive is available.
|
|
gated := false
|
|
for _, el := range lit.Elts {
|
|
kv, ok := el.(*ast.KeyValueExpr)
|
|
if !ok {
|
|
continue
|
|
}
|
|
k, ok := kv.Key.(*ast.Ident)
|
|
if !ok || k.Name != "gate" {
|
|
continue
|
|
}
|
|
if gl, ok := kv.Value.(*ast.CompositeLit); ok {
|
|
if gid, ok := gl.Type.(*ast.Ident); ok && gid.Name == "driveStartGate" {
|
|
gated = true
|
|
}
|
|
}
|
|
}
|
|
if !gated {
|
|
t.Fatal("the app-stop starter's gate is not a driveStartGate — the crash recovery and the " +
|
|
"boot sweep would answer \"may this app start?\" from two different implementations, " +
|
|
"which is the drift the extraction exists to prevent")
|
|
}
|
|
}
|
|
|
|
// TestBootDriveGateAndAppStopShareTheDrivePredicate pins the OTHER half of the same claim: the boot
|
|
// sweep must keep delegating to driveStartGate rather than growing its own copy of the drive checks.
|
|
//
|
|
// This is the "a comment asserting an invariant needs a test pinning it" rule. The claim — that the
|
|
// two gates cannot disagree — is true only while both call the same code.
|
|
func TestBootDriveGateAndAppStopShareTheDrivePredicate(t *testing.T) {
|
|
fset := token.NewFileSet()
|
|
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
|
if err != nil {
|
|
t.Fatalf("parse main.go: %v", err)
|
|
}
|
|
|
|
var mayStart *ast.FuncDecl
|
|
for _, decl := range f.Decls {
|
|
fn, ok := decl.(*ast.FuncDecl)
|
|
if !ok || fn.Name.Name != "MayStart" || fn.Recv == nil || len(fn.Recv.List) != 1 {
|
|
continue
|
|
}
|
|
if id, ok := fn.Recv.List[0].Type.(*ast.Ident); ok && id.Name == "bootDriveGate" {
|
|
mayStart = fn
|
|
}
|
|
}
|
|
if mayStart == nil {
|
|
t.Fatal("bootDriveGate.MayStart not found in main.go")
|
|
}
|
|
|
|
// It must call through to the shared predicate.
|
|
delegates := false
|
|
ast.Inspect(mayStart.Body, func(n ast.Node) bool {
|
|
call, ok := n.(*ast.CallExpr)
|
|
if !ok {
|
|
return true
|
|
}
|
|
sel, ok := call.Fun.(*ast.SelectorExpr)
|
|
if !ok || sel.Sel.Name != "MayStart" {
|
|
return true
|
|
}
|
|
if x, ok := sel.X.(*ast.SelectorExpr); ok && x.Sel.Name == "drive" {
|
|
delegates = true
|
|
}
|
|
return true
|
|
})
|
|
if !delegates {
|
|
t.Fatal("bootDriveGate.MayStart no longer delegates to the shared driveStartGate — the boot " +
|
|
"sweep and the app-stop crash recovery would each carry their own drive logic, and the " +
|
|
"two can then disagree about whether an app may start (R-174)")
|
|
}
|
|
}
|
|
|
|
// --- R-158 / R-167 seams: both new alerts must be WIRED in production ---------------------------
|
|
|
|
// TestMainWiresTheUnitCaptureAlert pins Part 1's seam. `SetUnitNotify` is nil-safe by design, so an
|
|
// unwired seam is not a crash — it is SILENTLY the pre-v0.191.0 behaviour, in which a per-app Tier-1
|
|
// capture failure is a `[WARN]` line and reaches no hub channel at all. Every behavioural test in
|
|
// internal/backup injects its own callback and passes with the production wiring gone, which is
|
|
// exactly the hole this closes. THIS PROJECT'S COUNT OF "BUILT BUT NEVER WIRED" REACHES FIVE WITH
|
|
// R-158 — the defect being fixed here IS an instance of it.
|
|
func TestMainWiresTheUnitCaptureAlert(t *testing.T) {
|
|
names := callsInMain(t, mainBody(t))
|
|
|
|
if indexOfCall(names, "SetUnitNotify") < 0 {
|
|
t.Fatal("func main() no longer calls backupMgr.SetUnitNotify — a per-app recovery-unit " +
|
|
"capture failure would reach no hub channel, which is R-158 un-fixed (the seam built " +
|
|
"and left disconnected, for the fifth time in this project)")
|
|
}
|
|
if indexOfCall(names, "NotifyRecoveryUnitCaptureFailed") < 0 {
|
|
t.Fatal("main.go no longer calls NotifyRecoveryUnitCaptureFailed — the seam is wired to " +
|
|
"something that pushes no event, which looks identical to a working alert from inside " +
|
|
"internal/backup")
|
|
}
|
|
}
|
|
|
|
// TestMainWiresTheFillWatcher pins Part 2's seam. Three separate things can be dropped and each one
|
|
// silently reverts the customer to "nothing warns before a disk fills": the watcher can go
|
|
// unconstructed, its notify can go unwired (the Watcher is nil-safe), or it can never be scheduled.
|
|
func TestMainWiresTheFillWatcher(t *testing.T) {
|
|
body := mainBody(t)
|
|
names := callsInMain(t, body)
|
|
|
|
if indexOfCall(names, "New") < 0 || !assignsIdent(body, "fillWatcher") {
|
|
t.Fatal("func main() no longer constructs the fill watcher — nothing warns the customer " +
|
|
"before a filesystem fills (R-167, decision D-c's customer half)")
|
|
}
|
|
if indexOfCall(names, "SetNotify") < 0 {
|
|
t.Fatal("func main() no longer calls SetNotify on the fill watcher — the Watcher is nil-safe, " +
|
|
"so it would run the checks, update its state, log, and tell the CUSTOMER nothing")
|
|
}
|
|
|
|
// It must actually be scheduled: a watcher nobody calls is a watcher that never fires.
|
|
scheduled := false
|
|
ast.Inspect(body, func(n ast.Node) bool {
|
|
call, ok := n.(*ast.CallExpr)
|
|
if !ok || len(call.Args) == 0 {
|
|
return true
|
|
}
|
|
sel, ok := call.Fun.(*ast.SelectorExpr)
|
|
if !ok || (sel.Sel.Name != "Daily" && sel.Sel.Name != "Every") {
|
|
return true
|
|
}
|
|
lit, ok := call.Args[0].(*ast.BasicLit)
|
|
if ok && strings.Contains(lit.Value, "fill-watch") {
|
|
scheduled = true
|
|
}
|
|
return true
|
|
})
|
|
if !scheduled {
|
|
t.Fatal("the fill watcher is never registered on the scheduler — it would be constructed, " +
|
|
"wired, and never run, which is indistinguishable from a filesystem that never fills")
|
|
}
|
|
|
|
// It must ALSO run once at startup. Neither `Every` nor `Daily` fires on registration (both wait
|
|
// for their first tick), so a schedule-only wiring means a box that BOOTS with a filesystem
|
|
// already over the line stays silent for up to 24 hours — a real fault visible only after a
|
|
// deadline elapses, which is the R-100 shape. The hub's own checkers leave already-breached keys
|
|
// unseeded at init for exactly this reason.
|
|
if indexOfCall(names, "After") < 0 {
|
|
t.Fatal("nothing delays a startup fill check — see fillWatchStartupDelay")
|
|
}
|
|
startupRun := false
|
|
ast.Inspect(body, func(n ast.Node) bool {
|
|
g, ok := n.(*ast.GoStmt)
|
|
if !ok || g.Call == nil {
|
|
return true
|
|
}
|
|
lit, ok := g.Call.Fun.(*ast.FuncLit)
|
|
if !ok {
|
|
return true
|
|
}
|
|
var sawDelay, sawCheck bool
|
|
ast.Inspect(lit.Body, func(m ast.Node) bool {
|
|
if id, ok := m.(*ast.Ident); ok && id.Name == "fillWatchStartupDelay" {
|
|
sawDelay = true
|
|
}
|
|
if call, ok := m.(*ast.CallExpr); ok {
|
|
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Check" {
|
|
if x, ok := sel.X.(*ast.Ident); ok && x.Name == "fillWatcher" {
|
|
sawCheck = true
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
if sawDelay && sawCheck {
|
|
startupRun = true
|
|
}
|
|
return true
|
|
})
|
|
if !startupRun {
|
|
t.Fatal("the fill watcher never runs at STARTUP — Daily/Every both wait for their first " +
|
|
"tick, so a box that boots with a full disk would not warn for up to 24 hours (the " +
|
|
"R-100 shape: a real fault visible only after a deadline elapses)")
|
|
}
|
|
}
|
|
|
|
// assignsIdent reports whether a block assigns to the named identifier.
|
|
func assignsIdent(body *ast.BlockStmt, want string) bool {
|
|
for _, n := range assignedIdentsIn(body) {
|
|
if n == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// assignedIdentsIn returns the names assigned to in a block (plain `=` and `:=`).
|
|
func assignedIdentsIn(body *ast.BlockStmt) []string {
|
|
var names []string
|
|
ast.Inspect(body, func(n ast.Node) bool {
|
|
as, ok := n.(*ast.AssignStmt)
|
|
if !ok {
|
|
return true
|
|
}
|
|
for _, lhs := range as.Lhs {
|
|
if id, ok := lhs.(*ast.Ident); ok {
|
|
names = append(names, id.Name)
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
return names
|
|
}
|