v0.191.0 — warn before the wall comes down (R-167, R-158, R-174)

R-167: new internal/fillwatch warns the CUSTOMER before a filesystem fills. It emits the PRE-EXISTING disk_warning/disk_critical pair, which was allowlisted, copy'd, default-enabled and checkbox'd with no producer in any repo — the sixth "built but never wired" instance here. Two threshold terms (85% or 5 GiB free; critical 95%/2 GiB) because a percentage alone lies at both ends of this fleet's size range. Edge-triggered on escalation only, state persisted, hysteresis dead zone at 75%/7 GiB pinned by a test. A nil usage read is never a warning and never clears one. Per filesystem, never per app. Daily 03:30, before the nightly app-data legs. R-158: new unitNotify seam fires per app when a Tier-1 recovery-unit capture fails, loop continuing, carrying the target filesystem's used/free bytes. Operator-tier (recovery_unit_capture_failed) — deliberately NOT backup_failed, which is customer-enabled and would email the customer about a failure they cannot act on. D-c overrides R-158's own proposal here. R-174: the app-stop guard no longer starts apps onto MISSING drives — a regression in v0.189.0 code, found by review and closed the same session. SetStarter got the raw stack manager, whose StartStack has no drive gate, and Recover runs at startup. R-171 one path over. bootDriveGate could not be reused whole (its holder #2 is the guard's own marker, and holders #1/#2 read vars assigned after Recover runs), so holder #3 is extracted into a shared driveStartGate with a test pinning the delegation. ErrStartRefused splits a refusal from a failure: both keep the marker, only Failed alarms, because routing a deliberate hold into NotifyBackupFailed is the same false alarm. Tests 1157 -> 1184. All red-proofs demonstrated failing and restored.
2026-08-02 23:18:51 +02:00
parent 95eb5c2c1a
commit cf48214f6c
12 changed files with 1785 additions and 22 deletions
@@ -4,6 +4,7 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
+	"strings"
 	"testing"
 )

@@ -140,31 +141,56 @@ func TestMainReportsTheInterruptedOperation(t *testing.T) {
 	}
 	// It must be guarded, not unconditional: a box with nothing to recover must not email an operator
 	// on every single boot.
-	guarded := false
+	//
+	// R-174 STRENGTHENED THIS. `!= nil` alone is no longer sufficient, because Recover now returns a
+	// non-nil result for a recovery that merely REFUSED starts (an absent data drive) — the drive
+	// gate working as designed. `NotifyBackupFailed` sends `backup_failed`, which is customer-enabled
+	// by default (settings.DefaultEnabledEvents), so a nil-only guard would email the customer
+	// "A biztonsági mentés sikertelen!" about an app nothing is wrong with. The guard must consult
+	// Alarming().
+	guardedByNil, guardedByAlarming := false, false
 	ast.Inspect(body, func(n ast.Node) bool {
 		ifst, ok := n.(*ast.IfStmt)
 		if !ok || ifst.Cond == nil {
 			return true
 		}
-		bin, ok := ifst.Cond.(*ast.BinaryExpr)
-		if !ok {
-			return true
-		}
-		x, ok := bin.X.(*ast.Ident)
-		if !ok || x.Name != "appStopRecovery" {
-			return true
-		}
+		carries := false
 		for _, name := range callsInMain(t, ifst.Body) {
 			if name == "NotifyBackupFailed" {
-				guarded = true
+				carries = true
 			}
 		}
+		if !carries {
+			return true
+		}
+		// Walk the whole condition: it may be `a != nil && a.Alarming()`.
+		ast.Inspect(ifst.Cond, func(c ast.Node) bool {
+			switch e := c.(type) {
+			case *ast.BinaryExpr:
+				if x, ok := e.X.(*ast.Ident); ok && x.Name == "appStopRecovery" && e.Op == token.NEQ {
+					guardedByNil = true
+				}
+			case *ast.CallExpr:
+				if sel, ok := e.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Alarming" {
+					if x, ok := sel.X.(*ast.Ident); ok && x.Name == "appStopRecovery" {
+						guardedByAlarming = true
+					}
+				}
+			}
+			return true
+		})
 		return true
 	})
-	if !guarded {
-		t.Fatal("the interrupted-operation alert is not guarded by `if appStopRecovery != nil` — every " +
+	if !guardedByNil {
+		t.Fatal("the interrupted-operation alert is not guarded by `appStopRecovery != nil` — every " +
 			"healthy boot would page the operator about a backup that was never interrupted")
 	}
+	if !guardedByAlarming {
+		t.Fatal("the interrupted-operation alert is not guarded by appStopRecovery.Alarming() — a " +
+			"recovery that only REFUSED starts (drive absent) would be reported through " +
+			"NotifyBackupFailed, a customer-enabled event type, telling the customer their backup " +
+			"failed when the drive gate was simply doing its job (R-174)")
+	}
 }

 // --- R-171 seam: the boot drive gate must be WIRED in production -------------------------------
@@ -213,6 +239,200 @@ func TestMainWiresBootDriveGate(t *testing.T) {
 	}
 }

+// --- R-174 seam: the app-stop guard's starter must be GATED in production -----------------------
+
+// TestMainWiresGatedAppStopStarter pins Part 0's production wiring. `SetStarter(stackMgr)` — the raw
+// manager, which is what shipped in v0.189.0 — compiles, passes every behavioural test in
+// internal/backup (they inject their own gating starter), and silently starts apps onto absent
+// drives at boot. The ONLY thing that distinguishes the fixed wiring from the broken one is the
+// argument at the call site, so that is what this reads.
+//
+// AST, not strings.Contains: a commented-out call still contains the string.
+func TestMainWiresGatedAppStopStarter(t *testing.T) {
+	body := mainBody(t)
+
+	var arg ast.Expr
+	found := false
+	ast.Inspect(body, func(n ast.Node) bool {
+		call, ok := n.(*ast.CallExpr)
+		if !ok {
+			return true
+		}
+		sel, ok := call.Fun.(*ast.SelectorExpr)
+		if !ok || sel.Sel.Name != "SetStarter" || len(call.Args) != 1 {
+			return true
+		}
+		// Only the app-stop guard's SetStarter, not some other type's.
+		if x, ok := sel.X.(*ast.Ident); !ok || x.Name != "appStopGuard" {
+			return true
+		}
+		arg, found = call.Args[0], true
+		return false
+	})
+	if !found {
+		t.Fatal("func main() no longer calls appStopGuard.SetStarter — Recover would find the marker " +
+			"and be unable to start anything")
+	}
+
+	// The argument must be a gatedAppStopStarter composite literal. A bare identifier (`stackMgr`)
+	// is precisely the v0.189.0 defect.
+	lit, ok := arg.(*ast.CompositeLit)
+	if !ok {
+		t.Fatalf("appStopGuard.SetStarter is wired with %T, not a gatedAppStopStarter literal — an "+
+			"un-gated starter restarts apps onto MISSING drives at boot (R-174, the R-171 defect one "+
+			"path over)", arg)
+	}
+	id, ok := lit.Type.(*ast.Ident)
+	if !ok || id.Name != "gatedAppStopStarter" {
+		t.Fatalf("appStopGuard.SetStarter is wired with a %v literal, want gatedAppStopStarter", lit.Type)
+	}
+
+	// And that gate must be a driveStartGate — the SAME predicate the boot sweep uses, so the two
+	// cannot disagree about whether an app's drive is available.
+	gated := false
+	for _, el := range lit.Elts {
+		kv, ok := el.(*ast.KeyValueExpr)
+		if !ok {
+			continue
+		}
+		k, ok := kv.Key.(*ast.Ident)
+		if !ok || k.Name != "gate" {
+			continue
+		}
+		if gl, ok := kv.Value.(*ast.CompositeLit); ok {
+			if gid, ok := gl.Type.(*ast.Ident); ok && gid.Name == "driveStartGate" {
+				gated = true
+			}
+		}
+	}
+	if !gated {
+		t.Fatal("the app-stop starter's gate is not a driveStartGate — the crash recovery and the " +
+			"boot sweep would answer \"may this app start?\" from two different implementations, " +
+			"which is the drift the extraction exists to prevent")
+	}
+}
+
+// TestBootDriveGateAndAppStopShareTheDrivePredicate pins the OTHER half of the same claim: the boot
+// sweep must keep delegating to driveStartGate rather than growing its own copy of the drive checks.
+//
+// This is the "a comment asserting an invariant needs a test pinning it" rule. The claim — that the
+// two gates cannot disagree — is true only while both call the same code.
+func TestBootDriveGateAndAppStopShareTheDrivePredicate(t *testing.T) {
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "main.go", nil, 0)
+	if err != nil {
+		t.Fatalf("parse main.go: %v", err)
+	}
+
+	var mayStart *ast.FuncDecl
+	for _, decl := range f.Decls {
+		fn, ok := decl.(*ast.FuncDecl)
+		if !ok || fn.Name.Name != "MayStart" || fn.Recv == nil || len(fn.Recv.List) != 1 {
+			continue
+		}
+		if id, ok := fn.Recv.List[0].Type.(*ast.Ident); ok && id.Name == "bootDriveGate" {
+			mayStart = fn
+		}
+	}
+	if mayStart == nil {
+		t.Fatal("bootDriveGate.MayStart not found in main.go")
+	}
+
+	// It must call through to the shared predicate.
+	delegates := false
+	ast.Inspect(mayStart.Body, func(n ast.Node) bool {
+		call, ok := n.(*ast.CallExpr)
+		if !ok {
+			return true
+		}
+		sel, ok := call.Fun.(*ast.SelectorExpr)
+		if !ok || sel.Sel.Name != "MayStart" {
+			return true
+		}
+		if x, ok := sel.X.(*ast.SelectorExpr); ok && x.Sel.Name == "drive" {
+			delegates = true
+		}
+		return true
+	})
+	if !delegates {
+		t.Fatal("bootDriveGate.MayStart no longer delegates to the shared driveStartGate — the boot " +
+			"sweep and the app-stop crash recovery would each carry their own drive logic, and the " +
+			"two can then disagree about whether an app may start (R-174)")
+	}
+}
+
+// --- R-158 / R-167 seams: both new alerts must be WIRED in production ---------------------------
+
+// TestMainWiresTheUnitCaptureAlert pins Part 1's seam. `SetUnitNotify` is nil-safe by design, so an
+// unwired seam is not a crash — it is SILENTLY the pre-v0.191.0 behaviour, in which a per-app Tier-1
+// capture failure is a `[WARN]` line and reaches no hub channel at all. Every behavioural test in
+// internal/backup injects its own callback and passes with the production wiring gone, which is
+// exactly the hole this closes. THIS PROJECT'S COUNT OF "BUILT BUT NEVER WIRED" REACHES FIVE WITH
+// R-158 — the defect being fixed here IS an instance of it.
+func TestMainWiresTheUnitCaptureAlert(t *testing.T) {
+	names := callsInMain(t, mainBody(t))
+
+	if indexOfCall(names, "SetUnitNotify") < 0 {
+		t.Fatal("func main() no longer calls backupMgr.SetUnitNotify — a per-app recovery-unit " +
+			"capture failure would reach no hub channel, which is R-158 un-fixed (the seam built " +
+			"and left disconnected, for the fifth time in this project)")
+	}
+	if indexOfCall(names, "NotifyRecoveryUnitCaptureFailed") < 0 {
+		t.Fatal("main.go no longer calls NotifyRecoveryUnitCaptureFailed — the seam is wired to " +
+			"something that pushes no event, which looks identical to a working alert from inside " +
+			"internal/backup")
+	}
+}
+
+// TestMainWiresTheFillWatcher pins Part 2's seam. Three separate things can be dropped and each one
+// silently reverts the customer to "nothing warns before a disk fills": the watcher can go
+// unconstructed, its notify can go unwired (the Watcher is nil-safe), or it can never be scheduled.
+func TestMainWiresTheFillWatcher(t *testing.T) {
+	body := mainBody(t)
+	names := callsInMain(t, body)
+
+	if indexOfCall(names, "New") < 0 || !assignsIdent(body, "fillWatcher") {
+		t.Fatal("func main() no longer constructs the fill watcher — nothing warns the customer " +
+			"before a filesystem fills (R-167, decision D-c's customer half)")
+	}
+	if indexOfCall(names, "SetNotify") < 0 {
+		t.Fatal("func main() no longer calls SetNotify on the fill watcher — the Watcher is nil-safe, " +
+			"so it would run the checks, update its state, log, and tell the CUSTOMER nothing")
+	}
+
+	// It must actually be scheduled: a watcher nobody calls is a watcher that never fires.
+	scheduled := false
+	ast.Inspect(body, func(n ast.Node) bool {
+		call, ok := n.(*ast.CallExpr)
+		if !ok || len(call.Args) == 0 {
+			return true
+		}
+		sel, ok := call.Fun.(*ast.SelectorExpr)
+		if !ok || (sel.Sel.Name != "Daily" && sel.Sel.Name != "Every") {
+			return true
+		}
+		lit, ok := call.Args[0].(*ast.BasicLit)
+		if ok && strings.Contains(lit.Value, "fill-watch") {
+			scheduled = true
+		}
+		return true
+	})
+	if !scheduled {
+		t.Fatal("the fill watcher is never registered on the scheduler — it would be constructed, " +
+			"wired, and never run, which is indistinguishable from a filesystem that never fills")
+	}
+}
+
+// assignsIdent reports whether a block assigns to the named identifier.
+func assignsIdent(body *ast.BlockStmt, want string) bool {
+	for _, n := range assignedIdentsIn(body) {
+		if n == want {
+			return true
+		}
+	}
+	return false
+}
+
 // assignedIdentsIn returns the names assigned to in a block (plain `=` and `:=`).
 func assignedIdentsIn(body *ast.BlockStmt) []string {
 	var names []string