From 5adae4dad985985b12b7b586262651181be3621e Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 2 Aug 2026 23:27:35 +0200 Subject: [PATCH] =?UTF-8?q?v0.191.1=20=E2=80=94=20the=20fill=20check=20als?= =?UTF-8?q?o=20runs=20at=20startup=20(R-167)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found live on 9201: neither sched.Daily nor sched.Every fires on registration, so a box booting with a filesystem already over the line would stay silent for up to 24h — the R-100 shape, and the same gap the hub's own checkers avoid by leaving already-breached keys unseeded at init. The watcher now runs once 90s after startup as well. Safe because the check is edge-triggered against persisted state: an already-warned filesystem stays silent. The delay lets mounts settle so a drive still returning reads as unreadable and is skipped rather than warned about. Pinned by an AST assertion — the schedule registration alone no longer satisfies the test. --- CHANGELOG.md | 20 +++++++++ .../cmd/controller/appstop_wiring_test.go | 43 +++++++++++++++++++ controller/cmd/controller/main.go | 26 +++++++++++ 3 files changed, 89 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab1c5e3..630c3c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ ## Changelog +### v0.191.1 — the fill check also runs at startup (2026-08-02, R-167) — MinAgent: none + +**Found while live-validating v0.191.0 on guest 9201: the fill check was reachable only on its daily +schedule, and neither `sched.Daily` nor `sched.Every` fires on registration — both wait for their +first tick.** So a box that BOOTS with a filesystem already over the line would have stayed silent for +up to 24 hours. That is the R-100 shape — a real fault visible only after a deadline elapses — and the +hub's own checkers handle exactly this case deliberately, leaving already-breached keys unseeded at +init so their first `Check` emits (the F2 lesson, `monitor/storage_fill.go`). A daily-only schedule +here would have been the same gap one component over. + +The watcher now also runs **once, 90 s after startup**. The delay lets mounts settle and the drive +gate tick first, so a drive still coming back reads as unreadable and is skipped (§8.4) rather than +warned about. It is safe to add because the check is edge-triggered against PERSISTED state: a +filesystem the customer has already been warned about stays silent, so this adds a warning only where +one is genuinely owed. Pinned by an AST assertion in `TestMainWiresTheFillWatcher` — the schedule +registration alone no longer satisfies it. + +**Disclosure:** this also made the flow live-validatable at all. There is still no operator-triggerable +"run the fill check now" path; that is recorded as an observation, not fixed here. + ### v0.191.0 — warn before the wall comes down (2026-08-02, R-167 · R-158 · R-174) — MinAgent: none **Storage monitoring and backup alerts, decision D-c, landing BEFORE the `mp1`→`mp0` merge (D-a / diff --git a/controller/cmd/controller/appstop_wiring_test.go b/controller/cmd/controller/appstop_wiring_test.go index 05d54f7..755cda7 100644 --- a/controller/cmd/controller/appstop_wiring_test.go +++ b/controller/cmd/controller/appstop_wiring_test.go @@ -421,6 +421,49 @@ func TestMainWiresTheFillWatcher(t *testing.T) { t.Fatal("the fill watcher is never registered on the scheduler — it would be constructed, " + "wired, and never run, which is indistinguishable from a filesystem that never fills") } + + // It must ALSO run once at startup. Neither `Every` nor `Daily` fires on registration (both wait + // for their first tick), so a schedule-only wiring means a box that BOOTS with a filesystem + // already over the line stays silent for up to 24 hours — a real fault visible only after a + // deadline elapses, which is the R-100 shape. The hub's own checkers leave already-breached keys + // unseeded at init for exactly this reason. + if indexOfCall(names, "After") < 0 { + t.Fatal("nothing delays a startup fill check — see fillWatchStartupDelay") + } + startupRun := false + ast.Inspect(body, func(n ast.Node) bool { + g, ok := n.(*ast.GoStmt) + if !ok || g.Call == nil { + return true + } + lit, ok := g.Call.Fun.(*ast.FuncLit) + if !ok { + return true + } + var sawDelay, sawCheck bool + ast.Inspect(lit.Body, func(m ast.Node) bool { + if id, ok := m.(*ast.Ident); ok && id.Name == "fillWatchStartupDelay" { + sawDelay = true + } + if call, ok := m.(*ast.CallExpr); ok { + if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Check" { + if x, ok := sel.X.(*ast.Ident); ok && x.Name == "fillWatcher" { + sawCheck = true + } + } + } + return true + }) + if sawDelay && sawCheck { + startupRun = true + } + return true + }) + if !startupRun { + t.Fatal("the fill watcher never runs at STARTUP — Daily/Every both wait for their first " + + "tick, so a box that boots with a full disk would not warn for up to 24 hours (the " + + "R-100 shape: a real fault visible only after a deadline elapses)") + } } // assignsIdent reports whether a block assigns to the named identifier. diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 6f166fe..a6e17ed 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -828,6 +828,27 @@ func main() { } sched.Daily("fill-watch", "03:30", func(ctx context.Context) error { return fillWatcher.Check() }) + // AND ONCE SHORTLY AFTER STARTUP. A box that BOOTS with a filesystem already over the line must + // warn now, not up to 24 hours later — that is the R-100 shape, a real fault visible only after a + // deadline elapses. The hub's own checkers handle exactly this case deliberately ("already-breached + // keys are left UNSEEDED at init so their first Check emits" — the F2 lesson, storage_fill.go), and + // a daily-only schedule here would have been the same gap one component over. + // + // It is SAFE because the check is edge-triggered against PERSISTED state: a filesystem the customer + // has already been warned about stays silent, so this adds a warning only where one is genuinely + // owed. The delay lets mounts settle and the drive gate tick first, so a drive that is merely slow + // to come back reads as unreadable (§8.4, skipped) rather than as anything at all. + go func() { + select { + case <-ctx.Done(): + return + case <-time.After(fillWatchStartupDelay): + } + if err := fillWatcher.Check(); err != nil { + logger.Printf("[WARN] [fillwatch] startup check failed: %v", err) + } + }() + // --- Central hub reporting schedule --- if hubPusher != nil { if cfg.Hub.Enabled { @@ -1528,6 +1549,11 @@ func (s gatedAppStopStarter) StartStack(name string) error { return s.inner.StartStack(name) } +// fillWatchStartupDelay is how long the startup fill check waits before its single run. Long enough +// for the drive gate's first reconcile and for mounts to settle — a drive still coming back must read +// as unreadable (and be skipped, §8.4), never as a filesystem worth warning about. +const fillWatchStartupDelay = 90 * time.Second + // fillTargets is §8.1's watch list: the app-data volume, the system-data volume, and every // registered drive. Resolved at CHECK time, not at startup, so a drive added or decommissioned // between checks is picked up without a controller restart.