v0.191.1 — the fill check also runs at startup (R-167)
gates / gates (push) Successful in 9s

Found live on 9201: neither sched.Daily nor sched.Every fires on
registration, so a box booting with a filesystem already over the line
would stay silent for up to 24h — the R-100 shape, and the same gap the
hub's own checkers avoid by leaving already-breached keys unseeded at init.

The watcher now runs once 90s after startup as well. Safe because the check
is edge-triggered against persisted state: an already-warned filesystem
stays silent. The delay lets mounts settle so a drive still returning reads
as unreadable and is skipped rather than warned about. Pinned by an AST
assertion — the schedule registration alone no longer satisfies the test.
This commit is contained in:
2026-08-02 23:27:35 +02:00
parent cf48214f6c
commit 5adae4dad9
3 changed files with 89 additions and 0 deletions
@@ -421,6 +421,49 @@ func TestMainWiresTheFillWatcher(t *testing.T) {
t.Fatal("the fill watcher is never registered on the scheduler — it would be constructed, " +
"wired, and never run, which is indistinguishable from a filesystem that never fills")
}
// It must ALSO run once at startup. Neither `Every` nor `Daily` fires on registration (both wait
// for their first tick), so a schedule-only wiring means a box that BOOTS with a filesystem
// already over the line stays silent for up to 24 hours — a real fault visible only after a
// deadline elapses, which is the R-100 shape. The hub's own checkers leave already-breached keys
// unseeded at init for exactly this reason.
if indexOfCall(names, "After") < 0 {
t.Fatal("nothing delays a startup fill check — see fillWatchStartupDelay")
}
startupRun := false
ast.Inspect(body, func(n ast.Node) bool {
g, ok := n.(*ast.GoStmt)
if !ok || g.Call == nil {
return true
}
lit, ok := g.Call.Fun.(*ast.FuncLit)
if !ok {
return true
}
var sawDelay, sawCheck bool
ast.Inspect(lit.Body, func(m ast.Node) bool {
if id, ok := m.(*ast.Ident); ok && id.Name == "fillWatchStartupDelay" {
sawDelay = true
}
if call, ok := m.(*ast.CallExpr); ok {
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Check" {
if x, ok := sel.X.(*ast.Ident); ok && x.Name == "fillWatcher" {
sawCheck = true
}
}
}
return true
})
if sawDelay && sawCheck {
startupRun = true
}
return true
})
if !startupRun {
t.Fatal("the fill watcher never runs at STARTUP — Daily/Every both wait for their first " +
"tick, so a box that boots with a full disk would not warn for up to 24 hours (the " +
"R-100 shape: a real fault visible only after a deadline elapses)")
}
}
// assignsIdent reports whether a block assigns to the named identifier.
+26
View File
@@ -828,6 +828,27 @@ func main() {
}
sched.Daily("fill-watch", "03:30", func(ctx context.Context) error { return fillWatcher.Check() })
// AND ONCE SHORTLY AFTER STARTUP. A box that BOOTS with a filesystem already over the line must
// warn now, not up to 24 hours later — that is the R-100 shape, a real fault visible only after a
// deadline elapses. The hub's own checkers handle exactly this case deliberately ("already-breached
// keys are left UNSEEDED at init so their first Check emits" — the F2 lesson, storage_fill.go), and
// a daily-only schedule here would have been the same gap one component over.
//
// It is SAFE because the check is edge-triggered against PERSISTED state: a filesystem the customer
// has already been warned about stays silent, so this adds a warning only where one is genuinely
// owed. The delay lets mounts settle and the drive gate tick first, so a drive that is merely slow
// to come back reads as unreadable (§8.4, skipped) rather than as anything at all.
go func() {
select {
case <-ctx.Done():
return
case <-time.After(fillWatchStartupDelay):
}
if err := fillWatcher.Check(); err != nil {
logger.Printf("[WARN] [fillwatch] startup check failed: %v", err)
}
}()
// --- Central hub reporting schedule ---
if hubPusher != nil {
if cfg.Hub.Enabled {
@@ -1528,6 +1549,11 @@ func (s gatedAppStopStarter) StartStack(name string) error {
return s.inner.StartStack(name)
}
// fillWatchStartupDelay is how long the startup fill check waits before its single run. Long enough
// for the drive gate's first reconcile and for mounts to settle — a drive still coming back must read
// as unreadable (and be skipped, §8.4), never as a filesystem worth warning about.
const fillWatchStartupDelay = 90 * time.Second
// fillTargets is §8.1's watch list: the app-data volume, the system-data volume, and every
// registered drive. Resolved at CHECK time, not at startup, so a drive added or decommissioned
// between checks is picked up without a controller restart.