Found live on 9201: neither sched.Daily nor sched.Every fires on registration, so a box booting with a filesystem already over the line would stay silent for up to 24h — the R-100 shape, and the same gap the hub's own checkers avoid by leaving already-breached keys unseeded at init. The watcher now runs once 90s after startup as well. Safe because the check is edge-triggered against persisted state: an already-warned filesystem stays silent. The delay lets mounts settle so a drive still returning reads as unreadable and is skipped rather than warned about. Pinned by an AST assertion — the schedule registration alone no longer satisfies the test.
This commit is contained in:
@@ -421,6 +421,49 @@ func TestMainWiresTheFillWatcher(t *testing.T) {
|
||||
t.Fatal("the fill watcher is never registered on the scheduler — it would be constructed, " +
|
||||
"wired, and never run, which is indistinguishable from a filesystem that never fills")
|
||||
}
|
||||
|
||||
// It must ALSO run once at startup. Neither `Every` nor `Daily` fires on registration (both wait
|
||||
// for their first tick), so a schedule-only wiring means a box that BOOTS with a filesystem
|
||||
// already over the line stays silent for up to 24 hours — a real fault visible only after a
|
||||
// deadline elapses, which is the R-100 shape. The hub's own checkers leave already-breached keys
|
||||
// unseeded at init for exactly this reason.
|
||||
if indexOfCall(names, "After") < 0 {
|
||||
t.Fatal("nothing delays a startup fill check — see fillWatchStartupDelay")
|
||||
}
|
||||
startupRun := false
|
||||
ast.Inspect(body, func(n ast.Node) bool {
|
||||
g, ok := n.(*ast.GoStmt)
|
||||
if !ok || g.Call == nil {
|
||||
return true
|
||||
}
|
||||
lit, ok := g.Call.Fun.(*ast.FuncLit)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
var sawDelay, sawCheck bool
|
||||
ast.Inspect(lit.Body, func(m ast.Node) bool {
|
||||
if id, ok := m.(*ast.Ident); ok && id.Name == "fillWatchStartupDelay" {
|
||||
sawDelay = true
|
||||
}
|
||||
if call, ok := m.(*ast.CallExpr); ok {
|
||||
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Check" {
|
||||
if x, ok := sel.X.(*ast.Ident); ok && x.Name == "fillWatcher" {
|
||||
sawCheck = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if sawDelay && sawCheck {
|
||||
startupRun = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !startupRun {
|
||||
t.Fatal("the fill watcher never runs at STARTUP — Daily/Every both wait for their first " +
|
||||
"tick, so a box that boots with a full disk would not warn for up to 24 hours (the " +
|
||||
"R-100 shape: a real fault visible only after a deadline elapses)")
|
||||
}
|
||||
}
|
||||
|
||||
// assignsIdent reports whether a block assigns to the named identifier.
|
||||
|
||||
@@ -828,6 +828,27 @@ func main() {
|
||||
}
|
||||
sched.Daily("fill-watch", "03:30", func(ctx context.Context) error { return fillWatcher.Check() })
|
||||
|
||||
// AND ONCE SHORTLY AFTER STARTUP. A box that BOOTS with a filesystem already over the line must
|
||||
// warn now, not up to 24 hours later — that is the R-100 shape, a real fault visible only after a
|
||||
// deadline elapses. The hub's own checkers handle exactly this case deliberately ("already-breached
|
||||
// keys are left UNSEEDED at init so their first Check emits" — the F2 lesson, storage_fill.go), and
|
||||
// a daily-only schedule here would have been the same gap one component over.
|
||||
//
|
||||
// It is SAFE because the check is edge-triggered against PERSISTED state: a filesystem the customer
|
||||
// has already been warned about stays silent, so this adds a warning only where one is genuinely
|
||||
// owed. The delay lets mounts settle and the drive gate tick first, so a drive that is merely slow
|
||||
// to come back reads as unreadable (§8.4, skipped) rather than as anything at all.
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(fillWatchStartupDelay):
|
||||
}
|
||||
if err := fillWatcher.Check(); err != nil {
|
||||
logger.Printf("[WARN] [fillwatch] startup check failed: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// --- Central hub reporting schedule ---
|
||||
if hubPusher != nil {
|
||||
if cfg.Hub.Enabled {
|
||||
@@ -1528,6 +1549,11 @@ func (s gatedAppStopStarter) StartStack(name string) error {
|
||||
return s.inner.StartStack(name)
|
||||
}
|
||||
|
||||
// fillWatchStartupDelay is how long the startup fill check waits before its single run. Long enough
|
||||
// for the drive gate's first reconcile and for mounts to settle — a drive still coming back must read
|
||||
// as unreadable (and be skipped, §8.4), never as a filesystem worth warning about.
|
||||
const fillWatchStartupDelay = 90 * time.Second
|
||||
|
||||
// fillTargets is §8.1's watch list: the app-data volume, the system-data volume, and every
|
||||
// registered drive. Resolved at CHECK time, not at startup, so a drive added or decommissioned
|
||||
// between checks is picked up without a controller restart.
|
||||
|
||||
Reference in New Issue
Block a user