Found live on 9201: neither sched.Daily nor sched.Every fires on registration, so a box booting with a filesystem already over the line would stay silent for up to 24h — the R-100 shape, and the same gap the hub's own checkers avoid by leaving already-breached keys unseeded at init. The watcher now runs once 90s after startup as well. Safe because the check is edge-triggered against persisted state: an already-warned filesystem stays silent. The delay lets mounts settle so a drive still returning reads as unreadable and is skipped rather than warned about. Pinned by an AST assertion — the schedule registration alone no longer satisfies the test.
This commit is contained in:
@@ -1,5 +1,25 @@
|
|||||||
## Changelog
|
## Changelog
|
||||||
|
|
||||||
|
### v0.191.1 — the fill check also runs at startup (2026-08-02, R-167) — MinAgent: none
|
||||||
|
|
||||||
|
**Found while live-validating v0.191.0 on guest 9201: the fill check was reachable only on its daily
|
||||||
|
schedule, and neither `sched.Daily` nor `sched.Every` fires on registration — both wait for their
|
||||||
|
first tick.** So a box that BOOTS with a filesystem already over the line would have stayed silent for
|
||||||
|
up to 24 hours. That is the R-100 shape — a real fault visible only after a deadline elapses — and the
|
||||||
|
hub's own checkers handle exactly this case deliberately, leaving already-breached keys unseeded at
|
||||||
|
init so their first `Check` emits (the F2 lesson, `monitor/storage_fill.go`). A daily-only schedule
|
||||||
|
here would have been the same gap one component over.
|
||||||
|
|
||||||
|
The watcher now also runs **once, 90 s after startup**. The delay lets mounts settle and the drive
|
||||||
|
gate tick first, so a drive still coming back reads as unreadable and is skipped (§8.4) rather than
|
||||||
|
warned about. It is safe to add because the check is edge-triggered against PERSISTED state: a
|
||||||
|
filesystem the customer has already been warned about stays silent, so this adds a warning only where
|
||||||
|
one is genuinely owed. Pinned by an AST assertion in `TestMainWiresTheFillWatcher` — the schedule
|
||||||
|
registration alone no longer satisfies it.
|
||||||
|
|
||||||
|
**Disclosure:** this also made the flow live-validatable at all. There is still no operator-triggerable
|
||||||
|
"run the fill check now" path; that is recorded as an observation, not fixed here.
|
||||||
|
|
||||||
### v0.191.0 — warn before the wall comes down (2026-08-02, R-167 · R-158 · R-174) — MinAgent: none
|
### v0.191.0 — warn before the wall comes down (2026-08-02, R-167 · R-158 · R-174) — MinAgent: none
|
||||||
|
|
||||||
**Storage monitoring and backup alerts, decision D-c, landing BEFORE the `mp1`→`mp0` merge (D-a /
|
**Storage monitoring and backup alerts, decision D-c, landing BEFORE the `mp1`→`mp0` merge (D-a /
|
||||||
|
|||||||
@@ -421,6 +421,49 @@ func TestMainWiresTheFillWatcher(t *testing.T) {
|
|||||||
t.Fatal("the fill watcher is never registered on the scheduler — it would be constructed, " +
|
t.Fatal("the fill watcher is never registered on the scheduler — it would be constructed, " +
|
||||||
"wired, and never run, which is indistinguishable from a filesystem that never fills")
|
"wired, and never run, which is indistinguishable from a filesystem that never fills")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// It must ALSO run once at startup. Neither `Every` nor `Daily` fires on registration (both wait
|
||||||
|
// for their first tick), so a schedule-only wiring means a box that BOOTS with a filesystem
|
||||||
|
// already over the line stays silent for up to 24 hours — a real fault visible only after a
|
||||||
|
// deadline elapses, which is the R-100 shape. The hub's own checkers leave already-breached keys
|
||||||
|
// unseeded at init for exactly this reason.
|
||||||
|
if indexOfCall(names, "After") < 0 {
|
||||||
|
t.Fatal("nothing delays a startup fill check — see fillWatchStartupDelay")
|
||||||
|
}
|
||||||
|
startupRun := false
|
||||||
|
ast.Inspect(body, func(n ast.Node) bool {
|
||||||
|
g, ok := n.(*ast.GoStmt)
|
||||||
|
if !ok || g.Call == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
lit, ok := g.Call.Fun.(*ast.FuncLit)
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
var sawDelay, sawCheck bool
|
||||||
|
ast.Inspect(lit.Body, func(m ast.Node) bool {
|
||||||
|
if id, ok := m.(*ast.Ident); ok && id.Name == "fillWatchStartupDelay" {
|
||||||
|
sawDelay = true
|
||||||
|
}
|
||||||
|
if call, ok := m.(*ast.CallExpr); ok {
|
||||||
|
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Check" {
|
||||||
|
if x, ok := sel.X.(*ast.Ident); ok && x.Name == "fillWatcher" {
|
||||||
|
sawCheck = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
if sawDelay && sawCheck {
|
||||||
|
startupRun = true
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
if !startupRun {
|
||||||
|
t.Fatal("the fill watcher never runs at STARTUP — Daily/Every both wait for their first " +
|
||||||
|
"tick, so a box that boots with a full disk would not warn for up to 24 hours (the " +
|
||||||
|
"R-100 shape: a real fault visible only after a deadline elapses)")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// assignsIdent reports whether a block assigns to the named identifier.
|
// assignsIdent reports whether a block assigns to the named identifier.
|
||||||
|
|||||||
@@ -828,6 +828,27 @@ func main() {
|
|||||||
}
|
}
|
||||||
sched.Daily("fill-watch", "03:30", func(ctx context.Context) error { return fillWatcher.Check() })
|
sched.Daily("fill-watch", "03:30", func(ctx context.Context) error { return fillWatcher.Check() })
|
||||||
|
|
||||||
|
// AND ONCE SHORTLY AFTER STARTUP. A box that BOOTS with a filesystem already over the line must
|
||||||
|
// warn now, not up to 24 hours later — that is the R-100 shape, a real fault visible only after a
|
||||||
|
// deadline elapses. The hub's own checkers handle exactly this case deliberately ("already-breached
|
||||||
|
// keys are left UNSEEDED at init so their first Check emits" — the F2 lesson, storage_fill.go), and
|
||||||
|
// a daily-only schedule here would have been the same gap one component over.
|
||||||
|
//
|
||||||
|
// It is SAFE because the check is edge-triggered against PERSISTED state: a filesystem the customer
|
||||||
|
// has already been warned about stays silent, so this adds a warning only where one is genuinely
|
||||||
|
// owed. The delay lets mounts settle and the drive gate tick first, so a drive that is merely slow
|
||||||
|
// to come back reads as unreadable (§8.4, skipped) rather than as anything at all.
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(fillWatchStartupDelay):
|
||||||
|
}
|
||||||
|
if err := fillWatcher.Check(); err != nil {
|
||||||
|
logger.Printf("[WARN] [fillwatch] startup check failed: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
// --- Central hub reporting schedule ---
|
// --- Central hub reporting schedule ---
|
||||||
if hubPusher != nil {
|
if hubPusher != nil {
|
||||||
if cfg.Hub.Enabled {
|
if cfg.Hub.Enabled {
|
||||||
@@ -1528,6 +1549,11 @@ func (s gatedAppStopStarter) StartStack(name string) error {
|
|||||||
return s.inner.StartStack(name)
|
return s.inner.StartStack(name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fillWatchStartupDelay is how long the startup fill check waits before its single run. Long enough
|
||||||
|
// for the drive gate's first reconcile and for mounts to settle — a drive still coming back must read
|
||||||
|
// as unreadable (and be skipped, §8.4), never as a filesystem worth warning about.
|
||||||
|
const fillWatchStartupDelay = 90 * time.Second
|
||||||
|
|
||||||
// fillTargets is §8.1's watch list: the app-data volume, the system-data volume, and every
|
// fillTargets is §8.1's watch list: the app-data volume, the system-data volume, and every
|
||||||
// registered drive. Resolved at CHECK time, not at startup, so a drive added or decommissioned
|
// registered drive. Resolved at CHECK time, not at startup, so a drive added or decommissioned
|
||||||
// between checks is picked up without a controller restart.
|
// between checks is picked up without a controller restart.
|
||||||
|
|||||||
Reference in New Issue
Block a user