R-521: an app a lost drive stopped no longer mails app_start_failed one by one

One unplugged drive sent storage_disconnected plus an app_start_failed per app on it (five operator
mails). The dead-app job now hands the notifier the apps of a disconnected drive (its recorded
StoppedStacks plus apps whose HDD_PATH is that drive) as not down; the storage event already names
them. The dashboard's dead list is unchanged. The hub-side cooldown (F6/F7) and a household mail
for a lost drive are not in this change.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-05 21:52:26 +02:00
parent f885100d29
commit 800b32ceec
2 changed files with 154 additions and 1 deletions
+60 -1
View File
@@ -930,7 +930,10 @@ func main() {
}
dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop, appStopGuard, backupMgr)
alertMgr.SetDeadAppAlerts(dead)
notifier.NotifyAppStartFailures(states)
// R-521: an app a lost drive stopped is already named in that drive's storage_disconnected
// event; its own app_start_failed adds nothing (one unplug mailed the operator five times). The
// dashboard's dead-app list (`dead`) is NOT masked — only the per-app mail.
notifier.NotifyAppStartFailures(maskDriveLost(states, driveLostApps(sett, stackMgr)))
// R-514: a worker OOM-killed inside a running container leaves the app „Fut". Surface it.
ooms, oerr := stackMgr.ScanOOMKilled()
if oerr != nil {
@@ -2678,6 +2681,62 @@ type updateHeldLister interface {
UpdateHeldStacks() map[string]bool
}
// driveLostApps (R-521) is the set of apps stopped because their drive is flagged disconnected: the
// stacks the disconnect recorded (StoppedStacks) plus any deployed app whose app.yaml HDD_PATH is that
// drive. Nil (no allocation, no app.yaml reads) when no drive is disconnected — the common case.
func driveLostApps(sett *settings.Settings, mgr hddPathReader) map[string]bool {
if sett == nil {
return nil
}
lost := map[string]bool{}
var out map[string]bool
for _, sp := range sett.GetStoragePaths() {
if !sp.Disconnected {
continue
}
lost[sp.Path] = true
if out == nil {
out = map[string]bool{}
}
for _, n := range sp.StoppedStacks {
out[n] = true
}
}
if len(lost) == 0 || mgr == nil {
return out
}
for _, st := range mgr.GetStacks() {
if !st.Deployed {
continue
}
if cfg := mgr.LoadAppConfigByName(st.Name); cfg != nil && lost[cfg.Env["HDD_PATH"]] {
out[st.Name] = true
}
}
return out
}
// hddPathReader is the slice of *stacks.Manager driveLostApps reads (a seam for the test).
type hddPathReader interface {
GetStacks() []stacks.Stack
LoadAppConfigByName(name string) *stacks.AppConfig
}
// maskDriveLost reports the drive-lost apps as not down to the NOTIFIER only (see R-521 at the call).
func maskDriveLost(states []notify.AppRunState, lost map[string]bool) []notify.AppRunState {
if len(lost) == 0 {
return states
}
out := make([]notify.AppRunState, len(states))
copy(out, states)
for i := range out {
if out[i].Down && lost[out[i].Name] {
out[i].Down = false
}
}
return out
}
// updateHeldSet is nil-safe over a nil interface (a box with backup disabled has no holds).
func updateHeldSet(l updateHeldLister) map[string]bool {
if l == nil {