R-521: an app a lost drive stopped no longer mails app_start_failed one by one
One unplugged drive sent storage_disconnected plus an app_start_failed per app on it (five operator mails). The dead-app job now hands the notifier the apps of a disconnected drive (its recorded StoppedStacks plus apps whose HDD_PATH is that drive) as not down; the storage event already names them. The dashboard's dead list is unchanged. The hub-side cooldown (F6/F7) and a household mail for a lost drive are not in this change. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -930,7 +930,10 @@ func main() {
|
||||
}
|
||||
dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop, appStopGuard, backupMgr)
|
||||
alertMgr.SetDeadAppAlerts(dead)
|
||||
notifier.NotifyAppStartFailures(states)
|
||||
// R-521: an app a lost drive stopped is already named in that drive's storage_disconnected
|
||||
// event; its own app_start_failed adds nothing (one unplug mailed the operator five times). The
|
||||
// dashboard's dead-app list (`dead`) is NOT masked — only the per-app mail.
|
||||
notifier.NotifyAppStartFailures(maskDriveLost(states, driveLostApps(sett, stackMgr)))
|
||||
// R-514: a worker OOM-killed inside a running container leaves the app „Fut". Surface it.
|
||||
ooms, oerr := stackMgr.ScanOOMKilled()
|
||||
if oerr != nil {
|
||||
@@ -2678,6 +2681,62 @@ type updateHeldLister interface {
|
||||
UpdateHeldStacks() map[string]bool
|
||||
}
|
||||
|
||||
// driveLostApps (R-521) is the set of apps stopped because their drive is flagged disconnected: the
|
||||
// stacks the disconnect recorded (StoppedStacks) plus any deployed app whose app.yaml HDD_PATH is that
|
||||
// drive. Nil (no allocation, no app.yaml reads) when no drive is disconnected — the common case.
|
||||
func driveLostApps(sett *settings.Settings, mgr hddPathReader) map[string]bool {
|
||||
if sett == nil {
|
||||
return nil
|
||||
}
|
||||
lost := map[string]bool{}
|
||||
var out map[string]bool
|
||||
for _, sp := range sett.GetStoragePaths() {
|
||||
if !sp.Disconnected {
|
||||
continue
|
||||
}
|
||||
lost[sp.Path] = true
|
||||
if out == nil {
|
||||
out = map[string]bool{}
|
||||
}
|
||||
for _, n := range sp.StoppedStacks {
|
||||
out[n] = true
|
||||
}
|
||||
}
|
||||
if len(lost) == 0 || mgr == nil {
|
||||
return out
|
||||
}
|
||||
for _, st := range mgr.GetStacks() {
|
||||
if !st.Deployed {
|
||||
continue
|
||||
}
|
||||
if cfg := mgr.LoadAppConfigByName(st.Name); cfg != nil && lost[cfg.Env["HDD_PATH"]] {
|
||||
out[st.Name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hddPathReader is the slice of *stacks.Manager driveLostApps reads (a seam for the test).
|
||||
type hddPathReader interface {
|
||||
GetStacks() []stacks.Stack
|
||||
LoadAppConfigByName(name string) *stacks.AppConfig
|
||||
}
|
||||
|
||||
// maskDriveLost reports the drive-lost apps as not down to the NOTIFIER only (see R-521 at the call).
|
||||
func maskDriveLost(states []notify.AppRunState, lost map[string]bool) []notify.AppRunState {
|
||||
if len(lost) == 0 {
|
||||
return states
|
||||
}
|
||||
out := make([]notify.AppRunState, len(states))
|
||||
copy(out, states)
|
||||
for i := range out {
|
||||
if out[i].Down && lost[out[i].Name] {
|
||||
out[i].Down = false
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// updateHeldSet is nil-safe over a nil interface (a box with backup disabled has no holds).
|
||||
func updateHeldSet(l updateHeldLister) map[string]bool {
|
||||
if l == nil {
|
||||
|
||||
Reference in New Issue
Block a user