From 838c8341699b3944ac51b3e22e6b011c8a71fba8 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 25 Oct 2026 22:05:23 +0100 Subject: [PATCH] =?UTF-8?q?R-1286:=20a=20held=20app=20says=20why=20it=20is?= =?UTF-8?q?=20held=20=E2=80=94=20each=20hold=20reason=20gets=20its=20own?= =?UTF-8?q?=20sentence=20in=20the=20start=20gate,=20the=20boot=20sweep=20a?= =?UTF-8?q?nd=20--restore-holds=20(an=20unhealthy=20stop=20read=20as=20a?= =?UTF-8?q?=20failed=20restore).=20Red-proved.=20Commit=20only.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 9 +++++ controller/cmd/controller/main.go | 29 +++++++++++--- .../cmd/controller/r1286_hold_why_test.go | 38 +++++++++++++++++++ 3 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 controller/cmd/controller/r1286_hold_why_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 90399f43..b6fdccb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ **MinAgent: 0.158.1** (unchanged). The hold acts only on a box whose agent reports `waiting_for_quiet` (≥ 0.165.0). +### A held app says WHY it is held (R-1286, found reading demo-hp's Kimai) + +- `cmd/controller/main.go` `holdWhy`: the start gate (and through it the boot sweep's line and `--restore-holds`) now + names each hold reason — a crash-loop / OOM stop („stopped by the box for a crash loop … the household's Start gives + it one more try"), a failed update, a failed off-site replay, a pending put-back, a failed restore. demo-hp's Kimai + was stopped by the crash-loop guard on 2026-10-14 09:06Z (the night power drill) and every surface called it „held + after a failed restore whose rollback also failed" for 11 days. `--restore-holds` prints the reason too. + `TestR1286_EveryHoldReasonSaysWhatItIs` (red-proof: felhom.eu `documentation/audits/night-2026-10-25/redproof-r1286.txt`). + ### A restore of a set-aside copy retires the replaced install's unit at once, quietly (R-1288, P4) - `internal/backup/unit_owner.go` `RestoreSetAsideUnit` → `retireReplacedUnit`: after the household restored an earlier diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index a6bdc95b..a7dd69f4 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -192,7 +192,7 @@ func main() { os.Exit(0) } for _, h := range holds { - fmt.Printf("%s\theld since %s\n replay error : %s\n rollback err : %s\n", h.Stack, h.At, h.ReplayError, h.RollbackErr) + fmt.Printf("%s\theld since %s\n why : %s\n replay error : %s\n rollback err : %s\n", h.Stack, h.At, holdWhy(h), h.ReplayError, h.RollbackErr) } os.Exit(0) } @@ -2558,6 +2558,28 @@ type driveStartGate struct { sett *settings.Settings } +// holdWhy says which operation put a hold in place and what lifts it — one sentence per reason (R-1286: an unhealthy +// stop read as „held after a failed restore whose rollback also failed" for 11 days on demo-hp, so the operator could +// not tell a crash loop from a broken restore). Pinned by TestR1286_EveryHoldReasonSaysWhatItIs. +func holdWhy(h settings.RestoreHold) string { + switch h.Reason { + case settings.HoldReasonUpdateFailed: + return "held after a failed update (" + h.At + ") — restore it from its backup to start it" + case settings.HoldReasonUnhealthyStop: + kind := h.UnhealthyKind + if kind == "" { + kind = "unhealthy" + } + return "stopped by the box for a " + strings.ReplaceAll(kind, "_", " ") + " (" + h.At + ") — the household's Start gives it one more try" + case settings.HoldReasonRestoreMixed: + return "held after an off-site restore failed its database replay (" + h.At + ") — clear the hold to start it" + case settings.HoldReasonPutBackPending: + return "held until the pre-restore copy is poured back (" + h.At + ") — the put-back starts it" + default: + return "held after a failed restore whose rollback also failed (" + h.At + ") — clear the hold to start it" + } +} + func (g driveStartGate) MayStart(stackName string) (bool, string) { // R-379/R-380 HOLDER, checked FIRST and deliberately ABOVE the `HDD_PATH == ""` early return // below. The apps this hold exists for are DRIVELESS — a failed database restore is the case, @@ -2569,10 +2591,7 @@ func (g driveStartGate) MayStart(stackName string) (bool, string) { // app-stop guard's Recover reaches it directly. A hold only one path honours is not a hold. if g.sett != nil { if h, ok := g.sett.GetRestoreHold(stackName); ok { - if h.Reason == settings.HoldReasonUpdateFailed { - return false, "held after a failed update (" + h.At + ") — restore it from its backup to start it" - } - return false, "held after a failed restore whose rollback also failed (" + h.At + ") — clear the hold to start it" + return false, holdWhy(h) } } if g.mgr == nil { diff --git a/controller/cmd/controller/r1286_hold_why_test.go b/controller/cmd/controller/r1286_hold_why_test.go new file mode 100644 index 00000000..cd331ee5 --- /dev/null +++ b/controller/cmd/controller/r1286_hold_why_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-controller/internal/settings" +) + +// R-1286: demo-hp's Kimai was held by the crash-loop guard (reason unhealthy_stop, 2026-10-14) and the boot sweep, the +// start gate and `--restore-holds` all said „held after a failed restore whose rollback also failed" for 11 days. +// Each reason now says what it is and what lifts it. +// RED-PROOF: holdWhy back to the old two-way branch → the unhealthy_stop, restore_mixed and putback_pending cases fail. +func TestR1286_EveryHoldReasonSaysWhatItIs(t *testing.T) { + sett := holdTestSettings(t) + g := driveStartGate{sett: sett} + for _, tc := range []struct { + hold settings.RestoreHold + want string + }{ + {settings.RestoreHold{Stack: "kimai", At: "2026-10-14T09:06:31Z", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "crash_loop"}, "stopped by the box for a crash loop"}, + {settings.RestoreHold{Stack: "a", At: "t", Reason: settings.HoldReasonUpdateFailed}, "failed update"}, + {settings.RestoreHold{Stack: "b", At: "t", Reason: settings.HoldReasonRestoreMixed}, "off-site restore failed its database replay"}, + {settings.RestoreHold{Stack: "c", At: "t", Reason: settings.HoldReasonPutBackPending}, "pre-restore copy is poured back"}, + {settings.RestoreHold{Stack: "d", At: "t"}, "failed restore whose rollback also failed"}, + } { + if err := sett.SetRestoreHold(tc.hold); err != nil { + t.Fatal(err) + } + ok, why := g.MayStart(tc.hold.Stack) + if ok || !strings.Contains(why, tc.want) { + t.Fatalf("%s (reason %q): the gate says %q, want it to contain %q", tc.hold.Stack, tc.hold.Reason, why, tc.want) + } + if tc.hold.Reason != "" && strings.Contains(why, "failed restore whose rollback also failed") { + t.Fatalf("%s: a %q hold must not read as a failed restore: %q", tc.hold.Stack, tc.hold.Reason, why) + } + } +}