R-1286: a held app says why it is held — each hold reason gets its own sentence in the start gate, the boot sweep and --restore-holds (an unhealthy stop read as a failed restore). Red-proved. Commit only.
gates / gates (push) Successful in 1m28s

This commit is contained in:
2026-10-25 22:05:23 +01:00
parent b5863a96eb
commit 838c834169
3 changed files with 71 additions and 5 deletions
+9
View File
@@ -2,6 +2,15 @@
**MinAgent: 0.158.1** (unchanged). The hold acts only on a box whose agent reports `waiting_for_quiet` (≥ 0.165.0). **MinAgent: 0.158.1** (unchanged). The hold acts only on a box whose agent reports `waiting_for_quiet` (≥ 0.165.0).
### A held app says WHY it is held (R-1286, found reading demo-hp's Kimai)
- `cmd/controller/main.go` `holdWhy`: the start gate (and through it the boot sweep's line and `--restore-holds`) now
names each hold reason — a crash-loop / OOM stop („stopped by the box for a crash loop … the household's Start gives
it one more try"), a failed update, a failed off-site replay, a pending put-back, a failed restore. demo-hp's Kimai
was stopped by the crash-loop guard on 2026-10-14 09:06Z (the night power drill) and every surface called it „held
after a failed restore whose rollback also failed" for 11 days. `--restore-holds` prints the reason too.
`TestR1286_EveryHoldReasonSaysWhatItIs` (red-proof: felhom.eu `documentation/audits/night-2026-10-25/redproof-r1286.txt`).
### A restore of a set-aside copy retires the replaced install's unit at once, quietly (R-1288, P4) ### A restore of a set-aside copy retires the replaced install's unit at once, quietly (R-1288, P4)
- `internal/backup/unit_owner.go` `RestoreSetAsideUnit` → `retireReplacedUnit`: after the household restored an earlier - `internal/backup/unit_owner.go` `RestoreSetAsideUnit` → `retireReplacedUnit`: after the household restored an earlier
+24 -5
View File
@@ -192,7 +192,7 @@ func main() {
os.Exit(0) os.Exit(0)
} }
for _, h := range holds { for _, h := range holds {
fmt.Printf("%s\theld since %s\n replay error : %s\n rollback err : %s\n", h.Stack, h.At, h.ReplayError, h.RollbackErr) fmt.Printf("%s\theld since %s\n why : %s\n replay error : %s\n rollback err : %s\n", h.Stack, h.At, holdWhy(h), h.ReplayError, h.RollbackErr)
} }
os.Exit(0) os.Exit(0)
} }
@@ -2558,6 +2558,28 @@ type driveStartGate struct {
sett *settings.Settings sett *settings.Settings
} }
// holdWhy says which operation put a hold in place and what lifts it — one sentence per reason (R-1286: an unhealthy
// stop read as „held after a failed restore whose rollback also failed" for 11 days on demo-hp, so the operator could
// not tell a crash loop from a broken restore). Pinned by TestR1286_EveryHoldReasonSaysWhatItIs.
func holdWhy(h settings.RestoreHold) string {
switch h.Reason {
case settings.HoldReasonUpdateFailed:
return "held after a failed update (" + h.At + ") — restore it from its backup to start it"
case settings.HoldReasonUnhealthyStop:
kind := h.UnhealthyKind
if kind == "" {
kind = "unhealthy"
}
return "stopped by the box for a " + strings.ReplaceAll(kind, "_", " ") + " (" + h.At + ") — the household's Start gives it one more try"
case settings.HoldReasonRestoreMixed:
return "held after an off-site restore failed its database replay (" + h.At + ") — clear the hold to start it"
case settings.HoldReasonPutBackPending:
return "held until the pre-restore copy is poured back (" + h.At + ") — the put-back starts it"
default:
return "held after a failed restore whose rollback also failed (" + h.At + ") — clear the hold to start it"
}
}
func (g driveStartGate) MayStart(stackName string) (bool, string) { func (g driveStartGate) MayStart(stackName string) (bool, string) {
// R-379/R-380 HOLDER, checked FIRST and deliberately ABOVE the `HDD_PATH == ""` early return // R-379/R-380 HOLDER, checked FIRST and deliberately ABOVE the `HDD_PATH == ""` early return
// below. The apps this hold exists for are DRIVELESS — a failed database restore is the case, // below. The apps this hold exists for are DRIVELESS — a failed database restore is the case,
@@ -2569,10 +2591,7 @@ func (g driveStartGate) MayStart(stackName string) (bool, string) {
// app-stop guard's Recover reaches it directly. A hold only one path honours is not a hold. // app-stop guard's Recover reaches it directly. A hold only one path honours is not a hold.
if g.sett != nil { if g.sett != nil {
if h, ok := g.sett.GetRestoreHold(stackName); ok { if h, ok := g.sett.GetRestoreHold(stackName); ok {
if h.Reason == settings.HoldReasonUpdateFailed { return false, holdWhy(h)
return false, "held after a failed update (" + h.At + ") — restore it from its backup to start it"
}
return false, "held after a failed restore whose rollback also failed (" + h.At + ") — clear the hold to start it"
} }
} }
if g.mgr == nil { if g.mgr == nil {
@@ -0,0 +1,38 @@
package main
import (
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// R-1286: demo-hp's Kimai was held by the crash-loop guard (reason unhealthy_stop, 2026-10-14) and the boot sweep, the
// start gate and `--restore-holds` all said „held after a failed restore whose rollback also failed" for 11 days.
// Each reason now says what it is and what lifts it.
// RED-PROOF: holdWhy back to the old two-way branch → the unhealthy_stop, restore_mixed and putback_pending cases fail.
func TestR1286_EveryHoldReasonSaysWhatItIs(t *testing.T) {
sett := holdTestSettings(t)
g := driveStartGate{sett: sett}
for _, tc := range []struct {
hold settings.RestoreHold
want string
}{
{settings.RestoreHold{Stack: "kimai", At: "2026-10-14T09:06:31Z", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "crash_loop"}, "stopped by the box for a crash loop"},
{settings.RestoreHold{Stack: "a", At: "t", Reason: settings.HoldReasonUpdateFailed}, "failed update"},
{settings.RestoreHold{Stack: "b", At: "t", Reason: settings.HoldReasonRestoreMixed}, "off-site restore failed its database replay"},
{settings.RestoreHold{Stack: "c", At: "t", Reason: settings.HoldReasonPutBackPending}, "pre-restore copy is poured back"},
{settings.RestoreHold{Stack: "d", At: "t"}, "failed restore whose rollback also failed"},
} {
if err := sett.SetRestoreHold(tc.hold); err != nil {
t.Fatal(err)
}
ok, why := g.MayStart(tc.hold.Stack)
if ok || !strings.Contains(why, tc.want) {
t.Fatalf("%s (reason %q): the gate says %q, want it to contain %q", tc.hold.Stack, tc.hold.Reason, why, tc.want)
}
if tc.hold.Reason != "" && strings.Contains(why, "failed restore whose rollback also failed") {
t.Fatalf("%s: a %q hold must not read as a failed restore: %q", tc.hold.Stack, tc.hold.Reason, why)
}
}
}