R-1286: a held app says why it is held — each hold reason gets its own sentence in the start gate, the boot sweep and --restore-holds (an unhealthy stop read as a failed restore). Red-proved. Commit only.
gates / gates (push) Successful in 1m28s
gates / gates (push) Successful in 1m28s
This commit is contained in:
@@ -2,6 +2,15 @@
|
|||||||
|
|
||||||
**MinAgent: 0.158.1** (unchanged). The hold acts only on a box whose agent reports `waiting_for_quiet` (≥ 0.165.0).
|
**MinAgent: 0.158.1** (unchanged). The hold acts only on a box whose agent reports `waiting_for_quiet` (≥ 0.165.0).
|
||||||
|
|
||||||
|
### A held app says WHY it is held (R-1286, found reading demo-hp's Kimai)
|
||||||
|
|
||||||
|
- `cmd/controller/main.go` `holdWhy`: the start gate (and through it the boot sweep's line and `--restore-holds`) now
|
||||||
|
names each hold reason — a crash-loop / OOM stop („stopped by the box for a crash loop … the household's Start gives
|
||||||
|
it one more try"), a failed update, a failed off-site replay, a pending put-back, a failed restore. demo-hp's Kimai
|
||||||
|
was stopped by the crash-loop guard on 2026-10-14 09:06Z (the night power drill) and every surface called it „held
|
||||||
|
after a failed restore whose rollback also failed" for 11 days. `--restore-holds` prints the reason too.
|
||||||
|
`TestR1286_EveryHoldReasonSaysWhatItIs` (red-proof: felhom.eu `documentation/audits/night-2026-10-25/redproof-r1286.txt`).
|
||||||
|
|
||||||
### A restore of a set-aside copy retires the replaced install's unit at once, quietly (R-1288, P4)
|
### A restore of a set-aside copy retires the replaced install's unit at once, quietly (R-1288, P4)
|
||||||
|
|
||||||
- `internal/backup/unit_owner.go` `RestoreSetAsideUnit` → `retireReplacedUnit`: after the household restored an earlier
|
- `internal/backup/unit_owner.go` `RestoreSetAsideUnit` → `retireReplacedUnit`: after the household restored an earlier
|
||||||
|
|||||||
@@ -192,7 +192,7 @@ func main() {
|
|||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
for _, h := range holds {
|
for _, h := range holds {
|
||||||
fmt.Printf("%s\theld since %s\n replay error : %s\n rollback err : %s\n", h.Stack, h.At, h.ReplayError, h.RollbackErr)
|
fmt.Printf("%s\theld since %s\n why : %s\n replay error : %s\n rollback err : %s\n", h.Stack, h.At, holdWhy(h), h.ReplayError, h.RollbackErr)
|
||||||
}
|
}
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
@@ -2558,6 +2558,28 @@ type driveStartGate struct {
|
|||||||
sett *settings.Settings
|
sett *settings.Settings
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// holdWhy says which operation put a hold in place and what lifts it — one sentence per reason (R-1286: an unhealthy
|
||||||
|
// stop read as „held after a failed restore whose rollback also failed" for 11 days on demo-hp, so the operator could
|
||||||
|
// not tell a crash loop from a broken restore). Pinned by TestR1286_EveryHoldReasonSaysWhatItIs.
|
||||||
|
func holdWhy(h settings.RestoreHold) string {
|
||||||
|
switch h.Reason {
|
||||||
|
case settings.HoldReasonUpdateFailed:
|
||||||
|
return "held after a failed update (" + h.At + ") — restore it from its backup to start it"
|
||||||
|
case settings.HoldReasonUnhealthyStop:
|
||||||
|
kind := h.UnhealthyKind
|
||||||
|
if kind == "" {
|
||||||
|
kind = "unhealthy"
|
||||||
|
}
|
||||||
|
return "stopped by the box for a " + strings.ReplaceAll(kind, "_", " ") + " (" + h.At + ") — the household's Start gives it one more try"
|
||||||
|
case settings.HoldReasonRestoreMixed:
|
||||||
|
return "held after an off-site restore failed its database replay (" + h.At + ") — clear the hold to start it"
|
||||||
|
case settings.HoldReasonPutBackPending:
|
||||||
|
return "held until the pre-restore copy is poured back (" + h.At + ") — the put-back starts it"
|
||||||
|
default:
|
||||||
|
return "held after a failed restore whose rollback also failed (" + h.At + ") — clear the hold to start it"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (g driveStartGate) MayStart(stackName string) (bool, string) {
|
func (g driveStartGate) MayStart(stackName string) (bool, string) {
|
||||||
// R-379/R-380 HOLDER, checked FIRST and deliberately ABOVE the `HDD_PATH == ""` early return
|
// R-379/R-380 HOLDER, checked FIRST and deliberately ABOVE the `HDD_PATH == ""` early return
|
||||||
// below. The apps this hold exists for are DRIVELESS — a failed database restore is the case,
|
// below. The apps this hold exists for are DRIVELESS — a failed database restore is the case,
|
||||||
@@ -2569,10 +2591,7 @@ func (g driveStartGate) MayStart(stackName string) (bool, string) {
|
|||||||
// app-stop guard's Recover reaches it directly. A hold only one path honours is not a hold.
|
// app-stop guard's Recover reaches it directly. A hold only one path honours is not a hold.
|
||||||
if g.sett != nil {
|
if g.sett != nil {
|
||||||
if h, ok := g.sett.GetRestoreHold(stackName); ok {
|
if h, ok := g.sett.GetRestoreHold(stackName); ok {
|
||||||
if h.Reason == settings.HoldReasonUpdateFailed {
|
return false, holdWhy(h)
|
||||||
return false, "held after a failed update (" + h.At + ") — restore it from its backup to start it"
|
|
||||||
}
|
|
||||||
return false, "held after a failed restore whose rollback also failed (" + h.At + ") — clear the hold to start it"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if g.mgr == nil {
|
if g.mgr == nil {
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// R-1286: demo-hp's Kimai was held by the crash-loop guard (reason unhealthy_stop, 2026-10-14) and the boot sweep, the
|
||||||
|
// start gate and `--restore-holds` all said „held after a failed restore whose rollback also failed" for 11 days.
|
||||||
|
// Each reason now says what it is and what lifts it.
|
||||||
|
// RED-PROOF: holdWhy back to the old two-way branch → the unhealthy_stop, restore_mixed and putback_pending cases fail.
|
||||||
|
func TestR1286_EveryHoldReasonSaysWhatItIs(t *testing.T) {
|
||||||
|
sett := holdTestSettings(t)
|
||||||
|
g := driveStartGate{sett: sett}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
hold settings.RestoreHold
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{settings.RestoreHold{Stack: "kimai", At: "2026-10-14T09:06:31Z", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "crash_loop"}, "stopped by the box for a crash loop"},
|
||||||
|
{settings.RestoreHold{Stack: "a", At: "t", Reason: settings.HoldReasonUpdateFailed}, "failed update"},
|
||||||
|
{settings.RestoreHold{Stack: "b", At: "t", Reason: settings.HoldReasonRestoreMixed}, "off-site restore failed its database replay"},
|
||||||
|
{settings.RestoreHold{Stack: "c", At: "t", Reason: settings.HoldReasonPutBackPending}, "pre-restore copy is poured back"},
|
||||||
|
{settings.RestoreHold{Stack: "d", At: "t"}, "failed restore whose rollback also failed"},
|
||||||
|
} {
|
||||||
|
if err := sett.SetRestoreHold(tc.hold); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ok, why := g.MayStart(tc.hold.Stack)
|
||||||
|
if ok || !strings.Contains(why, tc.want) {
|
||||||
|
t.Fatalf("%s (reason %q): the gate says %q, want it to contain %q", tc.hold.Stack, tc.hold.Reason, why, tc.want)
|
||||||
|
}
|
||||||
|
if tc.hold.Reason != "" && strings.Contains(why, "failed restore whose rollback also failed") {
|
||||||
|
t.Fatalf("%s: a %q hold must not read as a failed restore: %q", tc.hold.Stack, tc.hold.Reason, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user