controller v0.268.0: the undo finds volumes by definition; a held app names only a whole copy; one press = one tested step (R-658, R-659, R-660, R-651; 09 §6.4 part 5)
gates / gates (push) Successful in 27s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-24 08:15:27 +02:00
parent 80e6ad8c47
commit 206b0357d1
36 changed files with 2201 additions and 40 deletions
+49 -4
View File
@@ -622,6 +622,17 @@ func main() {
if backupMgr != nil && ev.CopyTier > 0 {
d.CopyHolds = backupMgr.UpdateCopyHoldsKey(ev.App, ev.CopyTier) // R-647: the key, never the Hungarian phrase
}
var hold settings.RestoreHold
haveHold := false
if ev.HoldRecorded && backupMgr != nil {
hold, haveHold = backupMgr.UpdateHold(ev.App)
}
if haveHold { // R-659: the details name the copy the SENTENCE names, not the precondition's
d.CopyTier, d.CopyDate, d.CopyHolds = hold.CopyTier, hold.CopyDate, ""
if hold.CopyTier > 0 {
d.CopyHolds = backupMgr.UpdateCopyHoldsKey(ev.App, hold.CopyTier)
}
}
notifier.NotifyAppUpdateHeld(d, func(lang string) string {
if ev.HoldRecorded && backupMgr != nil {
if held, why := backupMgr.RestoreHoldForLang(ev.App, lang); held {
@@ -630,6 +641,12 @@ func main() {
}
return util.Text(lang, "update.error.hold_unsaved")
})
// R-659 (v0.268.0; operator ruling 2026-09-24, option A): no copy on this box brings the app
// back whole — the household was just told support is informed; this is that information.
if haveHold && hold.NoWholeCopy {
notifier.NotifyAppHoldNoWholeCopy(notify.AppHoldNoWholeCopyDetails{App: ev.App, StackName: ev.App,
From: ev.From, To: ev.To, At: ev.At.UTC().Format(time.RFC3339), CopiesSeen: hold.CopiesSeen, UndoState: hold.UndoState})
}
}
})
@@ -843,7 +860,7 @@ func main() {
if time.Since(startTime) < deadAppBootGrace {
return nil // still inside the startup settle window
}
dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop, appStopGuard)
dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop, appStopGuard, backupMgr)
alertMgr.SetDeadAppAlerts(dead)
notifier.NotifyAppStartFailures(states)
// R-514: a worker OOM-killed inside a running container leaves the app „Fut". Surface it.
@@ -2353,7 +2370,7 @@ func recordLateRecovery(logger *log.Logger, started time.Time, res bootrecon.Res
// state-based dashboard banner) and EVERY deployed app's run state (for the notifier's one-event-per-
// transition tracking). Deploying apps are skipped (mid-deploy is not a fault). Pure over GetStacks()
// — the derivation itself lives in classifyRunStates so it is testable without a live Manager.
func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop, g *backup.AppStopGuard) ([]web.DeadApp, []notify.AppRunState) {
func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop, g *backup.AppStopGuard, held updateHeldLister) ([]web.DeadApp, []notify.AppRunState) {
// R-97b: a stack THIS controller stopped for a backup is not a fault. q may be nil (unprovisioned
// guest) — SuppressedStacks is nil-safe and returns nothing, i.e. suppress nothing.
//
@@ -2366,9 +2383,26 @@ func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop, g *backup.Ap
// for health, and it ends in healthy or HELD. Counting it as dead mid-update would be R-330's false
// alarm one mechanism over.
suppressed := unionSuppressed(unionSuppressed(q.SuppressedStacks(), g.SuppressedStacks()), mgr.UpdatingStacks())
// R-660 (v0.268.0): a FOURTH — an app HELD after a failed update is stopped by the product and has
// its own event (`app_update_held`). Without this each hold was followed by an `app_start_failed`
// for the same app. Pinned by TestR660_UpdateHeldAppIsNotReportedDown + the wiring test beside it.
suppressed = unionSuppressed(suppressed, updateHeldSet(held))
return classifyRunStates(mgr.GetStacks(), suppressed, q.FailedRestarts(), time.Now())
}
// updateHeldLister is the backup manager's UpdateHeldStacks, as a seam for the test.
type updateHeldLister interface {
UpdateHeldStacks() map[string]bool
}
// updateHeldSet is nil-safe over a nil interface (a box with backup disabled has no holds).
func updateHeldSet(l updateHeldLister) map[string]bool {
if l == nil {
return nil
}
return l.UpdateHeldStacks()
}
// unionSuppressed merges the suppression sets of the two mechanisms that stop apps on purpose.
// Returns nil when both are empty so the common case allocates nothing.
func unionSuppressed(a, b map[string]bool) map[string]bool {
@@ -3545,6 +3579,17 @@ func (a *updateGuardsAdapter) HoldAfterFailedUpdate(name string, at time.Time, r
if a.b == nil {
return fmt.Errorf("backup is not enabled on this box — the hold cannot be recorded")
}
// R-479: the sentence names what the chosen copy holds, decided by the app's data layout NOW.
return a.b.HoldAfterFailedUpdateHolding(name, at, rp.ProvenAt, rp.Tier, a.b.UpdateCopyHolds(name, rp.Tier), undoState)
// R-659 (v0.268.0): the hold names the newest copy that brings the app back WHOLE — not the
// precondition copy `rp`, which may be one the restore refuses — and none when there is none.
// R-479 stands inside it: the sentence still says what that copy holds.
_, err := a.b.HoldAfterFailedUpdateWhole(context.Background(), name, at, undoState)
return err
}
// HoldNoWholeCopy is the page's half of R-659: a hold naming no copy gets no restore button.
func (a *updateGuardsAdapter) HoldNoWholeCopy(name string) bool {
if a.b == nil {
return false
}
return a.b.HoldNoWholeCopy(name)
}
@@ -65,12 +65,12 @@ func TestR475_AdapterReadsEveryTier(t *testing.T) {
if cb := adapterMethodSelectors(t, "CanBackUp"); !strings.Contains(cb, " CanBackUpApp ") {
t.Errorf("CanBackUp must ask backup.CanBackUpApp; selectors:%s", cb)
}
if h := adapterMethodSelectors(t, "HoldAfterFailedUpdate"); !strings.Contains(h, " Tier ") {
t.Errorf("the hold must be told the chosen TIER, or it cannot name it; selectors:%s", h)
}
// R-479: and WHAT the copy holds, computed from the app's data layout at hold time.
if h := adapterMethodSelectors(t, "HoldAfterFailedUpdate"); !strings.Contains(h, " UpdateCopyHolds ") || !strings.Contains(h, " HoldAfterFailedUpdateHolding ") {
t.Errorf("the hold must carry UpdateCopyHolds through HoldAfterFailedUpdateHolding (R-479); selectors:%s", h)
// R-659 (v0.268.0) CHANGED THIS ON PURPOSE: the hold no longer names the precondition copy the
// adapter is handed (round 11: that copy was one the restore refuses). The tier, and R-479's
// "what the copy holds", are now chosen inside backup.HoldAfterFailedUpdateWhole from every tier —
// pinned there by TestR659_TheHoldNamesOnlyAWholeCopy (which asserts the holds-phrase too).
if h := adapterMethodSelectors(t, "HoldAfterFailedUpdate"); !strings.Contains(h, " HoldAfterFailedUpdateWhole ") {
t.Errorf("the hold must be recorded through HoldAfterFailedUpdateWhole (R-659); selectors:%s", h)
}
}
@@ -0,0 +1,47 @@
package main
import (
"go/ast"
"strings"
"testing"
)
// R-659 (v0.268.0) — the pieces are wired: the adapter records the hold through the WHOLE-copy walk
// (never the precondition copy again), and the held-event sink sends app_hold_no_whole_copy.
// An AST walk, not a substring (a commented-out call satisfies strings.Contains).
func TestR659_HoldAndEventAreWired(t *testing.T) {
_, f, _ := slice4CallLines(t)
var calls []string
for _, d := range f.Decls {
fn, ok := d.(*ast.FuncDecl)
if !ok || fn.Name.Name != "HoldAfterFailedUpdate" || fn.Recv == nil || fn.Body == nil {
continue
}
if st, ok := fn.Recv.List[0].Type.(*ast.StarExpr); !ok || st.X.(*ast.Ident).Name != "updateGuardsAdapter" {
continue
}
ast.Inspect(fn.Body, func(n ast.Node) bool {
if sel, ok := n.(*ast.SelectorExpr); ok {
calls = append(calls, sel.Sel.Name)
}
return true
})
}
body := strings.Join(calls, " ")
if !strings.Contains(body, "HoldAfterFailedUpdateWhole") {
t.Fatal("the adapter does not record the hold through HoldAfterFailedUpdateWhole — the precondition copy would be named again (R-659)")
}
if strings.Contains(body, "HoldAfterFailedUpdateHolding") {
t.Fatal("the adapter still calls HoldAfterFailedUpdateHolding with the precondition copy")
}
sends := false
ast.Inspect(mainBody(t), func(n ast.Node) bool {
if sel, ok := n.(*ast.SelectorExpr); ok && sel.Sel.Name == "NotifyAppHoldNoWholeCopy" {
sends = true
}
return true
})
if !sends {
t.Fatal("main never calls NotifyAppHoldNoWholeCopy — support would never be told")
}
}
@@ -0,0 +1,87 @@
package main
import (
"go/ast"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)
// R-660 (v0.268.0) — an app HELD after a failed update is stopped by the product and has its own
// event. Measured on 9202 2026-09-23 night (chaos rounds 8 and 11): `app_update_held` at 20:56:04 and
// 21:42:39, then `app_start_failed` 11 s and 13 s later for the same moment — two alarms, one fact.
type fakeHeld map[string]bool
func (f fakeHeld) UpdateHeldStacks() map[string]bool { return f }
// TestR660_UpdateHeldAppIsNotReportedDown — the consequence, through the suppression the scan builds.
// A genuinely exited app beside it must still alarm (the over-correction guard).
//
// COMPANION RED-PROOF (REPORT.md): make updateHeldSet return nil. This test then fails at "a held app
// was reported DOWN".
func TestR660_UpdateHeldAppIsNotReportedDown(t *testing.T) {
sts := []stacks.Stack{
stack("nextcloud", stacks.StateStopped, true, false), // held after a failed update
stack("romm", stacks.StateExited, true, false), // genuinely broken
}
// A held app's intent is "running" — the household never asked for the stop.
for i := range sts {
sts[i].AppConfig = &stacks.AppConfig{DesiredState: stacks.DesiredStateRunning}
}
dead, states := classifyRunStates(sts, unionSuppressed(nil, updateHeldSet(fakeHeld{"nextcloud": true})), nil, time.Now())
down := downByName(states)
if down["nextcloud"] || deadNames(dead)["nextcloud"] {
t.Fatal("a held app was reported DOWN — the second, false alarm after app_update_held (R-660)")
}
if !down["romm"] {
t.Fatal("a genuinely exited app stopped alarming — the fix silenced a real fault")
}
// Control: WITHOUT the held set the same app does alarm — so the suppression is what decides.
_, states = classifyRunStates(sts, nil, nil, time.Now())
if !downByName(states)["nextcloud"] {
t.Fatal("control failed: the fixture's app does not alarm even without the suppression")
}
}
// The wiring: main passes the backup manager, and the scan unions its held set.
func TestR660_ScanIsGivenTheHeldSet(t *testing.T) {
_, f, _ := slice4CallLines(t)
reads := false
for _, d := range f.Decls {
fn, ok := d.(*ast.FuncDecl)
if !ok || fn.Name.Name != "scanDeployedAppRunStates" || fn.Body == nil {
continue
}
ast.Inspect(fn.Body, func(n ast.Node) bool {
if call, ok := n.(*ast.CallExpr); ok {
if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "updateHeldSet" {
reads = true
}
}
return true
})
}
if !reads {
t.Fatal("scanDeployedAppRunStates does not read the update-held set (R-660)")
}
withB := false
ast.Inspect(mainBody(t), func(n ast.Node) bool {
call, ok := n.(*ast.CallExpr)
if !ok {
return true
}
if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "scanDeployedAppRunStates" {
for _, a := range call.Args {
if x, ok := a.(*ast.Ident); ok && x.Name == "backupMgr" {
withB = true
}
}
}
return true
})
if !withB {
t.Fatal("main calls scanDeployedAppRunStates without backupMgr — the held set is built and never read")
}
}