controller v0.268.0: the undo finds volumes by definition; a held app names only a whole copy; one press = one tested step (R-658, R-659, R-660, R-651; 09 §6.4 part 5)
gates / gates (push) Successful in 27s
gates / gates (push) Successful in 27s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -622,6 +622,17 @@ func main() {
|
||||
if backupMgr != nil && ev.CopyTier > 0 {
|
||||
d.CopyHolds = backupMgr.UpdateCopyHoldsKey(ev.App, ev.CopyTier) // R-647: the key, never the Hungarian phrase
|
||||
}
|
||||
var hold settings.RestoreHold
|
||||
haveHold := false
|
||||
if ev.HoldRecorded && backupMgr != nil {
|
||||
hold, haveHold = backupMgr.UpdateHold(ev.App)
|
||||
}
|
||||
if haveHold { // R-659: the details name the copy the SENTENCE names, not the precondition's
|
||||
d.CopyTier, d.CopyDate, d.CopyHolds = hold.CopyTier, hold.CopyDate, ""
|
||||
if hold.CopyTier > 0 {
|
||||
d.CopyHolds = backupMgr.UpdateCopyHoldsKey(ev.App, hold.CopyTier)
|
||||
}
|
||||
}
|
||||
notifier.NotifyAppUpdateHeld(d, func(lang string) string {
|
||||
if ev.HoldRecorded && backupMgr != nil {
|
||||
if held, why := backupMgr.RestoreHoldForLang(ev.App, lang); held {
|
||||
@@ -630,6 +641,12 @@ func main() {
|
||||
}
|
||||
return util.Text(lang, "update.error.hold_unsaved")
|
||||
})
|
||||
// R-659 (v0.268.0; operator ruling 2026-09-24, option A): no copy on this box brings the app
|
||||
// back whole — the household was just told support is informed; this is that information.
|
||||
if haveHold && hold.NoWholeCopy {
|
||||
notifier.NotifyAppHoldNoWholeCopy(notify.AppHoldNoWholeCopyDetails{App: ev.App, StackName: ev.App,
|
||||
From: ev.From, To: ev.To, At: ev.At.UTC().Format(time.RFC3339), CopiesSeen: hold.CopiesSeen, UndoState: hold.UndoState})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
@@ -843,7 +860,7 @@ func main() {
|
||||
if time.Since(startTime) < deadAppBootGrace {
|
||||
return nil // still inside the startup settle window
|
||||
}
|
||||
dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop, appStopGuard)
|
||||
dead, states := scanDeployedAppRunStates(stackMgr, quiesceLoop, appStopGuard, backupMgr)
|
||||
alertMgr.SetDeadAppAlerts(dead)
|
||||
notifier.NotifyAppStartFailures(states)
|
||||
// R-514: a worker OOM-killed inside a running container leaves the app „Fut". Surface it.
|
||||
@@ -2353,7 +2370,7 @@ func recordLateRecovery(logger *log.Logger, started time.Time, res bootrecon.Res
|
||||
// state-based dashboard banner) and EVERY deployed app's run state (for the notifier's one-event-per-
|
||||
// transition tracking). Deploying apps are skipped (mid-deploy is not a fault). Pure over GetStacks()
|
||||
// — the derivation itself lives in classifyRunStates so it is testable without a live Manager.
|
||||
func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop, g *backup.AppStopGuard) ([]web.DeadApp, []notify.AppRunState) {
|
||||
func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop, g *backup.AppStopGuard, held updateHeldLister) ([]web.DeadApp, []notify.AppRunState) {
|
||||
// R-97b: a stack THIS controller stopped for a backup is not a fault. q may be nil (unprovisioned
|
||||
// guest) — SuppressedStacks is nil-safe and returns nothing, i.e. suppress nothing.
|
||||
//
|
||||
@@ -2366,9 +2383,26 @@ func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop, g *backup.Ap
|
||||
// for health, and it ends in healthy or HELD. Counting it as dead mid-update would be R-330's false
|
||||
// alarm one mechanism over.
|
||||
suppressed := unionSuppressed(unionSuppressed(q.SuppressedStacks(), g.SuppressedStacks()), mgr.UpdatingStacks())
|
||||
// R-660 (v0.268.0): a FOURTH — an app HELD after a failed update is stopped by the product and has
|
||||
// its own event (`app_update_held`). Without this each hold was followed by an `app_start_failed`
|
||||
// for the same app. Pinned by TestR660_UpdateHeldAppIsNotReportedDown + the wiring test beside it.
|
||||
suppressed = unionSuppressed(suppressed, updateHeldSet(held))
|
||||
return classifyRunStates(mgr.GetStacks(), suppressed, q.FailedRestarts(), time.Now())
|
||||
}
|
||||
|
||||
// updateHeldLister is the backup manager's UpdateHeldStacks, as a seam for the test.
|
||||
type updateHeldLister interface {
|
||||
UpdateHeldStacks() map[string]bool
|
||||
}
|
||||
|
||||
// updateHeldSet is nil-safe over a nil interface (a box with backup disabled has no holds).
|
||||
func updateHeldSet(l updateHeldLister) map[string]bool {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
return l.UpdateHeldStacks()
|
||||
}
|
||||
|
||||
// unionSuppressed merges the suppression sets of the two mechanisms that stop apps on purpose.
|
||||
// Returns nil when both are empty so the common case allocates nothing.
|
||||
func unionSuppressed(a, b map[string]bool) map[string]bool {
|
||||
@@ -3545,6 +3579,17 @@ func (a *updateGuardsAdapter) HoldAfterFailedUpdate(name string, at time.Time, r
|
||||
if a.b == nil {
|
||||
return fmt.Errorf("backup is not enabled on this box — the hold cannot be recorded")
|
||||
}
|
||||
// R-479: the sentence names what the chosen copy holds, decided by the app's data layout NOW.
|
||||
return a.b.HoldAfterFailedUpdateHolding(name, at, rp.ProvenAt, rp.Tier, a.b.UpdateCopyHolds(name, rp.Tier), undoState)
|
||||
// R-659 (v0.268.0): the hold names the newest copy that brings the app back WHOLE — not the
|
||||
// precondition copy `rp`, which may be one the restore refuses — and none when there is none.
|
||||
// R-479 stands inside it: the sentence still says what that copy holds.
|
||||
_, err := a.b.HoldAfterFailedUpdateWhole(context.Background(), name, at, undoState)
|
||||
return err
|
||||
}
|
||||
|
||||
// HoldNoWholeCopy is the page's half of R-659: a hold naming no copy gets no restore button.
|
||||
func (a *updateGuardsAdapter) HoldNoWholeCopy(name string) bool {
|
||||
if a.b == nil {
|
||||
return false
|
||||
}
|
||||
return a.b.HoldNoWholeCopy(name)
|
||||
}
|
||||
|
||||
@@ -65,12 +65,12 @@ func TestR475_AdapterReadsEveryTier(t *testing.T) {
|
||||
if cb := adapterMethodSelectors(t, "CanBackUp"); !strings.Contains(cb, " CanBackUpApp ") {
|
||||
t.Errorf("CanBackUp must ask backup.CanBackUpApp; selectors:%s", cb)
|
||||
}
|
||||
if h := adapterMethodSelectors(t, "HoldAfterFailedUpdate"); !strings.Contains(h, " Tier ") {
|
||||
t.Errorf("the hold must be told the chosen TIER, or it cannot name it; selectors:%s", h)
|
||||
}
|
||||
// R-479: and WHAT the copy holds, computed from the app's data layout at hold time.
|
||||
if h := adapterMethodSelectors(t, "HoldAfterFailedUpdate"); !strings.Contains(h, " UpdateCopyHolds ") || !strings.Contains(h, " HoldAfterFailedUpdateHolding ") {
|
||||
t.Errorf("the hold must carry UpdateCopyHolds through HoldAfterFailedUpdateHolding (R-479); selectors:%s", h)
|
||||
// R-659 (v0.268.0) CHANGED THIS ON PURPOSE: the hold no longer names the precondition copy the
|
||||
// adapter is handed (round 11: that copy was one the restore refuses). The tier, and R-479's
|
||||
// "what the copy holds", are now chosen inside backup.HoldAfterFailedUpdateWhole from every tier —
|
||||
// pinned there by TestR659_TheHoldNamesOnlyAWholeCopy (which asserts the holds-phrase too).
|
||||
if h := adapterMethodSelectors(t, "HoldAfterFailedUpdate"); !strings.Contains(h, " HoldAfterFailedUpdateWhole ") {
|
||||
t.Errorf("the hold must be recorded through HoldAfterFailedUpdateWhole (R-659); selectors:%s", h)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-659 (v0.268.0) — the pieces are wired: the adapter records the hold through the WHOLE-copy walk
|
||||
// (never the precondition copy again), and the held-event sink sends app_hold_no_whole_copy.
|
||||
// An AST walk, not a substring (a commented-out call satisfies strings.Contains).
|
||||
func TestR659_HoldAndEventAreWired(t *testing.T) {
|
||||
_, f, _ := slice4CallLines(t)
|
||||
var calls []string
|
||||
for _, d := range f.Decls {
|
||||
fn, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fn.Name.Name != "HoldAfterFailedUpdate" || fn.Recv == nil || fn.Body == nil {
|
||||
continue
|
||||
}
|
||||
if st, ok := fn.Recv.List[0].Type.(*ast.StarExpr); !ok || st.X.(*ast.Ident).Name != "updateGuardsAdapter" {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fn.Body, func(n ast.Node) bool {
|
||||
if sel, ok := n.(*ast.SelectorExpr); ok {
|
||||
calls = append(calls, sel.Sel.Name)
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
body := strings.Join(calls, " ")
|
||||
if !strings.Contains(body, "HoldAfterFailedUpdateWhole") {
|
||||
t.Fatal("the adapter does not record the hold through HoldAfterFailedUpdateWhole — the precondition copy would be named again (R-659)")
|
||||
}
|
||||
if strings.Contains(body, "HoldAfterFailedUpdateHolding") {
|
||||
t.Fatal("the adapter still calls HoldAfterFailedUpdateHolding with the precondition copy")
|
||||
}
|
||||
sends := false
|
||||
ast.Inspect(mainBody(t), func(n ast.Node) bool {
|
||||
if sel, ok := n.(*ast.SelectorExpr); ok && sel.Sel.Name == "NotifyAppHoldNoWholeCopy" {
|
||||
sends = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !sends {
|
||||
t.Fatal("main never calls NotifyAppHoldNoWholeCopy — support would never be told")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
// R-660 (v0.268.0) — an app HELD after a failed update is stopped by the product and has its own
|
||||
// event. Measured on 9202 2026-09-23 night (chaos rounds 8 and 11): `app_update_held` at 20:56:04 and
|
||||
// 21:42:39, then `app_start_failed` 11 s and 13 s later for the same moment — two alarms, one fact.
|
||||
|
||||
type fakeHeld map[string]bool
|
||||
|
||||
func (f fakeHeld) UpdateHeldStacks() map[string]bool { return f }
|
||||
|
||||
// TestR660_UpdateHeldAppIsNotReportedDown — the consequence, through the suppression the scan builds.
|
||||
// A genuinely exited app beside it must still alarm (the over-correction guard).
|
||||
//
|
||||
// COMPANION RED-PROOF (REPORT.md): make updateHeldSet return nil. This test then fails at "a held app
|
||||
// was reported DOWN".
|
||||
func TestR660_UpdateHeldAppIsNotReportedDown(t *testing.T) {
|
||||
sts := []stacks.Stack{
|
||||
stack("nextcloud", stacks.StateStopped, true, false), // held after a failed update
|
||||
stack("romm", stacks.StateExited, true, false), // genuinely broken
|
||||
}
|
||||
// A held app's intent is "running" — the household never asked for the stop.
|
||||
for i := range sts {
|
||||
sts[i].AppConfig = &stacks.AppConfig{DesiredState: stacks.DesiredStateRunning}
|
||||
}
|
||||
dead, states := classifyRunStates(sts, unionSuppressed(nil, updateHeldSet(fakeHeld{"nextcloud": true})), nil, time.Now())
|
||||
down := downByName(states)
|
||||
if down["nextcloud"] || deadNames(dead)["nextcloud"] {
|
||||
t.Fatal("a held app was reported DOWN — the second, false alarm after app_update_held (R-660)")
|
||||
}
|
||||
if !down["romm"] {
|
||||
t.Fatal("a genuinely exited app stopped alarming — the fix silenced a real fault")
|
||||
}
|
||||
// Control: WITHOUT the held set the same app does alarm — so the suppression is what decides.
|
||||
_, states = classifyRunStates(sts, nil, nil, time.Now())
|
||||
if !downByName(states)["nextcloud"] {
|
||||
t.Fatal("control failed: the fixture's app does not alarm even without the suppression")
|
||||
}
|
||||
}
|
||||
|
||||
// The wiring: main passes the backup manager, and the scan unions its held set.
|
||||
func TestR660_ScanIsGivenTheHeldSet(t *testing.T) {
|
||||
_, f, _ := slice4CallLines(t)
|
||||
reads := false
|
||||
for _, d := range f.Decls {
|
||||
fn, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fn.Name.Name != "scanDeployedAppRunStates" || fn.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fn.Body, func(n ast.Node) bool {
|
||||
if call, ok := n.(*ast.CallExpr); ok {
|
||||
if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "updateHeldSet" {
|
||||
reads = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !reads {
|
||||
t.Fatal("scanDeployedAppRunStates does not read the update-held set (R-660)")
|
||||
}
|
||||
withB := false
|
||||
ast.Inspect(mainBody(t), func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "scanDeployedAppRunStates" {
|
||||
for _, a := range call.Args {
|
||||
if x, ok := a.(*ast.Ident); ok && x.Name == "backupMgr" {
|
||||
withB = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !withB {
|
||||
t.Fatal("main calls scanDeployedAppRunStates without backupMgr — the held set is built and never read")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user