v0.237.0: the Update button takes a backup first, and tells the truth (update arc slice 4 — R-448, R-443, R-439)

POST /api/stacks/{name}/update is now a guarded job answering 202:
cheap refusals (hold — R-439, busy, migration, deploying, memory via the
deploy's own memoryVerdict, a fixed 2 GB disk floor, and no restorable
Tier-2 copy) → backup-first when the proven copy is older than
update.backup_max_age (24h) → safety dump BEFORE the pin moves → pin →
pull (failure puts the pin back) → up → health (.felhom.yml check or 60 s
settle, update.health_timeout 5m). Not healthy → the app is stopped and
HELD (RestoreHold reason update_failed, same store and gate as R-379) and
the page names the backup to restore from; the pin stays. Success is only
ever update_phase=done after health (R-443). UpdateStack is deleted.

The restorable-unit predicate is EXTRACTED to backup.Tier2UnitRestorePoint
and shared with the backups page (row pinned unchanged). The copy is aged
by the last successful Tier-2 copy, not the manifest created_at — measured
on demo-hp that created_at moves only on definition changes.

Crash safety: update-journal.json before each phase; RecoverUpdates before
the boot sweep, ResumeInterruptedUpdates after the guards are wired.

Three unattended start paths ignored a hold and now honour it: the
drive-return gate (restart + boot recreate) and the nightly volume dump.
The nightly capture and Tier-2 run skip held apps so the restore point
survives. No automatic rollback — measured per-app; route back = restore.

Tests A–H across stacks/backup/api/web/cmd; six red-proofs seen to fail.

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/cmd/controller/main.go

@@ -476,9 +476,20 @@ func main() {
 	// R-171: hand the boot sweep the settings it needs to answer "is this app's drive live?" BEFORE
 	// the goroutine starts — an unwired gate is silently the pre-v0.190.0 behaviour that started apps
 	// onto absent drives. TestMainWiresBootDriveGate walks this file's AST for the assignment.
+	// --- Slice 4: guarded-update crash recovery (Scenario G) ---
+	// BEFORE the boot reconciler, and the order is load-bearing: an update interrupted after `up` is
+	// marked Updating here, and bootDriveGate refuses an Updating app — started after the sweep, the
+	// sweep could bring up a half-updated app with no record of why. Pin-backs for updates interrupted
+	// before anything ran happen here too. The resumed health wait itself is launched further down,
+	// once the backup side is wired, because a resumed update that fails must be able to HOLD.
+	if resumed := stackMgr.RecoverUpdates(); len(resumed) > 0 {
+		logger.Printf("[WARN] [update] %d interrupted update(s) will resume after the backup side is wired: %v", len(resumed), resumed)
+	}
+
 	bootDriveSettings = sett
 	bootQuiesceLoop = quiesceLoop
 	bootAppStopGuard = appStopGuard
+	bootStackMgr = stackMgr
 	go runBootReconcile(ctx, stackMgr, logger)
 
 	// --- Start CPU collector ---
@@ -537,6 +548,16 @@ func main() {
 		backupMgr.SetSharesReconciler(stackMgr.ReconcileSamba)
 	}
 
+	// --- Slice 4: the guarded update's backup side ---
+	// Wired unconditionally: with backup disabled the adapter answers "no restore point" and every
+	// update is refused with the no-backup sentence — the precondition cannot be met, which is true.
+	// An UNWIRED manager also refuses (fail closed); TestSlice4_UpdateGuardsAreWiredAtStartup walks
+	// this file for the call, because a seam built and never wired has shipped here seven times.
+	stackMgr.SetUpdateGuards(&updateGuardsAdapter{b: backupMgr, q: quiesceLoop})
+	if n := stackMgr.ResumeInterruptedUpdates(ctx); n > 0 {
+		logger.Printf("[WARN] [update] resumed %d interrupted update(s)", n)
+	}
+
 	// SLICE 2: the offsite apply-bridge is launched further down, AFTER the self-updater is constructed
 	// (R-71a: the bridge's settle-gate reads the updater's floor/update-running state to defer the
 	// consume past a managed day-0 floor-update). See "offsite apply-bridge" below.
@@ -1906,6 +1927,7 @@ var (
 	bootDriveSettings *settings.Settings
 	bootQuiesceLoop   *quiesce.Loop
 	bootAppStopGuard  *backup.AppStopGuard
+	bootStackMgr      *stacks.Manager
 )
 
 // bootDriveGate answers bootrecon.StartGate for the real controller. It enforces §8.2: an app that
@@ -1943,6 +1965,11 @@ func (g bootDriveGate) MayStart(stackName string) (bool, string) {
 	if bootQuiesceLoop.SuppressedStacks()[stackName] {
 		return false, "a whole-guest backup (quiesce) is holding it — the quiesce loop restarts its own stacks"
 	}
+	// 1b. slice 4: a guarded update is moving this app (including one RecoverUpdates marked for
+	//     resumption). The update job owns the bring-up and ends in healthy or held.
+	if bootStackMgr != nil && bootStackMgr.IsUpdating(stackName) {
+		return false, "a guarded update is in progress — the update brings it up and verifies it"
+	}
 	// 2. an app-data operation in flight
 	for _, held := range bootAppStopGuard.HeldStacks() {
 		if held == stackName {
@@ -1985,6 +2012,9 @@ func (g driveStartGate) MayStart(stackName string) (bool, string) {
 	// app-stop guard's Recover reaches it directly. A hold only one path honours is not a hold.
 	if g.sett != nil {
 		if h, ok := g.sett.GetRestoreHold(stackName); ok {
+			if h.Reason == settings.HoldReasonUpdateFailed {
+				return false, "held after a failed update (" + h.At + ") — restore it from its backup to start it"
+			}
 			return false, "held after a failed restore whose rollback also failed (" + h.At + ") — clear the hold to start it"
 		}
 	}
@@ -2236,7 +2266,11 @@ func scanDeployedAppRunStates(mgr *stacks.Manager, q *quiesce.Loop, g *backup.Ap
 	// `g` covers the per-app operations — the nightly volume dump, an off-site reconstitution and a
 	// .fab export. Both are nil-safe, and the union is taken here rather than inside classifyRunStates
 	// so that pure function keeps its single `quiesced` parameter and its existing tests.
-	return classifyRunStates(mgr.GetStacks(), unionSuppressed(q.SuppressedStacks(), g.SuppressedStacks()), q.FailedRestarts(), time.Now())
+	// Slice 4: a THIRD mechanism moves an app on purpose — the guarded update recreates it and waits
+	// for health, and it ends in healthy or HELD. Counting it as dead mid-update would be R-330's false
+	// alarm one mechanism over.
+	suppressed := unionSuppressed(unionSuppressed(q.SuppressedStacks(), g.SuppressedStacks()), mgr.UpdatingStacks())
+	return classifyRunStates(mgr.GetStacks(), suppressed, q.FailedRestarts(), time.Now())
 }
 
 // unionSuppressed merges the suppression sets of the two mechanisms that stop apps on purpose.
@@ -3313,3 +3347,61 @@ func integrityOKMsg(res backup.IntegrityResult) string {
 	}
 	return msg + ")"
 }
+
+// updateGuardsAdapter implements stacks.UpdateGuards over the backup manager (slice 4). The stacks
+// package cannot import backup, so this is the one place the two meet. Nil-safe on b: a box with
+// backup disabled has no restore point, and the update is refused for that true reason.
+type updateGuardsAdapter struct {
+	b *backup.Manager
+	q *quiesce.Loop
+}
+
+func (a *updateGuardsAdapter) HoldFor(name string) (bool, string) {
+	if a.b == nil {
+		return false, ""
+	}
+	return a.b.RestoreHoldFor(name)
+}
+
+func (a *updateGuardsAdapter) Busy(name string) (bool, string) {
+	if a.q.SuppressedStacks()[name] {
+		return true, "a whole-guest backup (quiesce) is holding it"
+	}
+	if a.b == nil {
+		return false, ""
+	}
+	return a.b.UpdateBusy(name)
+}
+
+func (a *updateGuardsAdapter) RestorePoint(name string) (stacks.UpdateRestorePoint, error) {
+	if a.b == nil {
+		return stacks.UpdateRestorePoint{}, fmt.Errorf("backup is not enabled on this box")
+	}
+	rp, err := a.b.Tier2UnitRestorePoint(name)
+	if err != nil {
+		return stacks.UpdateRestorePoint{}, err
+	}
+	at, proven := rp.ProvenCopyTime()
+	return stacks.UpdateRestorePoint{Restorable: rp.Restorable, Proven: proven, ProvenAt: at}, nil
+}
+
+func (a *updateGuardsAdapter) BackupNow(ctx context.Context, name string) error {
+	if a.b == nil {
+		return fmt.Errorf("backup is not enabled on this box")
+	}
+	return a.b.RunAppBackupNow(ctx, name)
+}
+
+func (a *updateGuardsAdapter) SafetyDump(ctx context.Context, name string) ([]string, error) {
+	if a.b == nil {
+		return nil, fmt.Errorf("backup is not enabled on this box")
+	}
+	return a.b.WriteUpdateSafetyDump(ctx, name)
+}
+
+func (a *updateGuardsAdapter) HoldAfterFailedUpdate(name string, at, provenCopyAt time.Time) error {
+	if a.b == nil {
+		return fmt.Errorf("backup is not enabled on this box — the hold cannot be recorded")
+	}
+	return a.b.HoldAfterFailedUpdate(name, at, provenCopyAt)
+}

controller/cmd/controller/slice4_wiring_test.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"strings"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+)
+
+// Update arc slice 4 — the seams are WIRED, in the order that makes them true. A seam built and never
+// wired has shipped in this project seven times; each assertion here fails if a line moves or goes.
+
+func slice4CallLines(t *testing.T) (map[string][]int, *ast.File, *token.FileSet) {
+	t.Helper()
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "main.go", nil, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	lines := map[string][]int{}
+	ast.Inspect(f, func(n ast.Node) bool {
+		if call, ok := n.(*ast.CallExpr); ok {
+			if sel, ok := call.Fun.(*ast.SelectorExpr); ok {
+				lines[sel.Sel.Name] = append(lines[sel.Sel.Name], fset.Position(call.Pos()).Line)
+			} else if id, ok := call.Fun.(*ast.Ident); ok {
+				lines[id.Name] = append(lines[id.Name], fset.Position(call.Pos()).Line)
+			}
+		}
+		return true
+	})
+	return lines, f, fset
+}
+
+func TestSlice4_UpdateGuardsAreWiredAtStartup(t *testing.T) {
+	lines, _, _ := slice4CallLines(t)
+	if len(lines["SetUpdateGuards"]) == 0 {
+		t.Fatal("SetUpdateGuards is never called — every update would be refused as unwired")
+	}
+	if len(lines["RecoverUpdates"]) == 0 || len(lines["runBootReconcile"]) == 0 {
+		t.Fatal("RecoverUpdates or the boot reconciler is missing from main.go")
+	}
+	if lines["RecoverUpdates"][0] > lines["runBootReconcile"][0] {
+		t.Errorf("RecoverUpdates (line %d) must run BEFORE the boot reconciler (line %d), or the sweep can start a half-updated app",
+			lines["RecoverUpdates"][0], lines["runBootReconcile"][0])
+	}
+	if len(lines["ResumeInterruptedUpdates"]) == 0 || lines["ResumeInterruptedUpdates"][0] < lines["SetUpdateGuards"][0] {
+		t.Error("ResumeInterruptedUpdates must run AFTER SetUpdateGuards — a resumed update that fails must be able to HOLD")
+	}
+}
+
+func funcBodySource(t *testing.T, f *ast.File, fset *token.FileSet, recv, name string) string {
+	t.Helper()
+	for _, d := range f.Decls {
+		fn, ok := d.(*ast.FuncDecl)
+		if !ok || fn.Name.Name != name || fn.Body == nil {
+			continue
+		}
+		if recv != "" {
+			if fn.Recv == nil {
+				continue
+			}
+			id, ok := fn.Recv.List[0].Type.(*ast.Ident)
+			if !ok || id.Name != recv {
+				continue
+			}
+		}
+		var names []string
+		ast.Inspect(fn.Body, func(n ast.Node) bool {
+			if sel, ok := n.(*ast.SelectorExpr); ok {
+				names = append(names, sel.Sel.Name)
+			}
+			return true
+		})
+		return strings.Join(names, " ")
+	}
+	t.Fatalf("func %s.%s not found", recv, name)
+	return ""
+}
+
+func TestSlice4_BootSweepAndDeadAppAlarmKnowAboutUpdates(t *testing.T) {
+	_, f, fset := slice4CallLines(t)
+	if !strings.Contains(funcBodySource(t, f, fset, "bootDriveGate", "MayStart"), "IsUpdating") {
+		t.Error("the boot sweep must refuse an app a guarded update is moving")
+	}
+	if !strings.Contains(funcBodySource(t, f, fset, "", "scanDeployedAppRunStates"), "UpdatingStacks") {
+		t.Error("the dead-app alarm must not count an app the update itself is recreating")
+	}
+}
+
+// The shared start gate names WHICH hold it is honouring — an operator reading the boot log must not
+// be told a restore failed when an update did.
+func TestSlice4_DriveStartGate_NamesAnUpdateHold(t *testing.T) {
+	sett := holdTestSettings(t)
+	if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "bookstack", At: "2026-09-13T08:00:00Z", Reason: settings.HoldReasonUpdateFailed}); err != nil {
+		t.Fatal(err)
+	}
+	ok, why := driveStartGate{sett: sett}.MayStart("bookstack")
+	if ok || !strings.Contains(why, "held after a failed update") {
+		t.Errorf("ok=%v why=%q", ok, why)
+	}
+}