R-645: the night backup skips an app whose pinned version is not the one it runs (09 decision 142)

Every night leg (DB dump, volume dump, recovery-unit capture, Tier-2 mirror) now leaves alone an app
whose app.yaml pin (pinned_images) differs from its running record (installed_images) - the state a
failed update leaves behind. A hold lifted by hand (--clear-restore-hold + restart) no longer lets the
capture write the just-failed definition over the good unit. Unknown (no pin, no record, a service not
observed) never skips. The log says it per leg; the backups page shows one amber line, hu + en
(backup.status.version_skip). Seam: backup.Manager.SetVersionCheck <- stacks.Manager.PinNotRunning.

Tests: TestR645_HandLiftedHoldKeepsTheGoodUnit (whole night run + Tier 2, unit tree fingerprint),
TestR645_VersionSkipSentence, TestR645_PinNotRunning_*, TestR645_BackupRowSaysTheNightBackupSkipsIt,
TestR645_VersionCheckIsWiredAtStartup.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/backup/backup.go

@@ -202,6 +202,8 @@ type Manager struct {
 
 	// updatingCheck (slice 4) — nil-safe; see isHeld / SetUpdatingCheck.
 	updatingCheck func(stackName string) bool
+	// versionCheck (R-645) — nil-safe; see versionSkip / SetVersionCheck.
+	versionCheck func(stackName string) (detail string, mismatch bool)
 	// undoCopyRemover (R-671, v0.272.0) deletes an app's leftover undo copies — stacks.Manager.RemoveUndoCopies,
 	// wired in main.go (SetUndoCopyRemover). nil-safe: without it the copies stay, as before.
 	undoCopyRemover func(stackName string) int
@@ -620,6 +622,14 @@ func (m *Manager) runDBDumpsInternal(ctx context.Context) error {
 			continue
 		}
 
+		// R-645 (09 §3 decision 142): an app not running its pinned version is left alone — its dump
+		// would land in the unit beside the good definition and become the restore point. A SKIP: the
+		// unit keeps the last good copy, and the backups page says why.
+		if why, skip := m.versionSkip(db.StackName); skip {
+			m.logger.Printf("[WARN] [backup] Skipping DB dump for %s — it is not running its pinned version (%s); the last good backup is kept (R-645)", db.StackName, why)
+			summary = append(summary, fmt.Sprintf("SKIP %s (not running its pinned version)", db.ContainerName))
+			continue
+		}
 		// R-181: the reserve, BEFORE the first byte of this app's backup is written. This is usually
 		// where an app's verdict is taken, because the DB leg runs first; the volume leg and the
 		// capture then read the same memo. SKIP, not FAIL — a deliberate hold is not a broken dump,
@@ -761,6 +771,12 @@ func (m *Manager) runVolumeDumps() (summary []string, dumped int, allOK bool) {
 			summary = append(summary, fmt.Sprintf("SKIP %s volumes (held)", stack.Name))
 			continue
 		}
+		// R-645: never stop, dump or restart an app that is not running its pinned version.
+		if why, skip := m.versionSkip(stack.Name); skip {
+			m.logger.Printf("[WARN] [backup] Skipping volume dump for %s — it is not running its pinned version (%s); the last good backup is kept (R-645)", stack.Name, why)
+			summary = append(summary, fmt.Sprintf("SKIP %s volumes (not running its pinned version)", stack.Name))
+			continue
+		}
 		// Volume check FIRST — a volume-less stack must not be stopped at all (see gate-order note).
 		if len(m.stackProvider.GetDockerVolumes(stack.Name)) == 0 {
 			if m.isDebug() {

controller/internal/backup/r645_version_skip_test.go

@@ -0,0 +1,141 @@
+package backup
+
+import (
+	"context"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// R-645 (09 §3 decision 142) — the night backup skips an app whose pinned version is not the version
+// it runs.
+//
+// THE HAND-LIFT SHAPE, measured 2026-09-23 on 9202: docmost was HELD after a failed 0.95.0 → 0.96.0
+// update; `--clear-restore-hold docmost` + the restart ran, and three seconds later the capture wrote
+// the 0.96.0 definition into the unit the hold sentence had named. This drives the whole night run
+// (DB leg, volume leg, capture) and then Tier 2 over that state — the hold is gone, the stack dir names
+// the failed version, the pin says 0.96.0, the running record says 0.95.0 — and asserts the
+// CONSEQUENCE: the good unit's tree is byte-identical afterwards, and its compose still names 0.95.0.
+//
+// COMPANION RED-PROOF: make versionSkip return ("", false) — the unit's compose/docker-compose.yml then
+// names docmost/docmost:0.96.0 and the tree fingerprint differs, and this test fails.
+func TestR645_HandLiftedHoldKeepsTheGoodUnit(t *testing.T) {
+	h := newAdmissionHarness(t, "docmost", "free")
+	h.m.settings = slice4Settings(t)
+	ns := h.nsRoot()
+
+	// The good unit: written by an earlier backup of 0.95.0. Its manifest carries no data stamps (a unit
+	// from before v0.275.0), so nothing else in the capture freezes its definition — this skip is the
+	// only thing standing between the failed definition and the restore point.
+	h.seedUnit(t, "docmost", 0)
+	goodCompose := "services:\n  docmost:\n    image: docmost/docmost:0.95.0\n"
+	if err := os.WriteFile(filepath.Join(RecoveryUnitComposePath(ns, "docmost"), "docker-compose.yml"), []byte(goodCompose), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	// The stack dir after the failed update: it names the version that just failed.
+	for _, app := range []string{"docmost", "free"} {
+		dir := filepath.Join(h.dir, "stacks", app)
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		img := "docmost/docmost:0.96.0"
+		if app == "free" {
+			img = "free/free:1.0"
+		}
+		if err := os.WriteFile(filepath.Join(dir, "docker-compose.yml"), []byte("services:\n  "+app+":\n    image: "+img+"\n"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// The hold existed and was lifted by hand — exactly the operator CLI's act.
+	if err := h.m.HoldAfterFailedUpdate("docmost", time.Now(), time.Now(), UpdateTierLocal); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := h.m.settings.ClearRestoreHold("docmost"); err != nil {
+		t.Fatal(err)
+	}
+	if held, _ := h.m.RestoreHoldFor("docmost"); held {
+		t.Fatal("setup: the hold must be lifted")
+	}
+
+	// stacks.Manager.PinNotRunning's answer for this shape (pinned by TestR645_PinNotRunning_* there).
+	h.m.SetVersionCheck(func(name string) (string, bool) {
+		if name == "docmost" {
+			return "docmost runs docmost/docmost:0.95.0, pinned docmost/docmost:0.96.0", true
+		}
+		return "", false
+	})
+
+	var dumped []string
+	h.m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) {
+		return []DiscoveredDB{
+			{StackName: "docmost", ContainerName: "docmost-postgres"},
+			{StackName: "free", ContainerName: "free-postgres"},
+		}, nil
+	}
+	h.m.dumpOne = func(_ context.Context, db DiscoveredDB, dumpDir string, _ *log.Logger, _ bool) DumpResult {
+		dumped = append(dumped, db.StackName)
+		_ = os.MkdirAll(dumpDir, 0o755)
+		p := filepath.Join(dumpDir, db.StackName+"-postgres.sql")
+		_ = os.WriteFile(p, []byte("-- FRESH DUMP OF THE FAILED VERSION\n"), 0o644)
+		return DumpResult{DB: db, FilePath: p, Size: 36}
+	}
+
+	unitRoot := RecoveryUnitPath(ns, "docmost")
+	before := treeFingerprint(t, unitRoot)
+
+	_ = h.m.runDBDumpsInternal(context.Background())
+	var mirrored []string
+	h.m.perAppTier2 = func(name string) error { mirrored = append(mirrored, name); return nil }
+	h.m.RunAllTier2()
+
+	if after := treeFingerprint(t, unitRoot); after != before {
+		t.Errorf("the good unit was rewritten by the night run after a hand-lifted hold:\nbefore:\n%s\nafter:\n%s", before, after)
+	}
+	b, _ := os.ReadFile(filepath.Join(RecoveryUnitComposePath(ns, "docmost"), "docker-compose.yml"))
+	if !strings.Contains(string(b), "docmost/docmost:0.95.0") {
+		t.Errorf("the unit's definition must still be the good 0.95.0 one, got:\n%s", b)
+	}
+	for _, list := range [][]string{dumped, h.volDumped, h.prov.stopped, mirrored} {
+		for _, n := range list {
+			if n == "docmost" {
+				t.Errorf("a night leg touched docmost (db=%v vol=%v stopped=%v mirrored=%v)", dumped, h.volDumped, h.prov.stopped, mirrored)
+			}
+		}
+	}
+	// It says so in the log — the operator's half of the ruling.
+	if !strings.Contains(h.logs.String(), "Recovery unit NOT captured for docmost — it is not running its pinned version") {
+		t.Errorf("the skip must be logged, log:\n%s", h.logs.String())
+	}
+	// Positive control: the app that runs its pin is backed up as before.
+	if len(dumped) != 1 || dumped[0] != "free" || len(h.volDumped) != 1 || h.volDumped[0] != "free" || len(mirrored) != 1 || mirrored[0] != "free" {
+		t.Errorf("positive control: the other app must still be dumped and mirrored (db=%v vol=%v mirrored=%v)", dumped, h.volDumped, mirrored)
+	}
+	if _, err := os.Stat(RecoveryUnitManifestPath(ns, "free")); err != nil {
+		t.Errorf("positive control: the other app's unit must be captured: %v", err)
+	}
+}
+
+// The page's half: the household reads one plain sentence, in their language; nothing when the app
+// runs its pin or when nothing is wired (unknown never skips).
+func TestR645_VersionSkipSentence(t *testing.T) {
+	m := &Manager{}
+	if skip, why := m.VersionSkipFor("docmost", "hu"); skip || why != "" {
+		t.Fatalf("no check wired must read as no skip, got (%v, %q)", skip, why)
+	}
+	m.SetVersionCheck(func(name string) (string, bool) { return "x", name == "docmost" })
+	skip, hu := m.VersionSkipFor("docmost", "hu")
+	if !skip || !strings.Contains(hu, "docmost") || !strings.Contains(hu, "jszakai ment") || !strings.Contains(hu, "Vedd fel") {
+		t.Errorf("hu sentence = %q", hu)
+	}
+	_, en := m.VersionSkipFor("docmost", "en")
+	if !strings.Contains(en, "night backup leaves it out") || strings.Contains(en, "jszakai") {
+		t.Errorf("en sentence = %q", en)
+	}
+	if skip, _ := m.VersionSkipFor("free", "hu"); skip {
+		t.Error("an app running its pin must not read as skipped")
+	}
+}

controller/internal/backup/recovery_unit.go

@@ -458,6 +458,17 @@ func (m *Manager) captureAllRecoveryUnits(dataRun bool) {
 		if m.isHeld(stack.Name) {
 			continue
 		}
+		// R-645 (09 §3 decision 142): the unit of an app that is not running its pinned version stays
+		// as it is — the measured case wrote a just-failed definition over the good unit within seconds
+		// of a hand-lifted hold. Said at WARN on a data run; the periodic refresh repeats it only at DEBUG.
+		if why, skip := m.versionSkip(stack.Name); skip {
+			if dataRun {
+				m.logger.Printf("[WARN] [backup] Recovery unit NOT captured for %s — it is not running its pinned version (%s); the last good unit is kept (R-645)", stack.Name, why)
+			} else if m.isDebug() {
+				m.logger.Printf("[DEBUG] [backup] recovery-unit refresh skipped for %s — not running its pinned version (%s)", stack.Name, why)
+			}
+			continue
+		}
 		m.noteAttempted(stack.Name)
 		// The reserve, checked BEFORE anything is written. Per app, and the loop continues.
 		if !m.admitApp(stack.Name) {

controller/internal/backup/tier2.go

@@ -464,6 +464,11 @@ func (m *Manager) RunAllTier2() {
 			m.logger.Printf("[WARN] [backup] Tier 2 skipped for %s — the app is HELD; its copy is the restore point and is preserved", stack.Name)
 			continue
 		}
+		// R-645: the mirror of an app not running its pinned version stays as it is.
+		if why, skip := m.versionSkip(stack.Name); skip {
+			m.logger.Printf("[WARN] [backup] Tier 2 skipped for %s — it is not running its pinned version (%s); the last good copy is kept (R-645)", stack.Name, why)
+			continue
+		}
 		runOne := m.perAppTier2
 		if runOne == nil {
 			runOne = m.RunTier2

controller/internal/backup/update_guard.go

@@ -600,6 +600,39 @@ func (m *Manager) isHeld(stackName string) bool {
 	return m.updatingCheck != nil && m.updatingCheck(stackName)
 }
 
+// versionSkip (R-645, 09 §3 decision 142) reports whether the night backup must leave an app alone
+// because the version it is pinned to is NOT the version it runs, and names the difference.
+//
+// THE MEASURED CASE (9202, 2026-09-23): a failed update left docmost pinned to 0.96.0 while its running
+// record said 0.95.0. Lifting the hold by hand (`--clear-restore-hold` + the restart) let the capture
+// write the 0.96.0 definition — the version that had just failed — over the recovery unit the hold
+// sentence named, within seconds. The hold is what kept the legs off the app (isHeld); once it is gone,
+// this is what does. Where a leg also asks isHeld it asks this AFTER it, so a held app keeps its own line.
+//
+// Nil check or unknown ⇒ false (back the app up as before): a missing backup is the worse failure.
+// Pinned by TestR645_HandLiftedHoldKeepsTheGoodUnit (r645_version_skip_test.go).
+func (m *Manager) versionSkip(stackName string) (string, bool) {
+	if m == nil || m.versionCheck == nil {
+		return "", false
+	}
+	return m.versionCheck(stackName)
+}
+
+// VersionSkipFor is versionSkip for the backups page: the household's sentence in lang, or false.
+func (m *Manager) VersionSkipFor(stackName, lang string) (bool, string) {
+	if _, skip := m.versionSkip(stackName); !skip {
+		return false, ""
+	}
+	return true, util.Text(lang, "backup.status.version_skip", stackName)
+}
+
+// SetVersionCheck wires the "is this app running its pinned version" question (stacks.Manager.PinNotRunning).
+// INIT-ONLY, in main.go — pinned by TestR645_VersionCheckIsWiredAtStartup. The backup package cannot
+// import stacks, which is why it is a seam.
+func (m *Manager) SetVersionCheck(fn func(stackName string) (string, bool)) {
+	m.versionCheck = fn
+}
+
 // SetUpdatingCheck wires the "is a guarded update moving this app" question (stacks.Manager.IsUpdating).
 // INIT-ONLY, in main.go — pinned by TestSlice4_UpdatingCheckIsWiredAtStartup. The backup package cannot
 // import stacks, which is why it is a seam.