controller v0.265.0: R-634 cause fixed, held apps say so, OOM storm alarm, R-647 leftovers

R-634: a whole-box backup no longer stops/restarts a DEPLOYING app (the
measured cause of containers running under 'not deployed'); StopStack
and StartStack refuse a deploying stack for every caller.
R-625: held badge 'Stopped - restore needed', no Update button.
R-636: kernel oom_kill counter; 20+ in 30 min -> one app_oom_storm.
R-647: held error per reader, copy_holds key, two log wordings.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/stacks/deploy_errors.go

@@ -22,6 +22,11 @@ var (
 	ErrPathMissing = errors.New("path field does not exist")
 	// ErrNotEnoughMemory — the memory verdict refused the deploy (API: 400).
 	ErrNotEnoughMemory = errors.New("not enough memory")
+	// ErrStackDeploying — R-634 (v0.265.0): a stop or start was asked of a stack whose deploy is still
+	// running. Refused, because a `compose down` or a second `compose up -d` in the middle of the
+	// deploy's own `up` makes BOTH fail, and the deploy then records „not deployed" over whatever
+	// containers the race left behind. Pinned by TestR634_StopAndStartRefuseADeployingStack.
+	ErrStackDeploying = errors.New("stack is being deployed")
 )
 
 // msgHU renders a bundle message in Hungarian.

controller/internal/stacks/manager.go

@@ -1147,6 +1147,12 @@ func (m *Manager) StartStack(name string) error {
 	if !ok {
 		return fmt.Errorf("stack %q not found", name)
 	}
+	if stack.Deploying {
+		// R-634: a second `compose up -d` beside the deploy's own is the race that left apps running
+		// under „not deployed". The deploy brings the app up itself; nothing else may.
+		m.logger.Printf("[WARN] [stacks] StartStack %s refused: the app's deploy is still running (R-634)", name)
+		return fmt.Errorf("starting stack %s: %w", name, ErrStackDeploying)
+	}
 
 	if m.isDebug() {
 		m.logger.Printf("[DEBUG] [stacks] StartStack %s: current state=%s deployed=%v", name, stack.State, stack.Deployed)
@@ -1229,6 +1235,13 @@ func (m *Manager) StopStack(name string) error {
 	if !ok {
 		return fmt.Errorf("stack %q not found", name)
 	}
+	if stack.Deploying {
+		// R-634, the backstop for EVERY caller (backup, quiesce, restore, export, storage, the Stop
+		// button): `compose down` in the middle of a deploy was measured to make the deploy fail and
+		// record „not deployed" while the caller's own restart brought the containers back.
+		m.logger.Printf("[WARN] [stacks] StopStack %s refused: the app's deploy is still running (R-634)", name)
+		return fmt.Errorf("stopping stack %s: %w", name, ErrStackDeploying)
+	}
 
 	if m.isDebug() {
 		m.logger.Printf("[DEBUG] [stacks] StopStack %s: current state=%s deployed=%v containers=%d", name, stack.State, stack.Deployed, len(stack.Containers))

controller/internal/stacks/oom.go

@@ -1,7 +1,9 @@
 package stacks
 
 import (
+	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 )
 
@@ -18,6 +20,13 @@ type OOMContainer struct {
 	Stack     string
 	Container string
 	StartedAt string // identifies the container run — one event per run
+	// Kills is the kernel's own count of OOM kills in this container's cgroup (`memory.events`
+	// oom_kill) — R-636, v0.265.0. OOMKilled is a STICKY flag: it stays true for the container's whole
+	// life after ONE kill, so "the key re-fired" means nothing; this counter is what separates one
+	// hiccup from RomM's 4,530 kills in six hours. -1 when it could not be read.
+	Kills int64
+	// MemLimit / Peak are the cgroup's memory.max and memory.peak, as "<n>M" (or "max"); "" unread.
+	MemLimit, Peak string
 }
 
 // ScanOOMKilled inspects the containers of every deployed, running-ish stack in ONE docker call and
@@ -56,13 +65,53 @@ func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
 		}
 		cname := strings.TrimPrefix(f[0], "/")
 		if st, ok := owner[cname]; ok {
-			found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2]})
+			found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2], Kills: -1})
 		}
 	}
+	// R-636: only for the containers already flagged — one `docker exec` each, and the flagged set is
+	// empty on a healthy box. Read from INSIDE the container (its cgroup namespace makes
+	// /sys/fs/cgroup its own cgroup); the controller's own namespace cannot see the others.
+	for i := range found {
+		out, _ := m.execCommand("docker", "exec", found[i].Container, "cat",
+			"/sys/fs/cgroup/memory.events", "/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory.peak")
+		found[i].Kills, found[i].MemLimit, found[i].Peak = parseCgroupMemory(out)
+	}
 	m.setOOMCache(found)
 	return found, nil
 }
 
+// parseCgroupMemory reads `cat memory.events memory.max memory.peak`: the key/value lines give
+// oom_kill; the first bare line is memory.max, the second memory.peak (absent on older kernels — cat
+// then fails on that file and still prints the others). Unreadable → Kills -1.
+func parseCgroupMemory(out string) (kills int64, limit, peak string) {
+	kills = -1
+	var bare []string
+	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+		f := strings.Fields(line)
+		switch {
+		case len(f) == 2 && f[0] == "oom_kill":
+			if n, err := strconv.ParseInt(f[1], 10, 64); err == nil {
+				kills = n
+			}
+		case len(f) == 1:
+			bare = append(bare, f[0])
+		}
+	}
+	mb := func(v string) string {
+		if n, err := strconv.ParseInt(v, 10, 64); err == nil {
+			return fmt.Sprintf("%dM", n/(1024*1024))
+		}
+		return v // "max"
+	}
+	if len(bare) > 0 {
+		limit = mb(bare[0])
+	}
+	if len(bare) > 1 {
+		peak = mb(bare[1])
+	}
+	return kills, limit, peak
+}
+
 func (m *Manager) setOOMCache(found []OOMContainer) {
 	cache := map[string][]string{}
 	for _, o := range found {

controller/internal/stacks/oom_test.go

@@ -53,3 +53,46 @@ func TestScanOOMKilled_SeesKilledWorkerInRunningContainer(t *testing.T) {
 		t.Fatalf("cache not cleared after a clean scan: %v", got)
 	}
 }
+
+// R-636 — the kernel's kill counter, the limit and the peak are read from `cat memory.events
+// memory.max memory.peak` inside the container; an unreadable one is -1, never 0.
+func TestR636_ParseCgroupMemory(t *testing.T) {
+	out := "low 0\nhigh 0\nmax 4521\noom 4530\noom_kill 4530\noom_group_kill 0\n1342177280\n1341128704\n"
+	k, lim, peak := parseCgroupMemory(out)
+	if k != 4530 || lim != "1280M" || peak != "1279M" {
+		t.Fatalf("got kills=%d limit=%q peak=%q", k, lim, peak)
+	}
+	if k, lim, _ := parseCgroupMemory("oom_kill 3\nmax\n"); k != 3 || lim != "max" {
+		t.Fatalf("no memory.peak (older kernel), unlimited: got %d %q", k, lim)
+	}
+	if k, _, _ := parseCgroupMemory("OCI runtime exec failed: exec: \"cat\": executable file not found"); k != -1 {
+		t.Fatalf("an image without cat must read -1 (unknown), got %d", k)
+	}
+}
+
+// R-636 — the scan asks the kill counter of the FLAGGED containers only (one exec each; none on a
+// healthy box). COMPANION RED-PROOF (REPORT.md): drop the exec loop — Kills stays -1 and this fails.
+func TestR636_ScanReadsTheCounterOfFlaggedContainersOnly(t *testing.T) {
+	var execd []string
+	m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
+		"romm":      {Name: "romm", Deployed: true, Containers: []ContainerInfo{{Name: "romm", State: StateRunning}}},
+		"bookstack": {Name: "bookstack", Deployed: true, Containers: []ContainerInfo{{Name: "bookstack", State: StateRunning}}},
+	}}
+	m.execFn = func(name string, args ...string) (string, error) {
+		if args[0] == "exec" {
+			execd = append(execd, args[1])
+			return "oom_kill 377\n1342177280\n1341128704\n", nil
+		}
+		return "/bookstack|false|t0\n/romm|true|2026-09-22T09:08:00Z\n", nil
+	}
+	ooms, err := m.ScanOOMKilled()
+	if err != nil || len(ooms) != 1 {
+		t.Fatalf("ooms=%+v err=%v", ooms, err)
+	}
+	if ooms[0].Kills != 377 || ooms[0].MemLimit != "1280M" || ooms[0].Peak != "1279M" {
+		t.Errorf("the counter must be read for the flagged container: %+v", ooms[0])
+	}
+	if len(execd) != 1 || execd[0] != "romm" {
+		t.Errorf("exec must reach the flagged container only, got %v", execd)
+	}
+}

controller/internal/stacks/r634_deploying_test.go

@@ -0,0 +1,36 @@
+package stacks
+
+import (
+	"errors"
+	"testing"
+)
+
+// R-634 (v0.265.0) — the backstop for every caller: a stop or a start while the deploy's own
+// `compose up` is running is refused BEFORE any compose command.
+//
+// COMPANION RED-PROOF (REPORT.md): delete the Deploying check in StopStack — the stop falls through to
+// `compose down` and the error is no longer ErrStackDeploying.
+func TestR634_StopAndStartRefuseADeployingStack(t *testing.T) {
+	m, _, _, _ := newSlice4Manager(t)
+	// NEVER a real compose: the control below falls through to `compose down`, and the stack dir is
+	// named like a real app. An empty PATH + the v1 binary name guarantees the exec cannot find
+	// anything to run, on any machine this test runs on.
+	m.composeCmd = "docker-compose"
+	t.Setenv("PATH", t.TempDir())
+	m.mu.Lock()
+	m.stacks["nextcloud"].Deploying = true
+	m.mu.Unlock()
+	if err := m.StopStack("nextcloud"); !errors.Is(err, ErrStackDeploying) {
+		t.Fatalf("StopStack on a deploying stack = %v, want ErrStackDeploying", err)
+	}
+	if err := m.StartStack("nextcloud"); !errors.Is(err, ErrStackDeploying) {
+		t.Fatalf("StartStack on a deploying stack = %v, want ErrStackDeploying", err)
+	}
+	// Control: once the deploy is over, the same calls are not refused for THIS reason.
+	m.mu.Lock()
+	m.stacks["nextcloud"].Deploying = false
+	m.mu.Unlock()
+	if err := m.StopStack("nextcloud"); errors.Is(err, ErrStackDeploying) {
+		t.Fatal("control: a stack that is not deploying must not be refused as deploying")
+	}
+}

controller/internal/stacks/undo.go

@@ -471,7 +471,11 @@ func (m *Manager) emitUpdateEvent(kind, name string, entry *updateJournalEntry,
 	if entry != nil {
 		ev.From, ev.To = entry.PrevPin, entry.NewPin
 	}
-	m.logger.Printf("[INFO] [stacks] update %s: event %s (hold recorded: %v)", name, kind, holdRecorded)
+	if kind == UpdateEventHeld {
+		m.logger.Printf("[INFO] [stacks] update %s: event %s (hold recorded: %v)", name, kind, holdRecorded)
+	} else { // R-647: "hold recorded" means nothing for an undone update
+		m.logger.Printf("[INFO] [stacks] update %s: event %s", name, kind)
+	}
 	sink(ev)
 }
 

controller/internal/stacks/update.go

@@ -564,7 +564,7 @@ func (m *Manager) finishUpdate(name, phase, msg string) {
 // beside it for the page. key "" = `plain` is a finished sentence and is stored as it is.
 func (m *Manager) finishUpdateKey(name, phase, key, plain string, args ...interface{}) {
 	msg := plain
-	if key != "" {
+	if key != "" && key != UpdateErrorKeyHeld { // the held key names no bundle entry: plain IS the sentence
 		msg = util.Text(i18n.Default, key, args...)
 	}
 	m.mu.Lock()
@@ -590,12 +590,57 @@ func UpdatePhaseLabelIn(lang, phase string) string {
 
 // UpdateErrorIn is the stack's update sentence in lang (v0.264.0, R-606).
 func (s Stack) UpdateErrorIn(lang string) string {
-	if s.UpdateErrorKey == "" || lang == i18n.Default {
+	if s.UpdateErrorKey == "" || s.UpdateErrorKey == UpdateErrorKeyHeld || lang == i18n.Default {
 		return s.UpdateError
 	}
 	return util.Text(lang, s.UpdateErrorKey, s.UpdateErrorArgs...)
 }
 
+// UpdateErrorKeyHeld marks an UpdateError that IS the hold sentence (R-647, v0.265.0). It names no
+// bundle entry: the sentence is composed by the backup side, so UpdateErrorFor asks it for the
+// reader's language, and UpdateErrorIn (no manager to ask) returns the stored Hungarian.
+const UpdateErrorKeyHeld = "update.error.held"
+
+// holdLanguage is the OPTIONAL half of UpdateGuards that renders the hold sentence in a given
+// language (the production adapter implements it; the test fakes need not).
+type holdLanguage interface {
+	HoldForLang(name, lang string) (bool, string)
+}
+
+func holdForLang(g UpdateGuards, name, lang string) (bool, string) {
+	if g == nil {
+		return false, ""
+	}
+	if hl, ok := g.(holdLanguage); ok {
+		return hl.HoldForLang(name, lang)
+	}
+	return g.HoldFor(name)
+}
+
+// HoldReasonFor is a stack's hold sentence in the READER's language (R-647): "" when nothing holds it.
+// A Hungarian reader on a Hungarian box gets exactly Stack.HoldReason.
+func (m *Manager) HoldReasonFor(st Stack, lang string) string {
+	if st.HoldReason == "" {
+		return ""
+	}
+	if held, why := holdForLang(m.guards(), st.Name, lang); held && why != "" {
+		return why
+	}
+	return st.HoldReason
+}
+
+// UpdateErrorFor is a stack's update error in the READER's language (R-606 + R-647). A held update's
+// error is the hold sentence, rendered by the backup side for this reader.
+func (m *Manager) UpdateErrorFor(st Stack, lang string) string {
+	if st.UpdateErrorKey == UpdateErrorKeyHeld {
+		if held, why := holdForLang(m.guards(), st.Name, lang); held && why != "" {
+			return why
+		}
+		return st.UpdateError
+	}
+	return st.UpdateErrorIn(lang)
+}
+
 func (m *Manager) updateCompose(dir string, env []string, args ...string) (string, error) {
 	if m.updateComposeFn != nil {
 		return m.updateComposeFn(dir, env, args...)
@@ -880,7 +925,7 @@ func (m *Manager) failAndHold(ctx context.Context, name, dir string, env []strin
 		m.logger.Printf("[ERROR] [stacks] update %s: no UpdateGuards — the hold CANNOT be recorded", name)
 	} else if err := g.HoldAfterFailedUpdate(name, m.now(), rp, undoState); err != nil {
 		m.logger.Printf("[ERROR] [stacks] update %s: %v", name, err)
-	} else if _, w := g.HoldFor(name); w != "" {
+	} else if _, w := holdForLang(g, name, i18n.Default); w != "" {
 		holdWhy = w
 		m.markUpdateHeld(name)
 	}
@@ -890,8 +935,10 @@ func (m *Manager) failAndHold(ctx context.Context, name, dir string, env []strin
 	if holdWhy == "" {
 		m.finishUpdateKey(name, UpdatePhaseFailed, "update.error.hold_unsaved", "")
 	} else {
-		// The hold's own sentence (the page renders it per reader through RestoreHoldForLang).
-		m.finishUpdate(name, UpdatePhaseFailed, holdWhy)
+		// R-647 (v0.265.0): stored as the HELD key + the Hungarian sentence, so every reader — the
+		// `?lang=` switch, an operator reading a box in the other language — gets the hold sentence in
+		// THEIR language (UpdateErrorFor), and the stored text is Hungarian as for every other key.
+		m.finishUpdateKey(name, UpdatePhaseFailed, UpdateErrorKeyHeld, holdWhy)
 	}
 	m.emitUpdateEvent(UpdateEventHeld, name, entry, rp, holdWhy != "")
 }