R-1342: a restore copy under the stable-start watch is not deletable (D62 gap); an app removed during the watch raises no alarm — tests red first; commit only

CHANGELOG.md

@@ -1,3 +1,17 @@
+## Unreleased (2026-10-29, after v0.347.0) — R-1342: the restore copy is safe during the stable-start watch; a removed app does not alarm; commit only, rides the next release
+
+**MinAgent: 0.158.1** (unchanged). Found live on Tester 1 at 10:39 CET: a false `backup_run_failures` mail („Gokapi did
+not start … the pre-restore copy is kept") after the copy had been deleted by hand mid-watch and the app removed.
+- **(1) D62 gap — `internal/backup/restore_copies.go`:** while a stable-start watch (`releaseAfterStableStart`, and the
+  late-start watcher) holds a copy, the copy reads „checking" in the list (one line, no delete date) and
+  `DeleteRestoreCopy` refuses it (`ErrRestoreCopyNeeded`) — the operator's rule is „never while an operation runs on the
+  app". The mark lives in `Manager.watching` under `m.mu`, the lock the delete checks under.
+- **(2) R-1270 watch — `internal/backup/restore_verdict.go`:** the watch ends quietly (`stableRemoved`, one INFO line, no
+  alarm) when the provider says the app was removed (`StackRemovalReporter`; the production `stackAdapter.StackRemoved`
+  with a compile-time assertion in `cmd/controller/main.go`). An installed app that never runs stably still alarms once.
+- Tests `TestR1342_ACopyUnderTheStableWatchIsNotDeletable`, `TestR1342_AnAppRemovedDuringTheWatchRaisesNoAlarm` (each
+  red by a mutation), the control `TestR1342_AnInstalledUnstableAppStillAlarms`, and `TestR1342_EveryNeededReasonHasItsSentence`.
+
 ## v0.347.0 (2026-10-29) — D62: restore copies kept 30 days, listed, deletable by hand; R-1256: four restore edges, all proven live (D63); R-1254 controller half; R-528: the host's OOM count; an unknown app is a 404 — RELEASED (image `sha256:c79b06e6…`, from `fd75c7c`, CI 2491; rc.1 = `e3fce09` on scratch 9202); per-box floors 09:57 CET, global floor 2026-10-29 10:31 CET on the operator's yes (MinAgent 0.158.1)
 
 ### 2026-10-29 (attended) — D62 and D63

controller/cmd/controller/main.go

@@ -3197,6 +3197,17 @@ func (a *stackAdapter) StopStack(name string) error {
 	return a.mgr.StopStack(name)
 }
 
+// StackRemoved answers backup's optional StackRemovalReporter (R-1342): the app is known and no longer installed — a
+// stable-start watch then ends quietly instead of alarming „did not start". An app this box does not know at all is
+// not „removed" (no answer → the watch goes on as before).
+func (a *stackAdapter) StackRemoved(name string) bool {
+	s, ok := a.mgr.GetStack(name)
+	return ok && !s.Deployed
+}
+
+// The watch reaches StackRemoved through a type assertion; this line makes the adapter's answer a compile-time fact.
+var _ backup.StackRemovalReporter = (*stackAdapter)(nil)
+
 // IsDeploying answers backup's optional deployingReporter (R-634): the volume leg asks it again
 // immediately before it stops an app, because its list is taken once at the start of the run.
 func (a *stackAdapter) IsDeploying(name string) bool {

controller/internal/backup/backup.go

@@ -340,6 +340,9 @@ type Manager struct {
 	// R-550: persistence of the above (restore_record.go) and the per-app interrupted notices.
 	opRecordPath  string
 	opInterrupted map[string]RestoreOpResult
+	// R-1342: the restore copies a stable-start watch still holds (copy dir → true), guarded by mu. While one is here the
+	// copy is the way back if the restored app turns out unstable: „Törlés most" refuses it (restore_copies.go).
+	watching map[string]bool
 	// R-519: the app-data run record (run_record.go) and the start of the last run a stop cut off (zero = none).
 	runRecordPath  string
 	runInterrupted time.Time

controller/internal/backup/r1342_watch_test.go

@@ -0,0 +1,122 @@
+package backup
+
+import (
+	"errors"
+	"io"
+	"log"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+)
+
+// r1342Drive registers the rig's app drive, as every drive is registered on a box, so the list and the delete see it.
+func r1342Drive(t *testing.T, m *Manager) {
+	t.Helper()
+	if m.settings == nil {
+		sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), log.New(io.Discard, "", 0))
+		if err != nil {
+			t.Fatal(err)
+		}
+		m.settings = sett
+	}
+	if err := m.settings.AddStoragePath(settings.StoragePath{Path: m.GetAppDrivePath("app"), Label: "rig"}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// R-1342 (found live on Tester 1, 2026-10-29 10:39 CET): (1) while the box still watches whether a restored app runs
+// stably, its restore copy is the way back — „Törlés most" must not take it (D62: never while an operation runs on the
+// app); (2) an app REMOVED during that watch is not „did not start" — no operator alarm, no „the copy is kept".
+
+// removableProvider is the stable-watch rig's provider that can answer StackRemoved.
+type removableProvider struct {
+	*sampledProvider
+	removed atomic.Bool
+}
+
+func (p *removableProvider) StackRemoved(string) bool { return p.removed.Load() }
+
+// RED-PROOF: drop the watching check from restoreCopyNeededFor / DeleteRestoreCopy → the copy is offered and deleted
+// mid-watch → FAILS.
+func TestR1342_ACopyUnderTheStableWatchIsNotDeletable(t *testing.T) {
+	withStable(t, 400*time.Millisecond, 3*time.Second)
+	m, rig, alarms := fixARig(t, func(int) bool { return true },
+		func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0)} })
+	r1342Drive(t, m)
+	m.BeginRestoreOp("restore", "app")
+	if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
+		t.Fatal(err)
+	}
+	m.EndRestoreOp(true, "ok")
+	copies := m.RestoreCopies()
+	if len(copies) != 1 || copies[0].Needed != RestoreCopyNeededChecking || !copies[0].DeleteOn.IsZero() {
+		t.Fatalf("mid-watch the copy must read „checking\", got %+v", copies)
+	}
+	if _, err := m.DeleteRestoreCopy("app", copies[0].Stamp); !errors.Is(err, ErrRestoreCopyNeeded) {
+		t.Fatalf("a delete mid-watch: %v, want ErrRestoreCopyNeeded", err)
+	}
+	if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
+		t.Fatalf("the copy must stay through the watch, found %v", left)
+	}
+	if left := waitNoCopies(t, rig.undoRoot); len(left) != 0 {
+		t.Fatalf("after the stable window the watch releases it, found %v", left)
+	}
+	if len(*alarms) != 0 {
+		t.Fatalf("no alarm: %v", *alarms)
+	}
+	if m.watchingCopy(filepath.Join(rig.undoRoot, copies[0].Stamp)) {
+		t.Fatal("the watch mark must go when the watch ends")
+	}
+}
+
+// RED-PROOF: drop the stableRemoved case → the removed app „did not run stably" → one alarm → FAILS.
+func TestR1342_AnAppRemovedDuringTheWatchRaisesNoAlarm(t *testing.T) {
+	withStable(t, 2*time.Second, 400*time.Millisecond)
+	var up atomic.Bool
+	up.Store(true)
+	m, rig, alarms := fixARig(t, func(int) bool { return up.Load() },
+		func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0)} })
+	rp := &removableProvider{sampledProvider: m.stackProvider.(*sampledProvider)}
+	m.stackProvider = rp
+	r1342Drive(t, m)
+	m.BeginRestoreOp("restore", "app")
+	if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
+		t.Fatal(err)
+	}
+	m.EndRestoreOp(true, "ok")
+	// the household removes the app inside the watch
+	up.Store(false)
+	rp.removed.Store(true)
+	time.Sleep(700 * time.Millisecond)
+	if len(*alarms) != 0 {
+		t.Fatalf("an app removed during the watch must not alarm „did not start\": %v", *alarms)
+	}
+	copies := m.RestoreCopies()
+	if len(copies) != 1 || copies[0].Needed != "" {
+		t.Fatalf("after the watch ended the copy is an unused one (sweep / Törlés most): %+v", copies)
+	}
+	_ = rig
+}
+
+// Control: an app that stays installed and never runs stably still alarms once (the R-1270 rule is unchanged).
+func TestR1342_AnInstalledUnstableAppStillAlarms(t *testing.T) {
+	withStable(t, 2*time.Second, 400*time.Millisecond)
+	var up atomic.Bool
+	up.Store(true)
+	m, _, alarms := fixARig(t, func(int) bool { return up.Load() },
+		func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0)} })
+	m.stackProvider = &removableProvider{sampledProvider: m.stackProvider.(*sampledProvider)}
+	m.BeginRestoreOp("restore", "app")
+	if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
+		t.Fatal(err)
+	}
+	m.EndRestoreOp(true, "ok")
+	up.Store(false)
+	time.Sleep(700 * time.Millisecond)
+	if len(*alarms) != 1 {
+		t.Fatalf("an installed app that never runs stably alarms once, got %v", *alarms)
+	}
+}

controller/internal/backup/restore_copies.go

@@ -28,6 +28,9 @@ const (
 	RestoreCopyNeededHold    = "hold"    // the app is held (a failed put-back keeps its copy, D20) — support decides
 	RestoreCopyNeededPutBack = "putback" // a put-back waits for its drive (R-1251) or a cut restore names it (D22)
 	RestoreCopyNeededUnknown = "unknown" // the app-stop marker cannot be read: nothing is offered (fail closed)
+	// RestoreCopyNeededChecking (R-1342): the box still watches whether the restored app runs stably (up to 30 minutes
+	// after the restore, R-1270) — the copy is the way back if it does not.
+	RestoreCopyNeededChecking = "checking"
 )
 
 var (
@@ -119,7 +122,10 @@ func (m *Manager) RestoreCopies() []RestoreCopy {
 					continue
 				}
 				rc := RestoreCopy{App: app, Stamp: c.Name(), At: at, SizeBytes: dirSize(dir), Path: dir, Needed: needed}
-				if needed == "" {
+				if rc.Needed == "" && m.watchingCopy(dir) {
+					rc.Needed = RestoreCopyNeededChecking
+				}
+				if rc.Needed == "" {
 					rc.DeleteOn = at.Add(restoreCopyUnusedAge)
 				}
 				out = append(out, rc)
@@ -174,9 +180,37 @@ func (m *Manager) DeleteRestoreCopy(app, stamp string) (RestoreCopy, error) {
 	if found.Needed = m.restoreCopyNeededFor(app, named, ok); found.Needed != "" {
 		return *found, ErrRestoreCopyNeeded
 	}
+	if m.watching[found.Path] { // R-1342: under m.mu, the same lock the watch marks its copy under
+		found.Needed = RestoreCopyNeededChecking
+		return *found, ErrRestoreCopyNeeded
+	}
 	if err := os.RemoveAll(found.Path); err != nil {
 		return *found, fmt.Errorf("removing the restore copy %s: %w", found.Path, err)
 	}
 	_ = os.Remove(filepath.Dir(found.Path)) // the per-app folder, when it is now empty
 	return *found, nil
 }
+
+// setWatching marks (on) or clears a copy a stable-start watch holds (R-1342).
+func (m *Manager) setWatching(dir string, on bool) {
+	if dir == "" {
+		return
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if on {
+		if m.watching == nil {
+			m.watching = map[string]bool{}
+		}
+		m.watching[dir] = true
+		return
+	}
+	delete(m.watching, dir)
+}
+
+// watchingCopy reports whether a stable-start watch still holds the copy.
+func (m *Manager) watchingCopy(dir string) bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.watching[dir]
+}

controller/internal/backup/restore_verdict.go

@@ -71,9 +71,14 @@ func orNone(s string) string {
 // so and the kept copy is removed (the normal rule after a successful restore). A new restore of the app ends the watch
 // untouched.
 func (m *Manager) watchLateStart(stack, copyDir string, since time.Time, watch time.Duration) {
+	m.setWatching(copyDir, true) // R-1342: the copy is the way back while the app may still start
+	defer m.setWatching(copyDir, false)
 	switch m.waitStableStart(stack, since, since.Add(watch)) {
 	case stableAborted:
 		return // a new restore of this app owns it now
+	case stableRemoved:
+		m.logger.Printf("[INFO] [backup] %s: the app was removed while the box waited for it to start — the watch ends (R-1342)", stack)
+		return
 	case stableNot:
 		m.logger.Printf("[WARN] [backup] %s: still not running stably %s after its restore — the restore copy stays for support (%s) (R-1270)", stack, lateStartWatch, orNone(copyDir))
 		return
@@ -133,12 +138,26 @@ type containerSampler interface {
 	ContainerSamples(name string) ([]ContainerSample, error)
 }
 
+// StackRemovalReporter (R-1342) is the provider's optional answer to „was this app removed?" — true only when the app is
+// known and no longer installed. A watch that sees it ends quietly: an app the household removed did not „fail to
+// start". The production stackAdapter implements it (a compile-time assertion in cmd/controller/main.go).
+type StackRemovalReporter interface {
+	StackRemoved(name string) bool
+}
+
+// stackRemoved asks the provider; a provider without the answer never reads „removed".
+func (m *Manager) stackRemoved(stack string) bool {
+	r, ok := m.stackProvider.(StackRemovalReporter)
+	return ok && r.StackRemoved(stack)
+}
+
 type stableResult int
 
 const (
 	stableYes stableResult = iota
 	stableNot
 	stableAborted
+	stableRemoved // R-1342: the app was removed during the watch
 )
 
 // stableSample is one look: ok, and the fingerprint (container id → restart count) a stable window must keep.
@@ -201,6 +220,9 @@ func (m *Manager) waitStableFor(stack string, watchFrom, deadline time.Time, win
 		if m.restoreBegunAfter(stack, watchFrom) {
 			return stableAborted
 		}
+		if m.stackRemoved(stack) {
+			return stableRemoved
+		}
 		ok, fp := m.stableSample(stack)
 		switch {
 		case !ok:
@@ -226,13 +248,17 @@ func (m *Manager) releaseAfterStableStart(stack, kind string, undo *liveVolumeCo
 	dir := undo.dir
 	from := time.Now() // taken HERE, before the goroutine: a restore begun after this moment is a new one
 	until := from.Add(stableReleaseWatch)
+	m.setWatching(dir, true) // R-1342: „Törlés most" refuses the copy while the watch holds it
 	go func() {
+		defer m.setWatching(dir, false)
 		switch m.waitStableStart(stack, from, until) {
 		case stableYes:
 			removeCopyDir(m, stack, dir)
 			m.logger.Printf("[INFO] [backup] %s: ran stably for %s after its %s — the restore copy is released (R-1270)", stack, stableStartWindow, kind)
 		case stableAborted:
 			m.logger.Printf("[INFO] [backup] %s: a new restore began before the %s's start was stable — its restore copy stays for support (%s)", stack, kind, dir)
+		case stableRemoved:
+			m.logger.Printf("[INFO] [backup] %s: the app was removed during the stable-start check after its %s — no alarm; the restore copy, if the remove left it, is an unused one (%s) (R-1342)", stack, kind, dir)
 		default:
 			m.logger.Printf("[ERROR] [backup] %s: did not run stably within %s after its %s — the restore copy stays for support (%s) (R-1270)", stack, stableReleaseWatch, kind, dir)
 			if m.restoreNotStartedNotify != nil {
@@ -262,6 +288,8 @@ func (m *Manager) restoredAppStarted(stack string) error {
 		return nil
 	case stableAborted:
 		return fmt.Errorf("a new restore of %s began", stack)
+	case stableRemoved:
+		return fmt.Errorf("%s was removed during the restore's start check", stack)
 	}
 	return fmt.Errorf("stack %s did not run stably for %s (it restarted or turned unhealthy) after restore", stack, verdictStableWindow)
 }

controller/internal/i18n/locales/en.json

@@ -2641,6 +2641,7 @@
   "restore_copy.needed.hold": "Cannot be deleted: the app is stopped, and this copy is its way back. Support decides about it.",
   "restore_copy.needed.putback": "Cannot be deleted: an interrupted restore puts the data back from this copy.",
   "restore_copy.needed.unknown": "Cannot be deleted now: the box cannot tell whether this copy is still needed.",
+  "restore_copy.needed.checking": "Cannot be deleted now: the box is still checking that the app started well after the restore (up to 30 minutes).",
   "restore_copy.delete": "Delete now",
   "restore_copy.warning": "This is the copy of %s’s data from before the restore on %s. If you delete it, the restore can no longer be undone, and this copy cannot be brought back.",
   "restore_copy.app_busy": "Cannot be deleted now: an operation runs on the app. Try again when it has ended.",

controller/internal/i18n/locales/hu.json

@@ -2629,6 +2629,7 @@
   "restore_copy.needed.hold": "Nem törölhető: az alkalmazás le van állítva, és ez a másolat az út vissza. Az ügyfélszolgálat dönt róla.",
   "restore_copy.needed.putback": "Nem törölhető: egy félbeszakadt visszaállítás ebből a másolatból teszi vissza az adatokat.",
   "restore_copy.needed.unknown": "Most nem törölhető: a doboz nem tudja eldönteni, kell-e még ez a másolat.",
+  "restore_copy.needed.checking": "Most nem törölhető: a doboz még figyeli, hogy az alkalmazás rendben elindult-e a visszaállítás után (legfeljebb 30 percig).",
   "restore_copy.delete": "Törlés most",
   "restore_copy.warning": "Ez a(z) %s adatainak másolata a %s-i visszaállítás előttről. Ha törlöd, a visszaállítás többé nem vonható vissza, és ez a másolat nem hozható vissza.",
   "restore_copy.app_busy": "Most nem törölhető: az alkalmazáson éppen egy művelet fut. Próbáld újra, ha véget ért.",

controller/internal/web/d62_restore_copies_test.go

@@ -231,3 +231,18 @@ func TestD62_ADeleteRemovesOnlyThatCopyAndStoresTheEvent(t *testing.T) {
 		t.Fatalf("second press code %d", code)
 	}
 }
+
+// R-1342: every reason a copy can be needed has its one-line sentence in both languages (the row's key is built from the
+// reason, so a missing key would show the raw key). RED-PROOF: remove restore_copy.needed.checking from BOTH bundles →
+// FAILS (seen 2026-10-29; a key missing only in en.json falls back to the Hungarian line — i18n_missing_gate counts that).
+func TestR1342_EveryNeededReasonHasItsSentence(t *testing.T) {
+	f := newD62Fixture(t)
+	for _, why := range []string{backup.RestoreCopyNeededHold, backup.RestoreCopyNeededPutBack, backup.RestoreCopyNeededUnknown, backup.RestoreCopyNeededChecking} {
+		for _, lang := range []string{"hu", "en"} {
+			key := "restore_copy.needed." + why
+			if got := f.s.msgLang(lang, key); got == "" || got == key || strings.Contains(got, "restore_copy.") {
+				t.Errorf("%s/%s renders %q", lang, key, got)
+			}
+		}
+	}
+}

controller/scripts/i18n_go_keys.json

@@ -320,6 +320,7 @@
     "restore_copy.needed.hold": "BORN AS A KEY (D62, operator ruling 2026-10-29) -- restore copies in the kept-copies list: the line, the warning, the delete's refusals and flash. Pinned by internal/web/d62_restore_copies_test.go.",
     "restore_copy.needed.putback": "BORN AS A KEY (D62, operator ruling 2026-10-29) -- restore copies in the kept-copies list: the line, the warning, the delete's refusals and flash. Pinned by internal/web/d62_restore_copies_test.go.",
     "restore_copy.needed.unknown": "BORN AS A KEY (D62, operator ruling 2026-10-29) -- restore copies in the kept-copies list: the line, the warning, the delete's refusals and flash. Pinned by internal/web/d62_restore_copies_test.go.",
+    "restore_copy.needed.checking": "BORN AS A KEY (R-1342, 2026-10-29) -- a restore copy a stable-start watch still holds. Pinned by internal/web/d62_restore_copies_test.go TestR1342_EveryNeededReasonHasItsSentence.",
     "note.tier2.emptied_kept": "BORN AS A KEY (D25, operator ruling 2026-10-21; R-1175/R-1181) -- kept second-drive copies: their list line, reasons, the emptied-folder note and the „Eltávolítás” flashes. Pinned by internal/web/kept_copies_test.go and internal/backup/r1175_shares_kept_test.go.",
     "flash.kept.removed": "BORN AS A KEY (D25, operator ruling 2026-10-21; R-1175/R-1181) -- kept second-drive copies: their list line, reasons, the emptied-folder note and the „Eltávolítás” flashes. Pinned by internal/web/kept_copies_test.go and internal/backup/r1175_shares_kept_test.go.",
     "flash.kept.needs_confirmation": "BORN AS A KEY (D25, operator ruling 2026-10-21; R-1175/R-1181) -- kept second-drive copies: their list line, reasons, the emptied-folder note and the „Eltávolítás” flashes. Pinned by internal/web/kept_copies_test.go and internal/backup/r1175_shares_kept_test.go.",