v0.104.0: off-box unit discovery (durable, deployment-independent) + no-silent-success

offbox located each toggled app's recovery unit via AppNamespaceRoot→GetAppDrivePath,
which reads the app's LIVE app.yaml HDD_PATH and silently falls back to systemDataPath
when the app isn't deployed → looked on the wrong drive, backed up nothing, reported
ok/0 (DIAG root cause). Now:

- discoverOffboxUnit/offboxCandidateNSRoots scan the durable storage registry
  (schedulable non-decommissioned paths ∪ systemDataPath) for backups/primary/<app>,
  independent of deploy state; newest-by-manifest-CreatedAt wins on drive churn.
- RunOffboxBackup: runOffboxInternal returns (backedUp, missing, err); 0-of-N toggled →
  hard error + operator alert; partial → ok + new OffboxTarget.LastWarning (shown on
  /backups, preserved across config edit).
- AppNamespaceRoot + primary WRITE paths unchanged.
- 6 non-hollow tests (A-E + edge) + both companion red-proofs run (reverted).
- NOT yet live-validated against the Storage Box (spike creds torn down).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

controller/internal/backup/offbox.go

@@ -228,7 +228,14 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
 	start := time.Now()
 	_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.LastStatus = "running"; o.LastError = "" })
 
-	runErr := m.runOffboxInternal(ctx, apps, base, env)
+	backedUp, missing, runErr := m.runOffboxInternal(ctx, apps, base, env)
+
+	// No-silent-success: apps were toggled but NOTHING was captured (every unit missing) → promote to a
+	// hard error so the run reports "error" and the operator is alerted, instead of a misleading ok/0.
+	if runErr == nil && len(apps) > 0 && backedUp == 0 {
+		runErr = fmt.Errorf("off-box backup produced no snapshots: %d app(s) toggled but no recovery unit was found on any connected drive (missing: %s)",
+			len(apps), strings.Join(missing, ", "))
+	}
 
 	dur := time.Since(start)
 	snapshots := 0
@@ -241,64 +248,153 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
 		if runErr != nil {
 			o.LastStatus = "error"
 			o.LastError = runErr.Error()
+			o.LastWarning = ""
 		} else {
 			o.LastStatus = "ok"
 			o.LastError = ""
 			o.SnapshotCount = snapshots
+			if len(missing) == 0 {
+				o.LastWarning = ""
+			} else {
+				o.LastWarning = fmt.Sprintf("Figyelmeztetés: %d alkalmazásnak nincs elérhető mentése, ezek kimaradtak: %s",
+					len(missing), strings.Join(missing, ", "))
+			}
 		}
 	})
 	if m.offboxNotify != nil {
 		m.offboxNotify(dur, snapshots, runErr)
 	}
-	if runErr != nil {
+	switch {
+	case runErr != nil:
 		m.logger.Printf("[ERROR] [offbox] backup failed after %s: %v", dur.Round(time.Second), runErr)
-	} else {
-		m.logger.Printf("[INFO] [offbox] backup OK: %d app(s), %d snapshot(s), %s", len(apps), snapshots, dur.Round(time.Second))
+	case len(missing) > 0:
+		m.logger.Printf("[INFO] [offbox] backup OK: %d app(s) backed up, %d skipped (no unit), %d snapshot(s), %s",
+			backedUp, len(missing), snapshots, dur.Round(time.Second))
+	default:
+		m.logger.Printf("[INFO] [offbox] backup OK: %d app(s) backed up, %d snapshot(s), %s", backedUp, snapshots, dur.Round(time.Second))
 	}
 	return runErr
 }
 
-// runOffboxInternal does the repo-ensure + per-app backup + prune. Caller holds the running flag.
-func (m *Manager) runOffboxInternal(ctx context.Context, apps []string, base, env []string) error {
-	if err := m.ensureOffboxRepo(ctx, base, env); err != nil {
-		return err // fail fast (dead NAS surfaces here)
+// offboxCandidateNSRoots is the durable, deployment-state-INDEPENDENT set of felhom-data namespace roots
+// to search for a recovery unit: every registered SCHEDULABLE (non-decommissioned) storage path ∪ the
+// system-data fallback drive, deduped by resolved nsRoot string. This deliberately does NOT consult
+// GetAppDrivePath/AppNamespaceRoot — those read the app's LIVE app.yaml HDD_PATH and silently fall back
+// to systemDataPath when the app isn't currently deployed, which made offbox look on the wrong drive
+// (DIAG root cause). A disconnected drive's path simply isn't present on disk → os.Stat fails → the unit
+// is "not here" (correct: a disconnected drive can't be offsited). Boundary: a decommissioned or
+// non-schedulable drive is not searched (not an active managed backup location).
+func (m *Manager) offboxCandidateNSRoots() []string {
+	seen := map[string]bool{}
+	var nsRoots []string
+	add := func(nr string) {
+		if nr != "" && !seen[nr] {
+			seen[nr] = true
+			nsRoots = append(nsRoots, nr)
+		}
+	}
+	for _, sp := range m.settings.GetSchedulableStoragePaths() {
+		add(m.namespaceRoot(sp.Path))
+	}
+	if m.systemDataPath != "" {
+		add(m.namespaceRoot(m.systemDataPath))
+	}
+	return nsRoots
+}
+
+// discoverOffboxUnit locates an app's recovery unit (backups/primary/<app>) across the candidate nsRoots.
+// Returns the src path + true when exactly one exists; when the SAME app's unit exists on more than one
+// drive (drive churn / a stale copy left behind), it returns the NEWEST by manifest CreatedAt (falling
+// back to the unit dir mtime) and WARNs about the others. Independent of the app's live deploy state.
+func (m *Manager) discoverOffboxUnit(app string) (string, bool) {
+	var foundSrc, foundManifest []string
+	for _, nr := range m.offboxCandidateNSRoots() {
+		p := RecoveryUnitPath(nr, app)
+		if fi, err := os.Stat(p); err == nil && fi.IsDir() {
+			foundSrc = append(foundSrc, p)
+			foundManifest = append(foundManifest, RecoveryUnitManifestPath(nr, app))
+		}
+	}
+	switch len(foundSrc) {
+	case 0:
+		return "", false
+	case 1:
+		return foundSrc[0], true
+	default:
+		best := 0
+		bestT := offboxUnitTime(foundSrc[0], foundManifest[0])
+		for i := 1; i < len(foundSrc); i++ {
+			if t := offboxUnitTime(foundSrc[i], foundManifest[i]); t.After(bestT) {
+				best, bestT = i, t
+			}
+		}
+		var others []string
+		for i, p := range foundSrc {
+			if i != best {
+				others = append(others, p)
+			}
+		}
+		m.logger.Printf("[WARN] [offbox] %s: multiple recovery units found, using newest (%s); ignoring: %s",
+			app, foundSrc[best], strings.Join(others, ", "))
+		return foundSrc[best], true
+	}
+}
+
+// offboxUnitTime returns a recovery unit's timestamp for the multi-copy tiebreak: the manifest's
+// CreatedAt (RFC3339) if readable, else the unit dir's mtime (zero if neither is available).
+func offboxUnitTime(src, manifestPath string) time.Time {
+	if mf := readManifest(manifestPath); mf != nil {
+		if t, err := time.Parse(time.RFC3339, mf.CreatedAt); err == nil {
+			return t
+		}
+	}
+	if fi, err := os.Stat(src); err == nil {
+		return fi.ModTime()
+	}
+	return time.Time{}
+}
+
+// runOffboxInternal does the repo-ensure + per-app DISCOVER-then-backup + prune. Caller holds the running
+// flag. Returns how many apps were actually backed up, which toggled apps had no discoverable unit
+// (skipped), and the first hard error (repo-ensure or a restic backup exec failure).
+func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []string) (backedUp int, missing []string, err error) {
+	if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
+		return 0, nil, rerr // fail fast (dead NAS surfaces here)
 	}
 	var firstErr error
 	for _, stack := range apps {
-		nsRoot := m.AppNamespaceRoot(stack)
-		if nsRoot == "" {
-			continue
-		}
-		src := RecoveryUnitPath(nsRoot, stack) // backups/primary/<stack> = recovery unit + db-dumps + vol-tars
-		if _, err := os.Stat(src); err != nil {
-			m.logger.Printf("[INFO] [offbox] %s: no backup data yet (%s) — skipping", stack, src)
+		src, ok := m.discoverOffboxUnit(stack)
+		if !ok {
+			m.logger.Printf("[WARN] [offbox] %s: no recovery unit found on any connected drive — skipping", stack)
+			missing = append(missing, stack)
 			continue
 		}
 		bctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
 		args := append(append([]string{}, base...), "backup", "--tag", "felhom-offbox", "--tag", stack, src)
-		out, err := m.runner()(bctx, env, args...)
+		out, berr := m.runner()(bctx, env, args...)
 		cancel()
-		if err != nil {
-			m.logger.Printf("[ERROR] [offbox] backup %s failed: %v: %s", stack, err, truncate(out))
+		if berr != nil {
+			m.logger.Printf("[ERROR] [offbox] backup %s failed: %v: %s", stack, berr, truncate(out))
 			if firstErr == nil {
-				firstErr = fmt.Errorf("offbox backup %s: %w", stack, err)
+				firstErr = fmt.Errorf("offbox backup %s: %w", stack, berr)
 			}
 			continue
 		}
-		m.logger.Printf("[INFO] [offbox] backed up %s", stack)
+		backedUp++
+		m.logger.Printf("[INFO] [offbox] backed up %s (%s)", stack, src)
 	}
 	if firstErr != nil {
-		return firstErr
+		return backedUp, missing, firstErr
 	}
 	// Retention: keep a sane window, prune the rest. Repo-wide (grouped by host+paths by default).
 	fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
 	defer cancel()
-	args := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
-	if out, err := m.runner()(fctx, env, args...); err != nil {
+	fargs := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
+	if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
 		// A prune failure is non-fatal to the backup itself (data is safe) — log, don't fail the run.
-		m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", err, truncate(out))
+		m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", ferr, truncate(out))
 	}
-	return nil
+	return backedUp, missing, nil
 }
 
 // offboxRecordStats reads the snapshot count (best-effort) for the UI; also fills repo size when stats works.

controller/internal/backup/offbox_test.go

@@ -2,6 +2,8 @@ package backup
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"log"
 	"os"
 	"path/filepath"
@@ -290,6 +292,212 @@ func TestOffbox_ValidateRejectsInjection(t *testing.T) {
 	}
 }
 
+// --- Off-box unit DISCOVERY + no-silent-success (§7 A–E) ---
+
+// recordingOffboxRunner captures the exact `src` path of each restic `backup` call and returns success
+// for the repo probe/init/snapshots/stats/forget. Per-stack `backup` errors are configurable.
+type recordingOffboxRunner struct {
+	backupSrc []string         // src path per backup call, in order (the load-bearing effect to assert)
+	backupErr map[string]error // stack (last --tag) → error to return from `backup`
+}
+
+func (rr *recordingOffboxRunner) run(_ context.Context, _ []string, args ...string) ([]byte, error) {
+	switch {
+	case contains(args, "cat") && contains(args, "config"):
+		return []byte(`{"version":2}`), nil // repo exists (no init)
+	case contains(args, "backup"):
+		rr.backupSrc = append(rr.backupSrc, args[len(args)-1]) // last arg = src path
+		if e := rr.backupErr[tagOf(args)]; e != nil {
+			return []byte("restic backup failed"), e
+		}
+		return nil, nil
+	case contains(args, "forget"):
+		return nil, nil
+	case contains(args, "snapshots"):
+		return []byte(`[{"id":"s1"}]`), nil
+	case contains(args, "stats"):
+		return []byte(`{"total_size":123}`), nil
+	}
+	return nil, nil
+}
+
+// addSchedulablePath registers a schedulable (non-decommissioned) storage path — a candidate drive.
+func addSchedulablePath(t *testing.T, sett *settings.Settings, p string) {
+	t.Helper()
+	if err := sett.AddStoragePath(settings.StoragePath{Path: p, Schedulable: true, AddedAt: "2026-07-01T00:00:00Z"}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// writeUnit lays a recovery unit (dir + a manifest with the given CreatedAt) at <nsRoot>/backups/primary/<app>.
+func writeUnit(t *testing.T, nsRoot, app, createdAt string) {
+	t.Helper()
+	if err := os.MkdirAll(RecoveryUnitPath(nsRoot, app), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	b, _ := json.Marshal(RecoveryManifest{AppName: app, CreatedAt: createdAt})
+	if err := os.WriteFile(RecoveryUnitManifestPath(nsRoot, app), b, 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// A — undeployed-but-toggled app whose unit lives on a REGISTERED drive (not systemDataPath): discovery
+// must find it there. The harness has no stackProvider, so the OLD AppNamespaceRoot path would resolve to
+// systemDataPath (where no unit is) — the F1 bug. See the companion red-proof in REPORT.
+func TestOffbox_DiscoversUnitOnRegisteredDrive(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	usb := t.TempDir()
+	addSchedulablePath(t, sett, usb)
+	usbNS := m.namespaceRoot(usb) // registered drive → in-guest → nsRoot == usb
+	writeUnit(t, usbNS, "audiobookshelf", "2026-07-01T00:00:00Z")
+	// prove the OLD resolution would have looked elsewhere (no unit at systemDataPath nsRoot)
+	if _, err := os.Stat(RecoveryUnitPath(m.namespaceRoot(m.systemDataPath), "audiobookshelf")); err == nil {
+		t.Fatal("setup: unit must NOT exist on systemDataPath for this repro")
+	}
+	_ = sett.SetAppOffbox("audiobookshelf", true)
+
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+	wantSrc := RecoveryUnitPath(usbNS, "audiobookshelf")
+	if len(rr.backupSrc) != 1 || rr.backupSrc[0] != wantSrc {
+		t.Fatalf("backup src = %v, want exactly [%s] (discovered on the registered drive)", rr.backupSrc, wantSrc)
+	}
+	if st := sett.GetOffboxTarget(); st.LastStatus != "ok" || st.LastWarning != "" {
+		t.Fatalf("status = %+v, want ok / no warning", st)
+	}
+}
+
+// B — a toggled app with NO recovery unit anywhere: the run must be a HARD ERROR (no silent ok/0), alert
+// the operator, and record status=error naming the app. Companion red-proof in REPORT.
+func TestOffbox_NoUnitAnywhereIsHardError(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	_ = sett.SetAppOffbox("ghost", true) // toggled; no unit created anywhere
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	var notified bool
+	var notifiedErr error
+	m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { notified = true; notifiedErr = err })
+
+	err := m.RunOffboxBackup(context.Background())
+	if err == nil {
+		t.Fatal("0-of-N toggled apps backed up must ERROR (no silent success)")
+	}
+	if len(rr.backupSrc) != 0 {
+		t.Fatalf("no backup should have run, got %v", rr.backupSrc)
+	}
+	if !notified || notifiedErr == nil {
+		t.Fatal("the 0/N run must alert the operator with a non-nil err")
+	}
+	st := sett.GetOffboxTarget()
+	if st.LastStatus != "error" || st.LastError == "" {
+		t.Fatalf("status must be error, got %+v", st)
+	}
+	if !strings.Contains(st.LastError, "ghost") {
+		t.Fatalf("LastError should name the missing app, got %q", st.LastError)
+	}
+}
+
+// C — partial: one toggled app has a unit, another doesn't → the present one is backed up, status stays
+// ok, notify err is nil, but LastWarning (Hungarian) names the missing app.
+func TestOffbox_PartialRunWarnsNotErrors(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	usb := t.TempDir()
+	addSchedulablePath(t, sett, usb)
+	usbNS := m.namespaceRoot(usb)
+	writeUnit(t, usbNS, "present", "2026-07-01T00:00:00Z")
+	_ = sett.SetAppOffbox("present", true)
+	_ = sett.SetAppOffbox("gone", true) // no unit
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	notifiedErr := errors.New("sentinel") // must be cleared to nil by a non-erroring run
+	m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { notifiedErr = err })
+
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("a partial run must NOT error, got %v", err)
+	}
+	if len(rr.backupSrc) != 1 || rr.backupSrc[0] != RecoveryUnitPath(usbNS, "present") {
+		t.Fatalf("only 'present' should be backed up, got %v", rr.backupSrc)
+	}
+	if notifiedErr != nil {
+		t.Fatalf("partial run must notify with nil err, got %v", notifiedErr)
+	}
+	st := sett.GetOffboxTarget()
+	if st.LastStatus != "ok" {
+		t.Fatalf("status = %q, want ok", st.LastStatus)
+	}
+	if !strings.Contains(st.LastWarning, "gone") {
+		t.Fatalf("LastWarning must name the missing app 'gone', got %q", st.LastWarning)
+	}
+}
+
+// D — the same app's unit on TWO registered drives (drive churn): exactly ONE backup call, for the NEWER
+// unit (by manifest CreatedAt).
+func TestOffbox_MultipleUnitsPicksNewest(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	older, newer := t.TempDir(), t.TempDir()
+	addSchedulablePath(t, sett, older)
+	addSchedulablePath(t, sett, newer)
+	olderNS, newerNS := m.namespaceRoot(older), m.namespaceRoot(newer)
+	writeUnit(t, olderNS, "z", "2026-01-01T00:00:00Z")
+	writeUnit(t, newerNS, "z", "2026-07-01T00:00:00Z")
+	_ = sett.SetAppOffbox("z", true)
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+	wantSrc := RecoveryUnitPath(newerNS, "z")
+	if len(rr.backupSrc) != 1 || rr.backupSrc[0] != wantSrc {
+		t.Fatalf("must back up exactly the NEWER unit once; got %v want [%s]", rr.backupSrc, wantSrc)
+	}
+}
+
+// E — every toggled app present: all backed up, status ok, no warning, snapshot count reflects stats.
+func TestOffbox_AllPresentHappyPath(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	usb := t.TempDir()
+	addSchedulablePath(t, sett, usb)
+	usbNS := m.namespaceRoot(usb)
+	writeUnit(t, usbNS, "a", "2026-07-01T00:00:00Z")
+	writeUnit(t, usbNS, "b", "2026-07-01T00:00:00Z")
+	_ = sett.SetAppOffbox("a", true)
+	_ = sett.SetAppOffbox("b", true)
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+	if len(rr.backupSrc) != 2 {
+		t.Fatalf("both apps must be backed up, got %v", rr.backupSrc)
+	}
+	st := sett.GetOffboxTarget()
+	if st.LastStatus != "ok" || st.LastWarning != "" {
+		t.Fatalf("happy path wants ok + no warning, got %+v", st)
+	}
+	if st.SnapshotCount != 1 {
+		t.Fatalf("snapshot count should reflect the stats fake (1), got %d", st.SnapshotCount)
+	}
+}
+
+// Edge — zero toggled apps is a clean no-op ok (no error, no backup call).
+func TestOffbox_NoAppsToggledIsCleanOK(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("zero toggled apps must be a clean no-op, got %v", err)
+	}
+	if len(rr.backupSrc) != 0 {
+		t.Fatalf("no backup should run with 0 toggled apps, got %v", rr.backupSrc)
+	}
+	if st := sett.GetOffboxTarget(); st.LastStatus != "ok" || st.LastError != "" {
+		t.Fatalf("status = %+v, want ok / no error", st)
+	}
+}
+
 func runtimeIsUnix() bool { return os.PathSeparator == '/' }
 
 func contains(ss []string, want string) bool {

controller/internal/settings/settings.go

@@ -126,6 +126,9 @@ type OffboxTarget struct {
 	LastDuration  string `json:"last_duration,omitempty"`
 	RepoSizeHuman string `json:"repo_size_human,omitempty"`
 	SnapshotCount int    `json:"snapshot_count,omitempty"`
+	// LastWarning is a customer-visible notice set on an otherwise-OK run when SOME toggled apps had
+	// no discoverable recovery unit (partial run). Empty on a fully-successful or failed run.
+	LastWarning string `json:"last_warning,omitempty"`
 }
 
 // CrossDriveBackup configures per-app backup to a secondary drive.

controller/internal/web/offbox_handlers.go

@@ -80,6 +80,7 @@ func (s *Server) offboxConfigHandler(w http.ResponseWriter, r *http.Request) {
 	if prev != nil { // preserve runtime status fields across an edit
 		tgt.LastRun, tgt.LastStatus, tgt.LastError = prev.LastRun, prev.LastStatus, prev.LastError
 		tgt.LastDuration, tgt.RepoSizeHuman, tgt.SnapshotCount = prev.LastDuration, prev.RepoSizeHuman, prev.SnapshotCount
+		tgt.LastWarning = prev.LastWarning
 	}
 	if err := s.settings.SetOffboxTarget(tgt); err != nil {
 		offboxRedirect(w, r, "A beállítás mentése sikertelen.", true)

controller/internal/web/templates/backups.html

@@ -141,6 +141,7 @@
         </div>
     </div>
     {{if .Offbox.LastError}}<p class="form-hint" style="color:var(--crit)">Utolsó hiba: {{.Offbox.LastError}}</p>{{end}}
+    {{if .Offbox.LastWarning}}<p class="form-hint" style="color:var(--warn)">{{.Offbox.LastWarning}}</p>{{end}}
     {{if .OffboxConfigured}}
     <div class="schedule-actions" style="margin-top:1rem">
         <form method="POST" action="/backup/offbox/run" style="display:inline">{{.CSRFField}}