v0.104.0: off-box unit discovery (durable, deployment-independent) + no-silent-success

offbox located each toggled app's recovery unit via AppNamespaceRoot→GetAppDrivePath, which reads the app's LIVE app.yaml HDD_PATH and silently falls back to systemDataPath when the app isn't deployed → looked on the wrong drive, backed up nothing, reported ok/0 (DIAG root cause). Now: - discoverOffboxUnit/offboxCandidateNSRoots scan the durable storage registry (schedulable non-decommissioned paths ∪ systemDataPath) for backups/primary/<app>, independent of deploy state; newest-by-manifest-CreatedAt wins on drive churn. - RunOffboxBackup: runOffboxInternal returns (backedUp, missing, err); 0-of-N toggled → hard error + operator alert; partial → ok + new OffboxTarget.LastWarning (shown on /backups, preserved across config edit). - AppNamespaceRoot + primary WRITE paths unchanged. - 6 non-hollow tests (A-E + edge) + both companion red-proofs run (reverted). - NOT yet live-validated against the Storage Box (spike creds torn down). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-09 12:32:05 +02:00
parent 59eb3bea76
commit 908e4b906a
8 changed files with 390 additions and 26 deletions
@@ -228,7 +228,14 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
 	start := time.Now()
 	_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.LastStatus = "running"; o.LastError = "" })

-	runErr := m.runOffboxInternal(ctx, apps, base, env)
+	backedUp, missing, runErr := m.runOffboxInternal(ctx, apps, base, env)
+
+	// No-silent-success: apps were toggled but NOTHING was captured (every unit missing) → promote to a
+	// hard error so the run reports "error" and the operator is alerted, instead of a misleading ok/0.
+	if runErr == nil && len(apps) > 0 && backedUp == 0 {
+		runErr = fmt.Errorf("off-box backup produced no snapshots: %d app(s) toggled but no recovery unit was found on any connected drive (missing: %s)",
+			len(apps), strings.Join(missing, ", "))
+	}

 	dur := time.Since(start)
 	snapshots := 0
@@ -241,64 +248,153 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
 		if runErr != nil {
 			o.LastStatus = "error"
 			o.LastError = runErr.Error()
+			o.LastWarning = ""
 		} else {
 			o.LastStatus = "ok"
 			o.LastError = ""
 			o.SnapshotCount = snapshots
+			if len(missing) == 0 {
+				o.LastWarning = ""
+			} else {
+				o.LastWarning = fmt.Sprintf("Figyelmeztetés: %d alkalmazásnak nincs elérhető mentése, ezek kimaradtak: %s",
+					len(missing), strings.Join(missing, ", "))
+			}
 		}
 	})
 	if m.offboxNotify != nil {
 		m.offboxNotify(dur, snapshots, runErr)
 	}
-	if runErr != nil {
+	switch {
+	case runErr != nil:
 		m.logger.Printf("[ERROR] [offbox] backup failed after %s: %v", dur.Round(time.Second), runErr)
-	} else {
-		m.logger.Printf("[INFO] [offbox] backup OK: %d app(s), %d snapshot(s), %s", len(apps), snapshots, dur.Round(time.Second))
+	case len(missing) > 0:
+		m.logger.Printf("[INFO] [offbox] backup OK: %d app(s) backed up, %d skipped (no unit), %d snapshot(s), %s",
+			backedUp, len(missing), snapshots, dur.Round(time.Second))
+	default:
+		m.logger.Printf("[INFO] [offbox] backup OK: %d app(s) backed up, %d snapshot(s), %s", backedUp, snapshots, dur.Round(time.Second))
 	}
 	return runErr
 }

-// runOffboxInternal does the repo-ensure + per-app backup + prune. Caller holds the running flag.
-func (m *Manager) runOffboxInternal(ctx context.Context, apps []string, base, env []string) error {
-	if err := m.ensureOffboxRepo(ctx, base, env); err != nil {
-		return err // fail fast (dead NAS surfaces here)
+// offboxCandidateNSRoots is the durable, deployment-state-INDEPENDENT set of felhom-data namespace roots
+// to search for a recovery unit: every registered SCHEDULABLE (non-decommissioned) storage path ∪ the
+// system-data fallback drive, deduped by resolved nsRoot string. This deliberately does NOT consult
+// GetAppDrivePath/AppNamespaceRoot — those read the app's LIVE app.yaml HDD_PATH and silently fall back
+// to systemDataPath when the app isn't currently deployed, which made offbox look on the wrong drive
+// (DIAG root cause). A disconnected drive's path simply isn't present on disk → os.Stat fails → the unit
+// is "not here" (correct: a disconnected drive can't be offsited). Boundary: a decommissioned or
+// non-schedulable drive is not searched (not an active managed backup location).
+func (m *Manager) offboxCandidateNSRoots() []string {
+	seen := map[string]bool{}
+	var nsRoots []string
+	add := func(nr string) {
+		if nr != "" && !seen[nr] {
+			seen[nr] = true
+			nsRoots = append(nsRoots, nr)
+		}
+	}
+	for _, sp := range m.settings.GetSchedulableStoragePaths() {
+		add(m.namespaceRoot(sp.Path))
+	}
+	if m.systemDataPath != "" {
+		add(m.namespaceRoot(m.systemDataPath))
+	}
+	return nsRoots
+}
+
+// discoverOffboxUnit locates an app's recovery unit (backups/primary/<app>) across the candidate nsRoots.
+// Returns the src path + true when exactly one exists; when the SAME app's unit exists on more than one
+// drive (drive churn / a stale copy left behind), it returns the NEWEST by manifest CreatedAt (falling
+// back to the unit dir mtime) and WARNs about the others. Independent of the app's live deploy state.
+func (m *Manager) discoverOffboxUnit(app string) (string, bool) {
+	var foundSrc, foundManifest []string
+	for _, nr := range m.offboxCandidateNSRoots() {
+		p := RecoveryUnitPath(nr, app)
+		if fi, err := os.Stat(p); err == nil && fi.IsDir() {
+			foundSrc = append(foundSrc, p)
+			foundManifest = append(foundManifest, RecoveryUnitManifestPath(nr, app))
+		}
+	}
+	switch len(foundSrc) {
+	case 0:
+		return "", false
+	case 1:
+		return foundSrc[0], true
+	default:
+		best := 0
+		bestT := offboxUnitTime(foundSrc[0], foundManifest[0])
+		for i := 1; i < len(foundSrc); i++ {
+			if t := offboxUnitTime(foundSrc[i], foundManifest[i]); t.After(bestT) {
+				best, bestT = i, t
+			}
+		}
+		var others []string
+		for i, p := range foundSrc {
+			if i != best {
+				others = append(others, p)
+			}
+		}
+		m.logger.Printf("[WARN] [offbox] %s: multiple recovery units found, using newest (%s); ignoring: %s",
+			app, foundSrc[best], strings.Join(others, ", "))
+		return foundSrc[best], true
+	}
+}
+
+// offboxUnitTime returns a recovery unit's timestamp for the multi-copy tiebreak: the manifest's
+// CreatedAt (RFC3339) if readable, else the unit dir's mtime (zero if neither is available).
+func offboxUnitTime(src, manifestPath string) time.Time {
+	if mf := readManifest(manifestPath); mf != nil {
+		if t, err := time.Parse(time.RFC3339, mf.CreatedAt); err == nil {
+			return t
+		}
+	}
+	if fi, err := os.Stat(src); err == nil {
+		return fi.ModTime()
+	}
+	return time.Time{}
+}
+
+// runOffboxInternal does the repo-ensure + per-app DISCOVER-then-backup + prune. Caller holds the running
+// flag. Returns how many apps were actually backed up, which toggled apps had no discoverable unit
+// (skipped), and the first hard error (repo-ensure or a restic backup exec failure).
+func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []string) (backedUp int, missing []string, err error) {
+	if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
+		return 0, nil, rerr // fail fast (dead NAS surfaces here)
 	}
 	var firstErr error
 	for _, stack := range apps {
-		nsRoot := m.AppNamespaceRoot(stack)
-		if nsRoot == "" {
-			continue
-		}
-		src := RecoveryUnitPath(nsRoot, stack) // backups/primary/<stack> = recovery unit + db-dumps + vol-tars
-		if _, err := os.Stat(src); err != nil {
-			m.logger.Printf("[INFO] [offbox] %s: no backup data yet (%s) — skipping", stack, src)
+		src, ok := m.discoverOffboxUnit(stack)
+		if !ok {
+			m.logger.Printf("[WARN] [offbox] %s: no recovery unit found on any connected drive — skipping", stack)
+			missing = append(missing, stack)
 			continue
 		}
 		bctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
 		args := append(append([]string{}, base...), "backup", "--tag", "felhom-offbox", "--tag", stack, src)
-		out, err := m.runner()(bctx, env, args...)
+		out, berr := m.runner()(bctx, env, args...)
 		cancel()
-		if err != nil {
-			m.logger.Printf("[ERROR] [offbox] backup %s failed: %v: %s", stack, err, truncate(out))
+		if berr != nil {
+			m.logger.Printf("[ERROR] [offbox] backup %s failed: %v: %s", stack, berr, truncate(out))
 			if firstErr == nil {
-				firstErr = fmt.Errorf("offbox backup %s: %w", stack, err)
+				firstErr = fmt.Errorf("offbox backup %s: %w", stack, berr)
 			}
 			continue
 		}
-		m.logger.Printf("[INFO] [offbox] backed up %s", stack)
+		backedUp++
+		m.logger.Printf("[INFO] [offbox] backed up %s (%s)", stack, src)
 	}
 	if firstErr != nil {
-		return firstErr
+		return backedUp, missing, firstErr
 	}
 	// Retention: keep a sane window, prune the rest. Repo-wide (grouped by host+paths by default).
 	fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
 	defer cancel()
-	args := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
-	if out, err := m.runner()(fctx, env, args...); err != nil {
+	fargs := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
+	if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
 		// A prune failure is non-fatal to the backup itself (data is safe) — log, don't fail the run.
-		m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", err, truncate(out))
+		m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", ferr, truncate(out))
 	}
-	return nil
+	return backedUp, missing, nil
 }

 // offboxRecordStats reads the snapshot count (best-effort) for the UI; also fills repo size when stats works.