v0.142.0: offsite repo continuity — orphaned-repo guard (A) + run-status auto-refresh (C)

- Part A: classify restic cat-config failure (wrong-password=orphaned vs no-repo vs other); ORPHANED state + Hungarian card + offbox_repo_orphaned/reset events (once, not nightly); reset = move-aside (never delete) + init, unclaimed auto / claimed confirm. Red-proofs TestOffbox_OrphanDetection_* + ConfirmedReset. - Part C: GET /backup/offbox/status + poll on backups_remote → flips Fut→Rendben/Hiba without manual reload.
2026-07-17 10:47:30 +02:00
parent 1452dd2b17
commit 596505ed64
11 changed files with 587 additions and 18 deletions
@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -53,6 +54,162 @@ func (m *Manager) SetOffboxNotify(fn func(dur time.Duration, snapshots int, err
 	m.offboxNotify = fn
 }

+// SetOffboxOrphanEvent wires the offsite-repo continuity event push (main.go → notifier).
+func (m *Manager) SetOffboxOrphanEvent(fn func(eventType, renamedTo string)) { m.offboxOrphanEvent = fn }
+
+// SetOffboxSSH overrides the raw-ssh exec used for the orphaned-repo move-aside (tests).
+func (m *Manager) SetOffboxSSH(fn func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error)) {
+	m.offboxSSH = fn
+}
+
+// ErrOffboxOrphaned is the sentinel returned when the offsite repo exists but is keyed under a
+// passphrase this controller no longer has (the reinstall shape) — the run skips and the UI shows
+// the orphan card instead of the raw restic error.
+var ErrOffboxOrphaned = fmt.Errorf("offbox repo orphaned: exists but keyed under a previous, no-longer-available passphrase")
+
+// classifyResticProbe maps a `restic cat config` failure to a repo class. The signatures are the exact
+// restic stderr matched in the 2026-07-17 diagnosis + restic's no-repo message:
+//   - "orphaned": repo present, wrong key ("wrong password or no key found") — the definitive signal
+//   - "norepo":   no repo at the location (init is the correct path)
+//   - "other":    network/SFTP-auth/unknown — NOT orphaned; existing error handling
+func classifyResticProbe(out []byte, err error) string {
+	if err == nil {
+		return "" // success — repo good
+	}
+	s := strings.ToLower(string(out))
+	switch {
+	case strings.Contains(s, "wrong password or no key found"):
+		return "orphaned"
+	case strings.Contains(s, "unable to open config file"),
+		strings.Contains(s, "is there a repository at the following location"),
+		strings.Contains(s, "no such file"),
+		strings.Contains(s, "does not exist"):
+		return "norepo"
+	default:
+		return "other"
+	}
+}
+
+func defaultOffboxSSH(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error) {
+	if port == 0 {
+		port = 22
+	}
+	args := []string{
+		"-p", fmt.Sprint(port), "-oBatchMode=yes", fmt.Sprintf("-oConnectTimeout=%d", offboxConnectTimeoutSec),
+		"-oStrictHostKeyChecking=yes", "-oUserKnownHostsFile=" + knownHosts, "-i", keyPath,
+		user + "@" + host, remoteCmd,
+	}
+	cmd := exec.CommandContext(ctx, "ssh", args...)
+	return cmd.CombinedOutput()
+}
+
+func (m *Manager) sshRunner() func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error) {
+	if m.offboxSSH != nil {
+		return m.offboxSSH
+	}
+	return defaultOffboxSSH
+}
+
+// OffboxOrphaned reports whether the offsite repo is in the ORPHANED state (persisted).
+func (m *Manager) OffboxOrphaned() bool {
+	t := m.settings.GetOffboxTarget()
+	return t != nil && t.RepoState == "orphaned"
+}
+
+// OffboxOrphanedRenamedTo returns the last move-aside path (for the card copy; "" if none).
+func (m *Manager) OffboxOrphanedRenamedTo() string {
+	t := m.settings.GetOffboxTarget()
+	if t == nil {
+		return ""
+	}
+	return t.OrphanedRenamedTo
+}
+
+// markOrphaned sets the persistent ORPHANED state and, ONLY on the transition into it (not already
+// orphaned), pushes the offbox_repo_orphaned event — so scheduled runs never nightly-spam.
+func (m *Manager) markOrphaned() {
+	already := m.OffboxOrphaned()
+	if err := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
+		o.RepoState = "orphaned"
+		if o.OrphanedAt == "" || !already {
+			o.OrphanedAt = time.Now().UTC().Format(time.RFC3339)
+		}
+	}); err != nil {
+		m.logger.Printf("[WARN] [offbox] persist orphaned state failed: %v", err)
+	}
+	if !already {
+		m.logger.Printf("[WARN] [offbox] offsite repo ORPHANED — remote holds backups written under a previous, no-longer-available key; runs will skip until reset")
+		if m.offboxOrphanEvent != nil {
+			m.offboxOrphanEvent("offbox_repo_orphaned", "")
+		}
+	}
+}
+
+// resetOrphanedRepo moves the orphaned repo aside (never deletes) and re-inits a fresh repo under the
+// CURRENT passphrase. Reversible. Used by the unclaimed auto-reset (Scenario B) and the claimed
+// confirmed reset (Scenario C). Caller holds the single-flight guarantee (run mutex) OR is the handler.
+func (m *Manager) resetOrphanedRepo(ctx context.Context, base, env []string, reason string) error {
+	t := m.settings.GetOffboxTarget()
+	if t == nil {
+		return fmt.Errorf("no offsite target configured")
+	}
+	port := t.Port
+	if port == 0 {
+		port = 22
+	}
+	// Choose a move-aside name that never overwrites an earlier orphaned copy (edge rule: -2, -3).
+	date := time.Now().UTC().Format("20060102")
+	base1 := t.RepoPath + ".orphaned-" + date
+	newPath := base1
+	for i := 2; i <= 20; i++ {
+		// `test -e <p>` returns non-zero (exit 1) when absent — that is the name we want. A transport
+		// error also lands here; we then just try the mv and let it fail loudly rather than loop.
+		out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(), "test -e "+shellQuote(newPath))
+		if err != nil && !strings.Contains(strings.ToLower(string(out)), "denied") {
+			break // absent (test -e exit 1) → free name
+		}
+		newPath = fmt.Sprintf("%s-%d", base1, i)
+	}
+	m.logger.Printf("[WARN] [offbox] resetting orphaned repo (%s): move-aside %s -> %s, then re-init", reason, t.RepoPath, newPath)
+	if out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(),
+		fmt.Sprintf("mv %s %s", shellQuote(t.RepoPath), shellQuote(newPath))); err != nil {
+		return fmt.Errorf("offbox move-aside failed: %w: %s", err, truncate(out))
+	}
+	// Fresh init under the current passphrase.
+	ictx, icancel := context.WithTimeout(ctx, offboxProbeTimeout)
+	defer icancel()
+	if out, err := m.runner()(ictx, env, append(append([]string{}, base...), "init")...); err != nil {
+		return fmt.Errorf("offbox re-init after move-aside failed: %w: %s", err, truncate(out))
+	}
+	if err := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
+		o.RepoState = ""
+		o.OrphanedAt = ""
+		o.OrphanedRenamedTo = newPath
+		o.LastError = ""
+	}); err != nil {
+		m.logger.Printf("[WARN] [offbox] clear orphaned state failed: %v", err)
+	}
+	m.logger.Printf("[INFO] [offbox] orphaned repo reset complete — old history set aside at %s (move-aside, not deleted); fresh repo initialized", newPath)
+	if m.offboxOrphanEvent != nil {
+		m.offboxOrphanEvent("offbox_repo_reset", newPath)
+	}
+	return nil
+}
+
+// ResetOrphanedRepo is the handler entry point for the CLAIMED confirmed reset (Scenario C). It refuses
+// unless the repo is currently orphaned. It builds the base/env and runs the move-aside + re-init.
+func (m *Manager) ResetOrphanedRepo(ctx context.Context) error {
+	if !m.OffboxOrphaned() {
+		return fmt.Errorf("az offsite tároló nincs elárvult állapotban")
+	}
+	t := m.settings.GetOffboxTarget()
+	base, env := m.offboxBaseArgs(t)
+	return m.resetOrphanedRepo(ctx, base, env, "operator-confirmed (claimed)")
+}
+
+// shellQuote single-quotes a path for the remote shell (our repo paths have no single quotes).
+func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }
+
 // SetOffboxSizer overrides the mandatory-set byte estimator (tests). SetOffboxEnlargeBlockedNotifier
 // wires the edge-triggered enlargement-blocked notification (main.go). SetOffboxPlaceCopier overrides
 // the place-to-live missing-only merge (tests).
@@ -368,23 +525,47 @@ func (m *Manager) resticStep(ctx context.Context, env, base []string, label stri
 func (m *Manager) ensureOffboxRepo(ctx context.Context, base, env []string) error {
 	pctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
 	defer cancel()
-	if _, err := m.runner()(pctx, env, append(append([]string{}, base...), "cat", "config")...); err == nil {
-		return nil // repo exists
-	}
-	// Repo (probably) absent OR unreachable. Try init; if init succeeds the repo was absent. If init
-	// fails because it already exists (a race), treat as success; otherwise the error is real (e.g. a
-	// dead NAS — fail fast).
-	ictx, icancel := context.WithTimeout(ctx, offboxProbeTimeout)
-	defer icancel()
-	out, err := m.runner()(ictx, env, append(append([]string{}, base...), "init")...)
-	if err == nil {
-		m.logger.Printf("[INFO] [offbox] initialized restic repo")
+	pout, perr := m.runner()(pctx, env, append(append([]string{}, base...), "cat", "config")...)
+	switch classifyResticProbe(pout, perr) {
+	case "": // success — repo good (and clear any stale orphaned flag)
+		if m.OffboxOrphaned() {
+			_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.RepoState = ""; o.OrphanedAt = "" })
+		}
 		return nil
+	case "orphaned":
+		// The repo EXISTS but is keyed under a passphrase we no longer have (the reinstall shape). An
+		// UNCLAIMED (as-delivered) box auto-resets (Scenario B); a CLAIMED box surfaces the orphan card
+		// and skips until the customer confirms a reset (Scenario C). Move-aside, never delete.
+		if !m.settings.GetClaimed() {
+			m.logger.Printf("[INFO] [offbox] orphaned repo on an UNCLAIMED box — auto-resetting (move-aside + re-init)")
+			if m.offboxOrphanEvent != nil {
+				m.offboxOrphanEvent("offbox_repo_orphaned", "")
+			}
+			if rerr := m.resetOrphanedRepo(ctx, base, env, "auto (unclaimed)"); rerr != nil {
+				m.markOrphaned() // auto-reset failed → fall back to the orphan card so it isn't silent
+				return ErrOffboxOrphaned
+			}
+			return nil // repo is fresh under the current passphrase → the run proceeds
+		}
+		m.markOrphaned()
+		return ErrOffboxOrphaned
+	case "norepo":
+		// No repo at the location → init (the normal first-run path). A race where it already exists is
+		// treated as success; any other init error is real (e.g. a dead NAS — fail fast).
+		ictx, icancel := context.WithTimeout(ctx, offboxProbeTimeout)
+		defer icancel()
+		out, err := m.runner()(ictx, env, append(append([]string{}, base...), "init")...)
+		if err == nil {
+			m.logger.Printf("[INFO] [offbox] initialized restic repo")
+			return nil
+		}
+		if strings.Contains(string(out), "already initialized") || strings.Contains(string(out), "already exists") {
+			return nil
+		}
+		return fmt.Errorf("offbox repo unreachable / init failed: %w: %s", err, truncate(out))
+	default: // "other" — network/SFTP-auth/unknown; NOT orphaned. Surface as before (fail fast).
+		return fmt.Errorf("offbox repo unreachable: %w: %s", perr, truncate(pout))
 	}
-	if strings.Contains(string(out), "already initialized") || strings.Contains(string(out), "already exists") {
-		return nil
-	}
-	return fmt.Errorf("offbox repo unreachable / init failed: %w: %s", err, truncate(out))
 }

 // RunOffboxBackup backs up every off-box-toggled app's recovery unit (recovery unit + DB dumps + volume
@@ -400,6 +581,12 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
 		m.logger.Printf("[INFO] [offbox] skipped — pending key escrow (no offsite run until the repo password is escrowed under R)")
 		return nil
 	}
+	// Offsite-repo continuity (v0.142.0): once ORPHANED, scheduled runs SKIP (the event fired on the
+	// detection transition — no nightly spam) until a reset clears it. The remote page shows the card.
+	if m.OffboxOrphaned() {
+		m.logger.Printf("[INFO] [offbox] skipped — offsite repo orphaned (awaiting reset)")
+		return nil
+	}
 	if m.migrationActive() {
 		m.logger.Printf("[INFO] [offbox] skipped — migration in progress")
 		return nil
@@ -467,7 +654,13 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
 	if perr := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
 		o.LastRun = time.Now().UTC().Format(time.RFC3339)
 		o.LastDuration = dur.Round(time.Second).String()
-		if runErr != nil {
+		if errors.Is(runErr, ErrOffboxOrphaned) {
+			// First-detection of the orphaned repo: RepoState (set by markOrphaned) drives the orphan
+			// card — do NOT surface the raw restic/sentinel text as the last-error banner.
+			o.LastStatus = "error"
+			o.LastError = ""
+			o.LastWarning = ""
+		} else if runErr != nil {
 			o.LastStatus = "error"
 			o.LastError = runErr.Error()
 			o.LastWarning = ""
@@ -503,7 +696,9 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
 	}); perr != nil {
 		m.logger.Printf("[WARN] [offbox] status persist (final) failed: %v", perr)
 	}
-	if m.offboxNotify != nil {
+	// The orphaned case has its OWN dedicated event (offbox_repo_orphaned) — do NOT also fire the
+	// generic backup-failed notification (no double/raw alert; the orphan card is the customer surface).
+	if m.offboxNotify != nil && !errors.Is(runErr, ErrOffboxOrphaned) {
 		m.offboxNotify(dur, snapshots, runErr)
 	}
 	// Edge-triggered enlarge-blocked notification: only apps that NEWLY crossed into the blocked state
@@ -518,6 +713,9 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
 		}
 	}
 	switch {
+	case errors.Is(runErr, ErrOffboxOrphaned):
+		m.logger.Printf("[WARN] [offbox] run skipped — offsite repo orphaned (card shown; awaiting reset)")
+		return nil // the orphaned STATE + event are the signal; not a hard run error for the scheduler
 	case runErr != nil:
 		m.logger.Printf("[ERROR] [offbox] backup failed after %s: %v", dur.Round(time.Second), runErr)
 	case len(missing) > 0: