v0.110.0: offbox stale-lock self-heal (campaign C2) + crash-truthful status (C1)

resticStep escalates a restic lock error to `unlock --remove-all` + one
retry (safe: single-writer repo — sub-account isolation + single-flight
mutex); plain `unlock` is stale-only and can't clear a crash lock across a
container-hostname change. Pre-run stale unlock hygiene on run+restore.
C1: NewManager flips a persisted LastStatus=running to a truthful error.
Both red-proofed (A reproduces the exact campaign backup failure).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

controller/internal/backup/offbox.go

@@ -304,6 +304,45 @@ func (m *Manager) offboxBaseArgs(t *settings.OffboxTarget) ([]string, []string)
 	return args, env
 }
 
+// offboxLockRe matches restic's "already locked" error (both the exclusive and shared forms).
+var offboxLockRe = regexp.MustCompile(`repository is already locked`)
+
+// unlockStale runs `restic unlock` (stale-only) — cheap pre-run hygiene that removes any lock restic can
+// itself prove dead/old. Non-fatal (logged at debug). Called before every offbox run/restore.
+func (m *Manager) unlockStale(ctx context.Context, base, env []string) {
+	uctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
+	defer cancel()
+	if out, err := m.runner()(uctx, env, append(append([]string{}, base...), "unlock")...); err != nil {
+		m.logger.Printf("[DEBUG] [offbox] pre-run unlock (stale-only) non-fatal: %v: %s", err, truncate(out))
+	}
+}
+
+// resticStep runs one restic step (backup/prune/restore) under the offbox single-flight guarantee and
+// self-heals the C2 crash lock. On a lock error it escalates to `unlock --remove-all` and retries ONCE,
+// because THIS controller is the repo's ONLY legitimate writer — per-customer sub-account isolation gives
+// one repo one writer, and the in-process single-flight mutex (held by every caller of this method) proves
+// no sibling operation is live. Plain `restic unlock` is stale-ONLY and does NOT clear a crash lock: the
+// recreated container has a new hostname, so restic can't verify the dead PID and won't treat the lock as
+// stale for ~30 min (the overnight-campaign C2 finding — `unlock --remove-all` is required). A second lock
+// failure surfaces the error (never loops). BOUNDARY: a DR-cloned SECOND controller writing the same repo
+// would defeat the single-writer premise — that is operator-supervised territory (see README), out of scope.
+func (m *Manager) resticStep(ctx context.Context, env, base []string, label string, args ...string) ([]byte, error) {
+	full := append(append([]string{}, base...), args...)
+	out, err := m.runner()(ctx, env, full...)
+	if err == nil || !offboxLockRe.Match(out) {
+		return out, err
+	}
+	m.logger.Printf("[WARN] [offbox] cleared a stale exclusive lock left by a previous crash (single-writer repo) before %s; retrying once", label)
+	uctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
+	if uout, uerr := m.runner()(uctx, env, append(append([]string{}, base...), "unlock", "--remove-all")...); uerr != nil {
+		cancel()
+		m.logger.Printf("[WARN] [offbox] unlock --remove-all failed: %v: %s", uerr, truncate(uout))
+		return out, err // surface the original lock error (never loop)
+	}
+	cancel()
+	return m.runner()(ctx, env, full...) // retry exactly ONCE
+}
+
 // ensureOffboxRepo makes sure the SFTP repo exists: probe `cat config`; if absent, `init` (idempotent —
 // a present repo is reused, never re-init). A connect failure surfaces here (fast, via ConnectTimeout).
 func (m *Manager) ensureOffboxRepo(ctx context.Context, base, env []string) error {
@@ -508,6 +547,9 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
 	if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
 		return 0, nil, rerr // fail fast (dead NAS surfaces here)
 	}
+	// Pre-run hygiene: clear any lock restic can prove stale before we start (cheap; the --remove-all
+	// crash-lock escalation lives in resticStep for the locks restic can't self-detect).
+	m.unlockStale(ctx, base, env)
 	var firstErr error
 	for _, stack := range apps {
 		src, ok := m.discoverOffboxUnit(stack)
@@ -517,8 +559,7 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
 			continue
 		}
 		bctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
-		args := append(append([]string{}, base...), "backup", "--tag", "felhom-offbox", "--tag", stack, src)
-		out, berr := m.runner()(bctx, env, args...)
+		out, berr := m.resticStep(bctx, env, base, "backup:"+stack, "backup", "--tag", "felhom-offbox", "--tag", stack, src)
 		cancel()
 		if berr != nil {
 			m.logger.Printf("[ERROR] [offbox] backup %s failed: %v: %s", stack, berr, truncate(out))
@@ -534,10 +575,11 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
 		return backedUp, missing, firstErr
 	}
 	// Retention: keep a sane window, prune the rest. Repo-wide (grouped by host+paths by default).
+	// prune takes an EXCLUSIVE lock — the exact step whose crash left the C2 stale lock — so it goes
+	// through resticStep for the --remove-all self-heal too.
 	fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
 	defer cancel()
-	fargs := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
-	if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
+	if out, ferr := m.resticStep(fctx, env, base, "prune", "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune"); ferr != nil {
 		// A prune failure is non-fatal to the backup itself (data is safe) — log, don't fail the run.
 		m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", ferr, truncate(out))
 	}
@@ -673,8 +715,8 @@ func (m *Manager) RestoreOffbox(ctx context.Context, stackName, destDir string)
 	base, env := m.offboxBaseArgs(t)
 	rctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
 	defer cancel()
-	args := append(append([]string{}, base...), "restore", "latest", "--tag", stackName, "--target", destDir)
-	out, err := m.runner()(rctx, env, args...)
+	m.unlockStale(rctx, base, env) // pre-restore hygiene (crash-lock self-heal is in resticStep)
+	out, err := m.resticStep(rctx, env, base, "restore:"+stackName, "restore", "latest", "--tag", stackName, "--target", destDir)
 	if err != nil {
 		return fmt.Errorf("offbox restore %s: %w: %s", stackName, err, truncate(out))
 	}