v0.110.0: offbox stale-lock self-heal (campaign C2) + crash-truthful status (C1)

resticStep escalates a restic lock error to `unlock --remove-all` + one
retry (safe: single-writer repo — sub-account isolation + single-flight
mutex); plain `unlock` is stale-only and can't clear a crash lock across a
container-hostname change. Pre-run stale unlock hygiene on run+restore.
C1: NewManager flips a persisted LastStatus=running to a truthful error.
Both red-proofed (A reproduces the exact campaign backup failure).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

controller/internal/backup/offbox_test.go

@@ -459,6 +459,129 @@ func TestApplyOffsiteTarget_PreservesEscrowAndStatusOnReapply(t *testing.T) {
 	}
 }
 
+// --- C2 stale-lock self-heal + C1 crash-truthful status (overnight campaign findings) ---
+
+const resticLockErr = "unable to create lock in backend: repository is already locked exclusively by PID 33174 on demo-felhom by root"
+
+// lockRunner records calls and returns the lock error for `backup` the first `lockTimes` times it's called.
+type lockRunner struct {
+	backups, prunes, unlockStale, unlockRemoveAll int
+	lockTimes                                      int
+	seq                                            []string
+}
+
+func (r *lockRunner) run(_ context.Context, _ []string, args ...string) ([]byte, error) {
+	switch {
+	case contains(args, "cat") && contains(args, "config"):
+		return []byte(`{}`), nil
+	case contains(args, "unlock") && contains(args, "--remove-all"):
+		r.unlockRemoveAll++
+		r.seq = append(r.seq, "unlock-all")
+		return nil, nil
+	case contains(args, "unlock"):
+		r.unlockStale++
+		r.seq = append(r.seq, "unlock-stale")
+		return nil, nil
+	case contains(args, "backup"):
+		r.backups++
+		r.seq = append(r.seq, "backup")
+		if r.backups <= r.lockTimes {
+			return []byte(resticLockErr), errors.New("exit status 1")
+		}
+		return nil, nil
+	case contains(args, "forget"):
+		r.prunes++
+		r.seq = append(r.seq, "prune")
+		return nil, nil
+	case contains(args, "snapshots"):
+		return []byte(`[{"id":"a"}]`), nil
+	case contains(args, "stats"):
+		return []byte(`{"total_size":1}`), nil
+	}
+	return nil, nil
+}
+
+func newLockManager(t *testing.T, lr *lockRunner) (*Manager, *settings.Settings) {
+	t.Helper()
+	m, sett := newOffboxManager(t)
+	nsRoot := m.AppNamespaceRoot("rallly")
+	if err := os.MkdirAll(RecoveryUnitPath(nsRoot, "rallly"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	_ = sett.SetAppOffbox("rallly", true)
+	m.SetOffboxRunner(lr.run)
+	return m, sett
+}
+
+// Scenario A — a lock error on backup → unlock --remove-all → retry succeeds → run ok.
+func TestOffbox_LockSelfHealsAndRetries(t *testing.T) {
+	lr := &lockRunner{lockTimes: 1}
+	m, sett := newLockManager(t, lr)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("a self-healable lock must recover, got %v", err)
+	}
+	if lr.unlockRemoveAll != 1 {
+		t.Fatalf("exactly one unlock --remove-all expected, got %d", lr.unlockRemoveAll)
+	}
+	if lr.backups != 2 {
+		t.Fatalf("backup must be retried once after the unlock (2 calls), got %d", lr.backups)
+	}
+	if lr.unlockStale < 1 {
+		t.Fatal("the pre-run stale unlock must have been issued (Scenario C)")
+	}
+	if got := sett.GetOffboxTarget().LastStatus; got != "ok" {
+		t.Fatalf("after self-heal the run must be ok, got %q", got)
+	}
+}
+
+// Scenario B — the lock error persists (twice) → exactly one --remove-all + one retry → error surfaced,
+// no unlock loop.
+func TestOffbox_LockPersistsSurfacesErrorNoLoop(t *testing.T) {
+	lr := &lockRunner{lockTimes: 2}
+	m, sett := newLockManager(t, lr)
+	if err := m.RunOffboxBackup(context.Background()); err == nil {
+		t.Fatal("a persistent lock must surface an error")
+	}
+	if lr.unlockRemoveAll != 1 {
+		t.Fatalf("must NOT loop unlocks — exactly 1 --remove-all, got %d", lr.unlockRemoveAll)
+	}
+	if lr.backups != 2 {
+		t.Fatalf("backup attempted exactly twice (original + one retry), got %d", lr.backups)
+	}
+	if got := sett.GetOffboxTarget().LastStatus; got != "error" {
+		t.Fatalf("status must be error, got %q", got)
+	}
+}
+
+// Scenario E — no lock error → the escalation NEVER fires (unlock --remove-all count stays 0).
+func TestOffbox_NoLockNoRemoveAll(t *testing.T) {
+	lr := &lockRunner{lockTimes: 0}
+	m, _ := newLockManager(t, lr)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if lr.unlockRemoveAll != 0 {
+		t.Fatalf("--remove-all must NEVER fire without a lock error, got %d", lr.unlockRemoveAll)
+	}
+	if lr.unlockStale < 1 {
+		t.Fatal("the pre-run stale unlock still runs on a clean run")
+	}
+}
+
+// Scenario D (C1) — a Manager built while settings say LastStatus="running" (crash mid-run) flips it to a
+// truthful error with the Hungarian interrupted message.
+func TestOffbox_CrashedRunStatusReconciled(t *testing.T) {
+	_, sett := newOffboxManager(t)
+	_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.LastStatus = "running" })
+	// a fresh Manager over the same settings = a restart after a crash
+	m2 := NewManager(&config.Config{}, sett, log.New(os.Stderr, "", 0))
+	_ = m2
+	got := sett.GetOffboxTarget()
+	if got.LastStatus != "error" || !strings.Contains(got.LastError, "megszakadt futás") {
+		t.Fatalf("a crash-interrupted run must become a truthful error, got status=%q err=%q", got.LastStatus, got.LastError)
+	}
+}
+
 // TestOffbox_SingleFlight: an off-box run while another backup holds m.running skips (no runner call).
 func TestOffbox_SingleFlight(t *testing.T) {
 	m, _ := newOffboxManager(t)