v0.104.0: off-box unit discovery (durable, deployment-independent) + no-silent-success

offbox located each toggled app's recovery unit via AppNamespaceRoot→GetAppDrivePath, which reads the app's LIVE app.yaml HDD_PATH and silently falls back to systemDataPath when the app isn't deployed → looked on the wrong drive, backed up nothing, reported ok/0 (DIAG root cause). Now: - discoverOffboxUnit/offboxCandidateNSRoots scan the durable storage registry (schedulable non-decommissioned paths ∪ systemDataPath) for backups/primary/<app>, independent of deploy state; newest-by-manifest-CreatedAt wins on drive churn. - RunOffboxBackup: runOffboxInternal returns (backedUp, missing, err); 0-of-N toggled → hard error + operator alert; partial → ok + new OffboxTarget.LastWarning (shown on /backups, preserved across config edit). - AppNamespaceRoot + primary WRITE paths unchanged. - 6 non-hollow tests (A-E + edge) + both companion red-proofs run (reverted). - NOT yet live-validated against the Storage Box (spike creds torn down). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-09 12:32:05 +02:00
parent 59eb3bea76
commit 908e4b906a
8 changed files with 390 additions and 26 deletions
@@ -2,6 +2,8 @@ package backup

 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"log"
 	"os"
 	"path/filepath"
@@ -290,6 +292,212 @@ func TestOffbox_ValidateRejectsInjection(t *testing.T) {
 	}
 }

+// --- Off-box unit DISCOVERY + no-silent-success (§7 A–E) ---
+
+// recordingOffboxRunner captures the exact `src` path of each restic `backup` call and returns success
+// for the repo probe/init/snapshots/stats/forget. Per-stack `backup` errors are configurable.
+type recordingOffboxRunner struct {
+	backupSrc []string         // src path per backup call, in order (the load-bearing effect to assert)
+	backupErr map[string]error // stack (last --tag) → error to return from `backup`
+}
+
+func (rr *recordingOffboxRunner) run(_ context.Context, _ []string, args ...string) ([]byte, error) {
+	switch {
+	case contains(args, "cat") && contains(args, "config"):
+		return []byte(`{"version":2}`), nil // repo exists (no init)
+	case contains(args, "backup"):
+		rr.backupSrc = append(rr.backupSrc, args[len(args)-1]) // last arg = src path
+		if e := rr.backupErr[tagOf(args)]; e != nil {
+			return []byte("restic backup failed"), e
+		}
+		return nil, nil
+	case contains(args, "forget"):
+		return nil, nil
+	case contains(args, "snapshots"):
+		return []byte(`[{"id":"s1"}]`), nil
+	case contains(args, "stats"):
+		return []byte(`{"total_size":123}`), nil
+	}
+	return nil, nil
+}
+
+// addSchedulablePath registers a schedulable (non-decommissioned) storage path — a candidate drive.
+func addSchedulablePath(t *testing.T, sett *settings.Settings, p string) {
+	t.Helper()
+	if err := sett.AddStoragePath(settings.StoragePath{Path: p, Schedulable: true, AddedAt: "2026-07-01T00:00:00Z"}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// writeUnit lays a recovery unit (dir + a manifest with the given CreatedAt) at <nsRoot>/backups/primary/<app>.
+func writeUnit(t *testing.T, nsRoot, app, createdAt string) {
+	t.Helper()
+	if err := os.MkdirAll(RecoveryUnitPath(nsRoot, app), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	b, _ := json.Marshal(RecoveryManifest{AppName: app, CreatedAt: createdAt})
+	if err := os.WriteFile(RecoveryUnitManifestPath(nsRoot, app), b, 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// A — undeployed-but-toggled app whose unit lives on a REGISTERED drive (not systemDataPath): discovery
+// must find it there. The harness has no stackProvider, so the OLD AppNamespaceRoot path would resolve to
+// systemDataPath (where no unit is) — the F1 bug. See the companion red-proof in REPORT.
+func TestOffbox_DiscoversUnitOnRegisteredDrive(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	usb := t.TempDir()
+	addSchedulablePath(t, sett, usb)
+	usbNS := m.namespaceRoot(usb) // registered drive → in-guest → nsRoot == usb
+	writeUnit(t, usbNS, "audiobookshelf", "2026-07-01T00:00:00Z")
+	// prove the OLD resolution would have looked elsewhere (no unit at systemDataPath nsRoot)
+	if _, err := os.Stat(RecoveryUnitPath(m.namespaceRoot(m.systemDataPath), "audiobookshelf")); err == nil {
+		t.Fatal("setup: unit must NOT exist on systemDataPath for this repro")
+	}
+	_ = sett.SetAppOffbox("audiobookshelf", true)
+
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+	wantSrc := RecoveryUnitPath(usbNS, "audiobookshelf")
+	if len(rr.backupSrc) != 1 || rr.backupSrc[0] != wantSrc {
+		t.Fatalf("backup src = %v, want exactly [%s] (discovered on the registered drive)", rr.backupSrc, wantSrc)
+	}
+	if st := sett.GetOffboxTarget(); st.LastStatus != "ok" || st.LastWarning != "" {
+		t.Fatalf("status = %+v, want ok / no warning", st)
+	}
+}
+
+// B — a toggled app with NO recovery unit anywhere: the run must be a HARD ERROR (no silent ok/0), alert
+// the operator, and record status=error naming the app. Companion red-proof in REPORT.
+func TestOffbox_NoUnitAnywhereIsHardError(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	_ = sett.SetAppOffbox("ghost", true) // toggled; no unit created anywhere
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	var notified bool
+	var notifiedErr error
+	m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { notified = true; notifiedErr = err })
+
+	err := m.RunOffboxBackup(context.Background())
+	if err == nil {
+		t.Fatal("0-of-N toggled apps backed up must ERROR (no silent success)")
+	}
+	if len(rr.backupSrc) != 0 {
+		t.Fatalf("no backup should have run, got %v", rr.backupSrc)
+	}
+	if !notified || notifiedErr == nil {
+		t.Fatal("the 0/N run must alert the operator with a non-nil err")
+	}
+	st := sett.GetOffboxTarget()
+	if st.LastStatus != "error" || st.LastError == "" {
+		t.Fatalf("status must be error, got %+v", st)
+	}
+	if !strings.Contains(st.LastError, "ghost") {
+		t.Fatalf("LastError should name the missing app, got %q", st.LastError)
+	}
+}
+
+// C — partial: one toggled app has a unit, another doesn't → the present one is backed up, status stays
+// ok, notify err is nil, but LastWarning (Hungarian) names the missing app.
+func TestOffbox_PartialRunWarnsNotErrors(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	usb := t.TempDir()
+	addSchedulablePath(t, sett, usb)
+	usbNS := m.namespaceRoot(usb)
+	writeUnit(t, usbNS, "present", "2026-07-01T00:00:00Z")
+	_ = sett.SetAppOffbox("present", true)
+	_ = sett.SetAppOffbox("gone", true) // no unit
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	notifiedErr := errors.New("sentinel") // must be cleared to nil by a non-erroring run
+	m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { notifiedErr = err })
+
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("a partial run must NOT error, got %v", err)
+	}
+	if len(rr.backupSrc) != 1 || rr.backupSrc[0] != RecoveryUnitPath(usbNS, "present") {
+		t.Fatalf("only 'present' should be backed up, got %v", rr.backupSrc)
+	}
+	if notifiedErr != nil {
+		t.Fatalf("partial run must notify with nil err, got %v", notifiedErr)
+	}
+	st := sett.GetOffboxTarget()
+	if st.LastStatus != "ok" {
+		t.Fatalf("status = %q, want ok", st.LastStatus)
+	}
+	if !strings.Contains(st.LastWarning, "gone") {
+		t.Fatalf("LastWarning must name the missing app 'gone', got %q", st.LastWarning)
+	}
+}
+
+// D — the same app's unit on TWO registered drives (drive churn): exactly ONE backup call, for the NEWER
+// unit (by manifest CreatedAt).
+func TestOffbox_MultipleUnitsPicksNewest(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	older, newer := t.TempDir(), t.TempDir()
+	addSchedulablePath(t, sett, older)
+	addSchedulablePath(t, sett, newer)
+	olderNS, newerNS := m.namespaceRoot(older), m.namespaceRoot(newer)
+	writeUnit(t, olderNS, "z", "2026-01-01T00:00:00Z")
+	writeUnit(t, newerNS, "z", "2026-07-01T00:00:00Z")
+	_ = sett.SetAppOffbox("z", true)
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+	wantSrc := RecoveryUnitPath(newerNS, "z")
+	if len(rr.backupSrc) != 1 || rr.backupSrc[0] != wantSrc {
+		t.Fatalf("must back up exactly the NEWER unit once; got %v want [%s]", rr.backupSrc, wantSrc)
+	}
+}
+
+// E — every toggled app present: all backed up, status ok, no warning, snapshot count reflects stats.
+func TestOffbox_AllPresentHappyPath(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	usb := t.TempDir()
+	addSchedulablePath(t, sett, usb)
+	usbNS := m.namespaceRoot(usb)
+	writeUnit(t, usbNS, "a", "2026-07-01T00:00:00Z")
+	writeUnit(t, usbNS, "b", "2026-07-01T00:00:00Z")
+	_ = sett.SetAppOffbox("a", true)
+	_ = sett.SetAppOffbox("b", true)
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("run: %v", err)
+	}
+	if len(rr.backupSrc) != 2 {
+		t.Fatalf("both apps must be backed up, got %v", rr.backupSrc)
+	}
+	st := sett.GetOffboxTarget()
+	if st.LastStatus != "ok" || st.LastWarning != "" {
+		t.Fatalf("happy path wants ok + no warning, got %+v", st)
+	}
+	if st.SnapshotCount != 1 {
+		t.Fatalf("snapshot count should reflect the stats fake (1), got %d", st.SnapshotCount)
+	}
+}
+
+// Edge — zero toggled apps is a clean no-op ok (no error, no backup call).
+func TestOffbox_NoAppsToggledIsCleanOK(t *testing.T) {
+	m, sett := newOffboxManager(t)
+	rr := &recordingOffboxRunner{}
+	m.SetOffboxRunner(rr.run)
+	if err := m.RunOffboxBackup(context.Background()); err != nil {
+		t.Fatalf("zero toggled apps must be a clean no-op, got %v", err)
+	}
+	if len(rr.backupSrc) != 0 {
+		t.Fatalf("no backup should run with 0 toggled apps, got %v", rr.backupSrc)
+	}
+	if st := sett.GetOffboxTarget(); st.LastStatus != "ok" || st.LastError != "" {
+		t.Fatalf("status = %+v, want ok / no error", st)
+	}
+}
+
 func runtimeIsUnix() bool { return os.PathSeparator == '/' }

 func contains(ss []string, want string) bool {