felhom-controller/controller/internal/backup/offbox_test.go

package backup

import (
	"context"
	"encoding/json"
	"errors"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// newOffboxManager builds a Manager with a temp data dir + a configured + enabled off-box target and the
// 0600 secret files written, so OffboxConfigured() is true.
func newOffboxManager(t *testing.T) (*Manager, *settings.Settings) {
	t.Helper()
	logger := log.New(os.Stderr, "", 0)
	dataDir := t.TempDir()
	sett, err := settings.Load(filepath.Join(dataDir, "settings.json"), logger)
	if err != nil {
		t.Fatal(err)
	}
	cfg := &config.Config{}
	cfg.Paths.DataDir = dataDir
	cfg.Paths.SystemDataPath = filepath.Join(dataDir, "sys")
	m := NewManager(cfg, sett, logger)
	if err := sett.SetOffboxTarget(&settings.OffboxTarget{
		Enabled: true, Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo", Schedule: "daily",
	}); err != nil {
		t.Fatal(err)
	}
	if err := m.WriteOffboxSecrets("PRIVATE-KEY-MATERIAL", "nas.local ssh-ed25519 AAAAhostkey"); err != nil {
		t.Fatal(err)
	}
	return m, sett
}

// argsContainTimeout reports whether the restic arg vector carries the load-bearing ConnectTimeout.
func argsContainTimeout(args []string) bool {
	return strings.Contains(strings.Join(args, " "), "-oConnectTimeout=")
}

// TestOffbox_BaseArgsCarryConnectTimeout asserts the mandatory fail-fast + hardening args are present.
func TestOffbox_BaseArgsCarryConnectTimeout(t *testing.T) {
	m, sett := newOffboxManager(t)
	base, env := m.offboxBaseArgs(sett.GetOffboxTarget())
	joined := strings.Join(base, " ")
	for _, want := range []string{
		"-oConnectTimeout=10",        // THE spike Q8 fail-fast knob
		"-oStrictHostKeyChecking=yes", // no blind TOFU
		"-oUserKnownHostsFile=",       // pinned host key
		"-oBatchMode=yes",             // no interactive hang
		"sftp:felhom@nas.local:/srv/repo",
	} {
		if !strings.Contains(joined, want) {
			t.Errorf("base args missing %q: %s", want, joined)
		}
	}
	if len(env) != 1 || !strings.HasPrefix(env[0], "RESTIC_PASSWORD_FILE=") {
		t.Errorf("env must set RESTIC_PASSWORD_FILE only, got %v", env)
	}
}

// failFastFake models the SSH transport's ConnectTimeout honoring: if the restic args carry
// -oConnectTimeout it returns a connect error promptly (fail-fast); WITHOUT it, it blocks until the ctx
// deadline (the dead-NAS multi-minute hang). This is the seam the ConnectTimeout companion exercises.
func failFastFake(_ *testing.T) offboxRunner {
	return func(ctx context.Context, _ []string, args ...string) ([]byte, error) {
		if argsContainTimeout(args) {
			return []byte("dial tcp: connect: connection refused"), context.DeadlineExceeded // fast, bounded
		}
		<-ctx.Done() // no timeout arg → hang until the caller's deadline (the bug)
		return nil, ctx.Err()
	}
}

// TestOffbox_ConnectTimeoutIsLoadBearing is the §10 companion red-proof: WITH the arg the (fake) connect
// fails fast (well under the bound); WITHOUT it the connect blocks past the bound. A build that dropped
// the ConnectTimeout arg would take the slow path → this proves the arg is load-bearing.
func TestOffbox_ConnectTimeoutIsLoadBearing(t *testing.T) {
	m, sett := newOffboxManager(t)
	realArgs, env := m.offboxBaseArgs(sett.GetOffboxTarget())
	fake := failFastFake(t)

	// WITH the arg: returns promptly (we model fast as an immediate error, not a ctx hang).
	ctx1, c1 := context.WithTimeout(context.Background(), 2*time.Second)
	defer c1()
	start := time.Now()
	_, err := fake(ctx1, env, append(append([]string{}, realArgs...), "cat", "config")...)
	if elapsed := time.Since(start); elapsed > time.Second {
		t.Fatalf("with ConnectTimeout the connect must fail fast, took %s", elapsed)
	}
	_ = err

	// WITHOUT the arg (the bug): blocks until the ctx deadline.
	stripped := stripConnectTimeout(realArgs)
	if argsContainTimeout(stripped) {
		t.Fatal("test setup: stripped args still contain the timeout")
	}
	ctx2, c2 := context.WithTimeout(context.Background(), 300*time.Millisecond)
	defer c2()
	start = time.Now()
	_, err = fake(ctx2, env, append(append([]string{}, stripped...), "cat", "config")...)
	if err != context.DeadlineExceeded {
		t.Fatalf("without ConnectTimeout the connect should block to the deadline, got %v", err)
	}
	if elapsed := time.Since(start); elapsed < 250*time.Millisecond {
		t.Fatalf("without ConnectTimeout it should have hung to the bound, only took %s", elapsed)
	}
}

func stripConnectTimeout(args []string) []string {
	out := make([]string, len(args))
	for i, a := range args {
		out[i] = strings.ReplaceAll(a, "-oConnectTimeout=10 ", "")
	}
	return out
}

// TestOffbox_RunFailsFastAndAlerts: a dead-NAS run returns an error promptly, records status=error, and
// fires the operator alert.
func TestOffbox_RunFailsFastAndAlerts(t *testing.T) {
	m, sett := newOffboxManager(t)
	m.SetOffboxRunner(func(ctx context.Context, _ []string, args ...string) ([]byte, error) {
		// dead NAS: every op (incl. the repo probe) errors fast.
		return []byte("unable to open repository: connection refused"), context.DeadlineExceeded
	})
	var mu sync.Mutex
	var gotErr error
	var notified bool
	m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { mu.Lock(); defer mu.Unlock(); notified = true; gotErr = err })
	_ = sett.SetAppOffbox("rallly", true)

	start := time.Now()
	err := m.RunOffboxBackup(context.Background())
	if err == nil {
		t.Fatal("a dead NAS must produce a failed run")
	}
	if time.Since(start) > 5*time.Second {
		t.Fatalf("run should fail fast, took %s", time.Since(start))
	}
	if !notified || gotErr == nil {
		t.Fatal("a failed off-box run must alert the operator")
	}
	if st := sett.GetOffboxTarget(); st.LastStatus != "error" || st.LastError == "" {
		t.Fatalf("status must record the failure, got %+v", st)
	}
}

// TestOffbox_RepoIdempotent: when the repo exists (cat config succeeds), ensure must NOT init.
func TestOffbox_RepoIdempotent(t *testing.T) {
	m, sett := newOffboxManager(t)
	var inits int
	m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		switch {
		case contains(args, "cat") && contains(args, "config"):
			return []byte(`{"version":2}`), nil // repo exists
		case contains(args, "init"):
			inits++
			return nil, nil
		}
		return nil, nil
	})
	base, env := m.offboxBaseArgs(sett.GetOffboxTarget())
	if err := m.ensureOffboxRepo(context.Background(), base, env); err != nil {
		t.Fatal(err)
	}
	if inits != 0 {
		t.Fatalf("an existing repo must NOT be re-initialized, init called %d times", inits)
	}
}

// TestOffbox_RestoreRoundTrip: backup a temp tree (fake records src per tag), restore (fake copies the
// recorded tree to target) → byte-identical. Exercises the orchestration without real restic.
func TestOffbox_RestoreRoundTrip(t *testing.T) {
	m, sett := newOffboxManager(t)
	// Lay down an app's recovery-unit tree on disk (what RunOffboxBackup will back up).
	nsRoot := m.AppNamespaceRoot("rallly")
	src := RecoveryUnitPath(nsRoot, "rallly")
	if err := os.MkdirAll(filepath.Join(src, "db-dumps"), 0o755); err != nil {
		t.Fatal(err)
	}
	want := []byte("CREATE TABLE x; -- dump bytes")
	if err := os.WriteFile(filepath.Join(src, "db-dumps", "rallly.sql"), want, 0o644); err != nil {
		t.Fatal(err)
	}
	_ = sett.SetAppOffbox("rallly", true)

	captured := map[string]string{} // tag → src path
	m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		switch {
		case contains(args, "cat") && contains(args, "config"):
			return []byte(`{}`), nil
		case contains(args, "backup"):
			captured[tagOf(args)] = args[len(args)-1] // last arg = src path
			return nil, nil
		case contains(args, "forget"):
			return nil, nil
		case contains(args, "restore"):
			target := valAfter(args, "--target")
			if err := copyTree(captured[tagOf(args)], target); err != nil {
				return nil, err
			}
			return nil, nil
		case contains(args, "snapshots"):
			return []byte(`[{"id":"abc"}]`), nil
		case contains(args, "stats"):
			return []byte(`{"total_size":123}`), nil
		}
		return nil, nil
	})

	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("backup: %v", err)
	}
	dest := t.TempDir()
	if err := m.RestoreOffbox(context.Background(), "rallly", dest); err != nil {
		t.Fatalf("restore: %v", err)
	}
	got, err := os.ReadFile(filepath.Join(dest, "db-dumps", "rallly.sql"))
	if err != nil {
		t.Fatalf("restored file missing: %v", err)
	}
	if string(got) != string(want) {
		t.Fatalf("restore not byte-identical: got %q want %q", got, want)
	}
}

// TestOffbox_SingleFlight: an off-box run while another backup holds m.running skips (no runner call).
func TestOffbox_SingleFlight(t *testing.T) {
	m, _ := newOffboxManager(t)
	called := false
	m.SetOffboxRunner(func(context.Context, []string, ...string) ([]byte, error) { called = true; return nil, nil })
	_ = m.acquireRunning() // simulate a concurrent backup holding the flag
	defer m.releaseRunning()
	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("single-flight skip should not error, got %v", err)
	}
	if called {
		t.Fatal("off-box must not run (race) while another backup holds the flag")
	}
}

// TestOffbox_SecretsAre0600: the SSH key + repo password files are 0600; the password is non-empty.
func TestOffbox_SecretsAre0600(t *testing.T) {
	m, _ := newOffboxManager(t)
	for _, p := range []string{m.offboxKeyPath(), m.offboxPwPath()} {
		info, err := os.Stat(p)
		if err != nil {
			t.Fatalf("secret file missing: %v", err)
		}
		if runtimeIsUnix() && info.Mode().Perm()&0o077 != 0 {
			t.Errorf("%s is group/other-readable (mode %v) — must be 0600", p, info.Mode().Perm())
		}
	}
	pw, _ := os.ReadFile(m.offboxPwPath())
	if len(strings.TrimSpace(string(pw))) < 32 {
		t.Errorf("repo password too short / empty")
	}
}

// --- tiny test helpers ---

// TestOffbox_ValidateRejectsInjection is the security companion: host/user/repo values that could inject
// an ssh option (leading '-' → e.g. -oProxyCommand) or a shell metacharacter must be REFUSED; a clean
// target is accepted. A build without this guard would let a hostile target reach the ssh exec → FAIL.
func TestOffbox_ValidateRejectsInjection(t *testing.T) {
	ok := &settings.OffboxTarget{Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo"}
	if err := ValidateOffboxTarget(ok); err != nil {
		t.Fatalf("clean target rejected: %v", err)
	}
	bad := []settings.OffboxTarget{
		{Host: "-oProxyCommand=touch /tmp/pwn", User: "felhom", RepoPath: "/srv/repo"}, // ssh option injection
		{Host: "nas;rm -rf /", User: "felhom", RepoPath: "/srv/repo"},                  // metacharacters
		{Host: "nas.local", User: "-oProxyCommand=x", RepoPath: "/srv/repo"},           // user option injection
		{Host: "nas.local", User: "felhom", RepoPath: "/srv/repo; evil"},              // path metacharacters
		{Host: "nas.local", User: "felhom", RepoPath: "/srv/../etc"},                  // traversal
		{Host: "nas local", User: "felhom", RepoPath: "/srv/repo"},                    // space
		{Host: "nas.local", User: "felhom", RepoPath: "relative/path"},                // non-absolute
	}
	for i, b := range bad {
		bb := b
		if err := ValidateOffboxTarget(&bb); err == nil {
			t.Errorf("case %d (%+v) must be rejected", i, bb)
		}
	}
}

// --- Off-box unit DISCOVERY + no-silent-success (§7 A–E) ---

// recordingOffboxRunner captures the exact `src` path of each restic `backup` call and returns success
// for the repo probe/init/snapshots/stats/forget. Per-stack `backup` errors are configurable.
type recordingOffboxRunner struct {
	backupSrc []string         // src path per backup call, in order (the load-bearing effect to assert)
	backupErr map[string]error // stack (last --tag) → error to return from `backup`
}

func (rr *recordingOffboxRunner) run(_ context.Context, _ []string, args ...string) ([]byte, error) {
	switch {
	case contains(args, "cat") && contains(args, "config"):
		return []byte(`{"version":2}`), nil // repo exists (no init)
	case contains(args, "backup"):
		rr.backupSrc = append(rr.backupSrc, args[len(args)-1]) // last arg = src path
		if e := rr.backupErr[tagOf(args)]; e != nil {
			return []byte("restic backup failed"), e
		}
		return nil, nil
	case contains(args, "forget"):
		return nil, nil
	case contains(args, "snapshots"):
		return []byte(`[{"id":"s1"}]`), nil
	case contains(args, "stats"):
		return []byte(`{"total_size":123}`), nil
	}
	return nil, nil
}

// addSchedulablePath registers a schedulable (non-decommissioned) storage path — a candidate drive.
func addSchedulablePath(t *testing.T, sett *settings.Settings, p string) {
	t.Helper()
	if err := sett.AddStoragePath(settings.StoragePath{Path: p, Schedulable: true, AddedAt: "2026-07-01T00:00:00Z"}); err != nil {
		t.Fatal(err)
	}
}

// writeUnit lays a recovery unit (dir + a manifest with the given CreatedAt) at <nsRoot>/backups/primary/<app>.
func writeUnit(t *testing.T, nsRoot, app, createdAt string) {
	t.Helper()
	if err := os.MkdirAll(RecoveryUnitPath(nsRoot, app), 0o755); err != nil {
		t.Fatal(err)
	}
	b, _ := json.Marshal(RecoveryManifest{AppName: app, CreatedAt: createdAt})
	if err := os.WriteFile(RecoveryUnitManifestPath(nsRoot, app), b, 0o644); err != nil {
		t.Fatal(err)
	}
}

// A — undeployed-but-toggled app whose unit lives on a REGISTERED drive (not systemDataPath): discovery
// must find it there. The harness has no stackProvider, so the OLD AppNamespaceRoot path would resolve to
// systemDataPath (where no unit is) — the F1 bug. See the companion red-proof in REPORT.
func TestOffbox_DiscoversUnitOnRegisteredDrive(t *testing.T) {
	m, sett := newOffboxManager(t)
	usb := t.TempDir()
	addSchedulablePath(t, sett, usb)
	usbNS := m.namespaceRoot(usb) // registered drive → in-guest → nsRoot == usb
	writeUnit(t, usbNS, "audiobookshelf", "2026-07-01T00:00:00Z")
	// prove the OLD resolution would have looked elsewhere (no unit at systemDataPath nsRoot)
	if _, err := os.Stat(RecoveryUnitPath(m.namespaceRoot(m.systemDataPath), "audiobookshelf")); err == nil {
		t.Fatal("setup: unit must NOT exist on systemDataPath for this repro")
	}
	_ = sett.SetAppOffbox("audiobookshelf", true)

	rr := &recordingOffboxRunner{}
	m.SetOffboxRunner(rr.run)
	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("run: %v", err)
	}
	wantSrc := RecoveryUnitPath(usbNS, "audiobookshelf")
	if len(rr.backupSrc) != 1 || rr.backupSrc[0] != wantSrc {
		t.Fatalf("backup src = %v, want exactly [%s] (discovered on the registered drive)", rr.backupSrc, wantSrc)
	}
	if st := sett.GetOffboxTarget(); st.LastStatus != "ok" || st.LastWarning != "" {
		t.Fatalf("status = %+v, want ok / no warning", st)
	}
}

// B — a toggled app with NO recovery unit anywhere: the run must be a HARD ERROR (no silent ok/0), alert
// the operator, and record status=error naming the app. Companion red-proof in REPORT.
func TestOffbox_NoUnitAnywhereIsHardError(t *testing.T) {
	m, sett := newOffboxManager(t)
	_ = sett.SetAppOffbox("ghost", true) // toggled; no unit created anywhere
	rr := &recordingOffboxRunner{}
	m.SetOffboxRunner(rr.run)
	var notified bool
	var notifiedErr error
	m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { notified = true; notifiedErr = err })

	err := m.RunOffboxBackup(context.Background())
	if err == nil {
		t.Fatal("0-of-N toggled apps backed up must ERROR (no silent success)")
	}
	if len(rr.backupSrc) != 0 {
		t.Fatalf("no backup should have run, got %v", rr.backupSrc)
	}
	if !notified || notifiedErr == nil {
		t.Fatal("the 0/N run must alert the operator with a non-nil err")
	}
	st := sett.GetOffboxTarget()
	if st.LastStatus != "error" || st.LastError == "" {
		t.Fatalf("status must be error, got %+v", st)
	}
	if !strings.Contains(st.LastError, "ghost") {
		t.Fatalf("LastError should name the missing app, got %q", st.LastError)
	}
}

// C — partial: one toggled app has a unit, another doesn't → the present one is backed up, status stays
// ok, notify err is nil, but LastWarning (Hungarian) names the missing app.
func TestOffbox_PartialRunWarnsNotErrors(t *testing.T) {
	m, sett := newOffboxManager(t)
	usb := t.TempDir()
	addSchedulablePath(t, sett, usb)
	usbNS := m.namespaceRoot(usb)
	writeUnit(t, usbNS, "present", "2026-07-01T00:00:00Z")
	_ = sett.SetAppOffbox("present", true)
	_ = sett.SetAppOffbox("gone", true) // no unit
	rr := &recordingOffboxRunner{}
	m.SetOffboxRunner(rr.run)
	notifiedErr := errors.New("sentinel") // must be cleared to nil by a non-erroring run
	m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { notifiedErr = err })

	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("a partial run must NOT error, got %v", err)
	}
	if len(rr.backupSrc) != 1 || rr.backupSrc[0] != RecoveryUnitPath(usbNS, "present") {
		t.Fatalf("only 'present' should be backed up, got %v", rr.backupSrc)
	}
	if notifiedErr != nil {
		t.Fatalf("partial run must notify with nil err, got %v", notifiedErr)
	}
	st := sett.GetOffboxTarget()
	if st.LastStatus != "ok" {
		t.Fatalf("status = %q, want ok", st.LastStatus)
	}
	if !strings.Contains(st.LastWarning, "gone") {
		t.Fatalf("LastWarning must name the missing app 'gone', got %q", st.LastWarning)
	}
}

// D — the same app's unit on TWO registered drives (drive churn): exactly ONE backup call, for the NEWER
// unit (by manifest CreatedAt).
func TestOffbox_MultipleUnitsPicksNewest(t *testing.T) {
	m, sett := newOffboxManager(t)
	older, newer := t.TempDir(), t.TempDir()
	addSchedulablePath(t, sett, older)
	addSchedulablePath(t, sett, newer)
	olderNS, newerNS := m.namespaceRoot(older), m.namespaceRoot(newer)
	writeUnit(t, olderNS, "z", "2026-01-01T00:00:00Z")
	writeUnit(t, newerNS, "z", "2026-07-01T00:00:00Z")
	_ = sett.SetAppOffbox("z", true)
	rr := &recordingOffboxRunner{}
	m.SetOffboxRunner(rr.run)
	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("run: %v", err)
	}
	wantSrc := RecoveryUnitPath(newerNS, "z")
	if len(rr.backupSrc) != 1 || rr.backupSrc[0] != wantSrc {
		t.Fatalf("must back up exactly the NEWER unit once; got %v want [%s]", rr.backupSrc, wantSrc)
	}
}

// E — every toggled app present: all backed up, status ok, no warning, snapshot count reflects stats.
func TestOffbox_AllPresentHappyPath(t *testing.T) {
	m, sett := newOffboxManager(t)
	usb := t.TempDir()
	addSchedulablePath(t, sett, usb)
	usbNS := m.namespaceRoot(usb)
	writeUnit(t, usbNS, "a", "2026-07-01T00:00:00Z")
	writeUnit(t, usbNS, "b", "2026-07-01T00:00:00Z")
	_ = sett.SetAppOffbox("a", true)
	_ = sett.SetAppOffbox("b", true)
	rr := &recordingOffboxRunner{}
	m.SetOffboxRunner(rr.run)
	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("run: %v", err)
	}
	if len(rr.backupSrc) != 2 {
		t.Fatalf("both apps must be backed up, got %v", rr.backupSrc)
	}
	st := sett.GetOffboxTarget()
	if st.LastStatus != "ok" || st.LastWarning != "" {
		t.Fatalf("happy path wants ok + no warning, got %+v", st)
	}
	if st.SnapshotCount != 1 {
		t.Fatalf("snapshot count should reflect the stats fake (1), got %d", st.SnapshotCount)
	}
}

// Edge — zero toggled apps is a clean no-op ok (no error, no backup call).
func TestOffbox_NoAppsToggledIsCleanOK(t *testing.T) {
	m, sett := newOffboxManager(t)
	rr := &recordingOffboxRunner{}
	m.SetOffboxRunner(rr.run)
	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("zero toggled apps must be a clean no-op, got %v", err)
	}
	if len(rr.backupSrc) != 0 {
		t.Fatalf("no backup should run with 0 toggled apps, got %v", rr.backupSrc)
	}
	if st := sett.GetOffboxTarget(); st.LastStatus != "ok" || st.LastError != "" {
		t.Fatalf("status = %+v, want ok / no error", st)
	}
}

func runtimeIsUnix() bool { return os.PathSeparator == '/' }

func contains(ss []string, want string) bool {
	for _, s := range ss {
		if s == want {
			return true
		}
	}
	return false
}

// tagOf returns the LAST --tag value (the per-stack tag; backup adds "felhom-offbox" then the stack).
func tagOf(args []string) string {
	tag := ""
	for i, a := range args {
		if a == "--tag" && i+1 < len(args) {
			tag = args[i+1]
		}
	}
	return tag
}

func valAfter(args []string, flag string) string {
	for i, a := range args {
		if a == flag && i+1 < len(args) {
			return args[i+1]
		}
	}
	return ""
}

func copyTree(src, dst string) error {
	return filepath.Walk(src, func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		rel, _ := filepath.Rel(src, p)
		target := filepath.Join(dst, rel)
		if info.IsDir() {
			return os.MkdirAll(target, 0o755)
		}
		b, rerr := os.ReadFile(p)
		if rerr != nil {
			return rerr
		}
		return os.WriteFile(target, b, 0o644)
	})
}