felhom-controller/controller/internal/backup/offbox_test.go

package backup

import (
	"context"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// newOffboxManager builds a Manager with a temp data dir + a configured + enabled off-box target and the
// 0600 secret files written, so OffboxConfigured() is true.
func newOffboxManager(t *testing.T) (*Manager, *settings.Settings) {
	t.Helper()
	logger := log.New(os.Stderr, "", 0)
	dataDir := t.TempDir()
	sett, err := settings.Load(filepath.Join(dataDir, "settings.json"), logger)
	if err != nil {
		t.Fatal(err)
	}
	cfg := &config.Config{}
	cfg.Paths.DataDir = dataDir
	cfg.Paths.SystemDataPath = filepath.Join(dataDir, "sys")
	m := NewManager(cfg, sett, logger)
	if err := sett.SetOffboxTarget(&settings.OffboxTarget{
		Enabled: true, Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo", Schedule: "daily",
	}); err != nil {
		t.Fatal(err)
	}
	if err := m.WriteOffboxSecrets("PRIVATE-KEY-MATERIAL", "nas.local ssh-ed25519 AAAAhostkey"); err != nil {
		t.Fatal(err)
	}
	return m, sett
}

// argsContainTimeout reports whether the restic arg vector carries the load-bearing ConnectTimeout.
func argsContainTimeout(args []string) bool {
	return strings.Contains(strings.Join(args, " "), "-oConnectTimeout=")
}

// TestOffbox_BaseArgsCarryConnectTimeout asserts the mandatory fail-fast + hardening args are present.
func TestOffbox_BaseArgsCarryConnectTimeout(t *testing.T) {
	m, sett := newOffboxManager(t)
	base, env := m.offboxBaseArgs(sett.GetOffboxTarget())
	joined := strings.Join(base, " ")
	for _, want := range []string{
		"-oConnectTimeout=10",        // THE spike Q8 fail-fast knob
		"-oStrictHostKeyChecking=yes", // no blind TOFU
		"-oUserKnownHostsFile=",       // pinned host key
		"-oBatchMode=yes",             // no interactive hang
		"sftp:felhom@nas.local:/srv/repo",
	} {
		if !strings.Contains(joined, want) {
			t.Errorf("base args missing %q: %s", want, joined)
		}
	}
	if len(env) != 1 || !strings.HasPrefix(env[0], "RESTIC_PASSWORD_FILE=") {
		t.Errorf("env must set RESTIC_PASSWORD_FILE only, got %v", env)
	}
}

// failFastFake models the SSH transport's ConnectTimeout honoring: if the restic args carry
// -oConnectTimeout it returns a connect error promptly (fail-fast); WITHOUT it, it blocks until the ctx
// deadline (the dead-NAS multi-minute hang). This is the seam the ConnectTimeout companion exercises.
func failFastFake(_ *testing.T) offboxRunner {
	return func(ctx context.Context, _ []string, args ...string) ([]byte, error) {
		if argsContainTimeout(args) {
			return []byte("dial tcp: connect: connection refused"), context.DeadlineExceeded // fast, bounded
		}
		<-ctx.Done() // no timeout arg → hang until the caller's deadline (the bug)
		return nil, ctx.Err()
	}
}

// TestOffbox_ConnectTimeoutIsLoadBearing is the §10 companion red-proof: WITH the arg the (fake) connect
// fails fast (well under the bound); WITHOUT it the connect blocks past the bound. A build that dropped
// the ConnectTimeout arg would take the slow path → this proves the arg is load-bearing.
func TestOffbox_ConnectTimeoutIsLoadBearing(t *testing.T) {
	m, sett := newOffboxManager(t)
	realArgs, env := m.offboxBaseArgs(sett.GetOffboxTarget())
	fake := failFastFake(t)

	// WITH the arg: returns promptly (we model fast as an immediate error, not a ctx hang).
	ctx1, c1 := context.WithTimeout(context.Background(), 2*time.Second)
	defer c1()
	start := time.Now()
	_, err := fake(ctx1, env, append(append([]string{}, realArgs...), "cat", "config")...)
	if elapsed := time.Since(start); elapsed > time.Second {
		t.Fatalf("with ConnectTimeout the connect must fail fast, took %s", elapsed)
	}
	_ = err

	// WITHOUT the arg (the bug): blocks until the ctx deadline.
	stripped := stripConnectTimeout(realArgs)
	if argsContainTimeout(stripped) {
		t.Fatal("test setup: stripped args still contain the timeout")
	}
	ctx2, c2 := context.WithTimeout(context.Background(), 300*time.Millisecond)
	defer c2()
	start = time.Now()
	_, err = fake(ctx2, env, append(append([]string{}, stripped...), "cat", "config")...)
	if err != context.DeadlineExceeded {
		t.Fatalf("without ConnectTimeout the connect should block to the deadline, got %v", err)
	}
	if elapsed := time.Since(start); elapsed < 250*time.Millisecond {
		t.Fatalf("without ConnectTimeout it should have hung to the bound, only took %s", elapsed)
	}
}

func stripConnectTimeout(args []string) []string {
	out := make([]string, len(args))
	for i, a := range args {
		out[i] = strings.ReplaceAll(a, "-oConnectTimeout=10 ", "")
	}
	return out
}

// TestOffbox_RunFailsFastAndAlerts: a dead-NAS run returns an error promptly, records status=error, and
// fires the operator alert.
func TestOffbox_RunFailsFastAndAlerts(t *testing.T) {
	m, sett := newOffboxManager(t)
	m.SetOffboxRunner(func(ctx context.Context, _ []string, args ...string) ([]byte, error) {
		// dead NAS: every op (incl. the repo probe) errors fast.
		return []byte("unable to open repository: connection refused"), context.DeadlineExceeded
	})
	var mu sync.Mutex
	var gotErr error
	var notified bool
	m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { mu.Lock(); defer mu.Unlock(); notified = true; gotErr = err })
	_ = sett.SetAppOffbox("rallly", true)

	start := time.Now()
	err := m.RunOffboxBackup(context.Background())
	if err == nil {
		t.Fatal("a dead NAS must produce a failed run")
	}
	if time.Since(start) > 5*time.Second {
		t.Fatalf("run should fail fast, took %s", time.Since(start))
	}
	if !notified || gotErr == nil {
		t.Fatal("a failed off-box run must alert the operator")
	}
	if st := sett.GetOffboxTarget(); st.LastStatus != "error" || st.LastError == "" {
		t.Fatalf("status must record the failure, got %+v", st)
	}
}

// TestOffbox_RepoIdempotent: when the repo exists (cat config succeeds), ensure must NOT init.
func TestOffbox_RepoIdempotent(t *testing.T) {
	m, sett := newOffboxManager(t)
	var inits int
	m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		switch {
		case contains(args, "cat") && contains(args, "config"):
			return []byte(`{"version":2}`), nil // repo exists
		case contains(args, "init"):
			inits++
			return nil, nil
		}
		return nil, nil
	})
	base, env := m.offboxBaseArgs(sett.GetOffboxTarget())
	if err := m.ensureOffboxRepo(context.Background(), base, env); err != nil {
		t.Fatal(err)
	}
	if inits != 0 {
		t.Fatalf("an existing repo must NOT be re-initialized, init called %d times", inits)
	}
}

// TestOffbox_RestoreRoundTrip: backup a temp tree (fake records src per tag), restore (fake copies the
// recorded tree to target) → byte-identical. Exercises the orchestration without real restic.
func TestOffbox_RestoreRoundTrip(t *testing.T) {
	m, sett := newOffboxManager(t)
	// Lay down an app's recovery-unit tree on disk (what RunOffboxBackup will back up).
	nsRoot := m.AppNamespaceRoot("rallly")
	src := RecoveryUnitPath(nsRoot, "rallly")
	if err := os.MkdirAll(filepath.Join(src, "db-dumps"), 0o755); err != nil {
		t.Fatal(err)
	}
	want := []byte("CREATE TABLE x; -- dump bytes")
	if err := os.WriteFile(filepath.Join(src, "db-dumps", "rallly.sql"), want, 0o644); err != nil {
		t.Fatal(err)
	}
	_ = sett.SetAppOffbox("rallly", true)

	captured := map[string]string{} // tag → src path
	m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		switch {
		case contains(args, "cat") && contains(args, "config"):
			return []byte(`{}`), nil
		case contains(args, "backup"):
			captured[tagOf(args)] = args[len(args)-1] // last arg = src path
			return nil, nil
		case contains(args, "forget"):
			return nil, nil
		case contains(args, "restore"):
			target := valAfter(args, "--target")
			if err := copyTree(captured[tagOf(args)], target); err != nil {
				return nil, err
			}
			return nil, nil
		case contains(args, "snapshots"):
			return []byte(`[{"id":"abc"}]`), nil
		case contains(args, "stats"):
			return []byte(`{"total_size":123}`), nil
		}
		return nil, nil
	})

	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("backup: %v", err)
	}
	dest := t.TempDir()
	if err := m.RestoreOffbox(context.Background(), "rallly", dest); err != nil {
		t.Fatalf("restore: %v", err)
	}
	got, err := os.ReadFile(filepath.Join(dest, "db-dumps", "rallly.sql"))
	if err != nil {
		t.Fatalf("restored file missing: %v", err)
	}
	if string(got) != string(want) {
		t.Fatalf("restore not byte-identical: got %q want %q", got, want)
	}
}

// TestOffbox_SingleFlight: an off-box run while another backup holds m.running skips (no runner call).
func TestOffbox_SingleFlight(t *testing.T) {
	m, _ := newOffboxManager(t)
	called := false
	m.SetOffboxRunner(func(context.Context, []string, ...string) ([]byte, error) { called = true; return nil, nil })
	_ = m.acquireRunning() // simulate a concurrent backup holding the flag
	defer m.releaseRunning()
	if err := m.RunOffboxBackup(context.Background()); err != nil {
		t.Fatalf("single-flight skip should not error, got %v", err)
	}
	if called {
		t.Fatal("off-box must not run (race) while another backup holds the flag")
	}
}

// TestOffbox_SecretsAre0600: the SSH key + repo password files are 0600; the password is non-empty.
func TestOffbox_SecretsAre0600(t *testing.T) {
	m, _ := newOffboxManager(t)
	for _, p := range []string{m.offboxKeyPath(), m.offboxPwPath()} {
		info, err := os.Stat(p)
		if err != nil {
			t.Fatalf("secret file missing: %v", err)
		}
		if runtimeIsUnix() && info.Mode().Perm()&0o077 != 0 {
			t.Errorf("%s is group/other-readable (mode %v) — must be 0600", p, info.Mode().Perm())
		}
	}
	pw, _ := os.ReadFile(m.offboxPwPath())
	if len(strings.TrimSpace(string(pw))) < 32 {
		t.Errorf("repo password too short / empty")
	}
}

// --- tiny test helpers ---

// TestOffbox_ValidateRejectsInjection is the security companion: host/user/repo values that could inject
// an ssh option (leading '-' → e.g. -oProxyCommand) or a shell metacharacter must be REFUSED; a clean
// target is accepted. A build without this guard would let a hostile target reach the ssh exec → FAIL.
func TestOffbox_ValidateRejectsInjection(t *testing.T) {
	ok := &settings.OffboxTarget{Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo"}
	if err := ValidateOffboxTarget(ok); err != nil {
		t.Fatalf("clean target rejected: %v", err)
	}
	bad := []settings.OffboxTarget{
		{Host: "-oProxyCommand=touch /tmp/pwn", User: "felhom", RepoPath: "/srv/repo"}, // ssh option injection
		{Host: "nas;rm -rf /", User: "felhom", RepoPath: "/srv/repo"},                  // metacharacters
		{Host: "nas.local", User: "-oProxyCommand=x", RepoPath: "/srv/repo"},           // user option injection
		{Host: "nas.local", User: "felhom", RepoPath: "/srv/repo; evil"},              // path metacharacters
		{Host: "nas.local", User: "felhom", RepoPath: "/srv/../etc"},                  // traversal
		{Host: "nas local", User: "felhom", RepoPath: "/srv/repo"},                    // space
		{Host: "nas.local", User: "felhom", RepoPath: "relative/path"},                // non-absolute
	}
	for i, b := range bad {
		bb := b
		if err := ValidateOffboxTarget(&bb); err == nil {
			t.Errorf("case %d (%+v) must be rejected", i, bb)
		}
	}
}

func runtimeIsUnix() bool { return os.PathSeparator == '/' }

func contains(ss []string, want string) bool {
	for _, s := range ss {
		if s == want {
			return true
		}
	}
	return false
}

// tagOf returns the LAST --tag value (the per-stack tag; backup adds "felhom-offbox" then the stack).
func tagOf(args []string) string {
	tag := ""
	for i, a := range args {
		if a == "--tag" && i+1 < len(args) {
			tag = args[i+1]
		}
	}
	return tag
}

func valAfter(args []string, flag string) string {
	for i, a := range args {
		if a == flag && i+1 < len(args) {
			return args[i+1]
		}
	}
	return ""
}

func copyTree(src, dst string) error {
	return filepath.Walk(src, func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		rel, _ := filepath.Rel(src, p)
		target := filepath.Join(dst, rel)
		if info.IsDir() {
			return os.MkdirAll(target, 0o755)
		}
		b, rerr := os.ReadFile(p)
		if rerr != nil {
			return rerr
		}
		return os.WriteFile(target, b, 0o644)
	})
}