felhom-controller/controller/internal/backup/offbox.go

package backup

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// Off-box (NAS) backup target — Part B. An ENCRYPTED restic repo reached over SFTP (no kernel mount;
// restic talks SFTP to the NAS directly). This is the "1 off-site" leg of 3-2-1 for the app-data tier
// (each off-box app's recovery unit + DB dumps + volume tars), distinct from the local cross-drive rsync
// copy and from the agent's PBS whole-CT DR. The NAS sees only ciphertext.
//
// THE load-bearing lesson (spike Q8): a dead NAS must FAIL FAST, never hang the backup runner — every
// restic invocation carries `-o sftp.args=…-oConnectTimeout=N…` so a black-holed endpoint errors in
// ~N seconds instead of a multi-minute TCP retry. We also check restic's OWN exit code (never
// pipe-swallow). Secrets (repo password + SSH key) live in 0600 files in the data dir — never logged,
// never in a committed/non-0600 file; they ride DR via the PBS whole-CT snapshot of the rootfs.

const (
	// offboxConnectTimeoutSec is the SSH ConnectTimeout (spike Q8) — load-bearing fail-fast.
	offboxConnectTimeoutSec = 10
	// offboxBackupTimeout bounds a full off-box run; offboxProbeTimeout bounds the quick repo probes.
	offboxBackupTimeout = 2 * time.Hour
	offboxProbeTimeout  = 90 * time.Second
)

// offboxRunner is the restic-exec seam (tests inject a fake so no real restic/ssh runs). It runs restic
// with args + extra env and returns combined output + the process error (whose ExitCode the caller checks).
type offboxRunner func(ctx context.Context, env []string, args ...string) ([]byte, error)

func defaultOffboxRunner(ctx context.Context, env []string, args ...string) ([]byte, error) {
	cmd := exec.CommandContext(ctx, "restic", args...)
	cmd.Env = append(os.Environ(), env...)
	return cmd.CombinedOutput()
}

// SetOffboxRunner overrides the restic exec (tests). SetOffboxNotify wires the failure→operator alert.
func (m *Manager) SetOffboxRunner(r offboxRunner)                              { m.offboxRunner = r }
func (m *Manager) SetOffboxNotify(fn func(dur time.Duration, snapshots int, err error)) { m.offboxNotify = fn }

func (m *Manager) runner() offboxRunner {
	if m.offboxRunner != nil {
		return m.offboxRunner
	}
	return defaultOffboxRunner
}

func (m *Manager) offboxDir() string         { return filepath.Join(m.cfg.Paths.DataDir, "offbox") }
func (m *Manager) offboxKeyPath() string     { return filepath.Join(m.offboxDir(), "ssh_key") }
func (m *Manager) offboxPwPath() string      { return filepath.Join(m.offboxDir(), "repo_password") }
func (m *Manager) offboxKnownHosts() string  { return filepath.Join(m.offboxDir(), "known_hosts") }

// WriteOffboxSecrets persists the SSH private key + (auto-generated if empty) repo password + the pinned
// known-host line as 0600/0644 files in the data dir. The key is provided out-of-band by the operator
// (UI), never logged. Returns the repo password so the caller need not read the file. Idempotent: an empty
// sshKey/knownHosts leaves the existing file untouched (a re-save of just the target shouldn't wipe keys).
func (m *Manager) WriteOffboxSecrets(sshKey, knownHosts string) error {
	if err := os.MkdirAll(m.offboxDir(), 0o700); err != nil {
		return fmt.Errorf("offbox dir: %w", err)
	}
	if strings.TrimSpace(sshKey) != "" {
		key := sshKey
		if !strings.HasSuffix(key, "\n") {
			key += "\n"
		}
		if err := os.WriteFile(m.offboxKeyPath(), []byte(key), 0o600); err != nil {
			return fmt.Errorf("offbox ssh key: %w", err)
		}
	}
	if strings.TrimSpace(knownHosts) != "" {
		kh := knownHosts
		if !strings.HasSuffix(kh, "\n") {
			kh += "\n"
		}
		if err := os.WriteFile(m.offboxKnownHosts(), []byte(kh), 0o644); err != nil {
			return fmt.Errorf("offbox known_hosts: %w", err)
		}
	}
	// Auto-generate the repo password once (0600), never log it.
	if _, err := os.Stat(m.offboxPwPath()); os.IsNotExist(err) {
		pw, gerr := generateOffboxPassword()
		if gerr != nil {
			return gerr
		}
		if werr := os.WriteFile(m.offboxPwPath(), []byte(pw), 0o600); werr != nil {
			return fmt.Errorf("offbox repo password: %w", werr)
		}
	}
	return nil
}

// generateOffboxPassword returns a 256-bit hex repo password.
func generateOffboxPassword() (string, error) {
	b := make([]byte, 32)
	if _, err := rand.Read(b); err != nil {
		return "", fmt.Errorf("offbox password gen: %w", err)
	}
	return hex.EncodeToString(b), nil
}

// Off-box target field validation — the security boundary for the values that flow into the `ssh … -s
// sftp` command restic runs. Host/user are charset-restricted AND must not start with '-' (an ssh
// OPTION-INJECTION vector: a host like "-oProxyCommand=evil" would make ssh execute an arbitrary command).
// RepoPath is an absolute, traversal-free, metacharacter-free path. This mirrors the agent's validate.go
// discipline: validate before any value reaches an exec.
var (
	reOffboxHost = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
	reOffboxUser = regexp.MustCompile(`^[A-Za-z0-9._-]+$`)
	reOffboxPath = regexp.MustCompile(`^/[A-Za-z0-9._/-]+$`)
)

// ValidateOffboxTarget rejects values that could inject into the ssh command line (option injection via a
// leading '-', shell/space metacharacters, path traversal). Returns nil for a safe target.
func ValidateOffboxTarget(t *settings.OffboxTarget) error {
	if t == nil {
		return fmt.Errorf("no off-box target")
	}
	if t.Host == "" || len(t.Host) > 255 || !reOffboxHost.MatchString(t.Host) || strings.HasPrefix(t.Host, "-") || strings.HasPrefix(t.Host, ".") {
		return fmt.Errorf("invalid NAS host (letters, digits, '.', '-', '_'; must not start with '-' or '.')")
	}
	if t.User == "" || len(t.User) > 64 || !reOffboxUser.MatchString(t.User) || strings.HasPrefix(t.User, "-") {
		return fmt.Errorf("invalid user (letters, digits, '.', '-', '_'; must not start with '-')")
	}
	if len(t.RepoPath) > 512 || !reOffboxPath.MatchString(t.RepoPath) || strings.Contains(t.RepoPath, "..") {
		return fmt.Errorf("invalid repo path (absolute, no spaces/metacharacters, no '..')")
	}
	if p := t.Port; p != 0 && (p < 1 || p > 65535) {
		return fmt.Errorf("invalid port")
	}
	return nil
}

// OffboxConfigured reports whether the target is set, enabled, VALID, and the key + password files exist
// (so the UI/scheduler can gate a run without leaking why). A target that fails validation is treated as
// not-configured — fail-closed, so a bad/hostile persisted target can never reach the ssh exec.
func (m *Manager) OffboxConfigured() bool {
	t := m.settings.GetOffboxTarget()
	if t == nil || !t.Enabled || t.Host == "" || t.User == "" || t.RepoPath == "" {
		return false
	}
	if err := ValidateOffboxTarget(t); err != nil {
		return false
	}
	if _, err := os.Stat(m.offboxKeyPath()); err != nil {
		return false
	}
	if _, err := os.Stat(m.offboxPwPath()); err != nil {
		return false
	}
	return true
}

// offboxBaseArgs builds the restic global args (repo + sftp.args carrying the ConnectTimeout, key, pinned
// known_hosts, port) and the env (RESTIC_PASSWORD_FILE). The ConnectTimeout is MANDATORY (fail-fast).
func (m *Manager) offboxBaseArgs(t *settings.OffboxTarget) ([]string, []string) {
	port := t.Port
	if port == 0 {
		port = 22
	}
	// restic's sftp backend connects via the `-o sftp.command` SSH invocation (the portable form across
	// restic versions — `sftp.args` is not recognized by restic 0.14). The ConnectTimeout makes a dead NAS
	// fail in ~N s (the load-bearing spike Q8 knob); StrictHostKeyChecking + a pinned known_hosts avoid
	// blind TOFU; BatchMode prevents any interactive prompt from hanging the runner. The value is one -o
	// token (restic takes everything after `sftp.command=`); our paths have no spaces (data dir).
	sftpCmd := fmt.Sprintf("ssh %s@%s -p %d -oBatchMode=yes -oConnectTimeout=%d -oStrictHostKeyChecking=yes -oUserKnownHostsFile=%s -i %s -s sftp",
		t.User, t.Host, port, offboxConnectTimeoutSec, m.offboxKnownHosts(), m.offboxKeyPath())
	repo := "sftp:" + t.User + "@" + t.Host + ":" + t.RepoPath
	args := []string{"-r", repo, "-o", "sftp.command=" + sftpCmd}
	env := []string{"RESTIC_PASSWORD_FILE=" + m.offboxPwPath()}
	return args, env
}

// ensureOffboxRepo makes sure the SFTP repo exists: probe `cat config`; if absent, `init` (idempotent —
// a present repo is reused, never re-init). A connect failure surfaces here (fast, via ConnectTimeout).
func (m *Manager) ensureOffboxRepo(ctx context.Context, base, env []string) error {
	pctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
	defer cancel()
	if _, err := m.runner()(pctx, env, append(append([]string{}, base...), "cat", "config")...); err == nil {
		return nil // repo exists
	}
	// Repo (probably) absent OR unreachable. Try init; if init succeeds the repo was absent. If init
	// fails because it already exists (a race), treat as success; otherwise the error is real (e.g. a
	// dead NAS — fail fast).
	ictx, icancel := context.WithTimeout(ctx, offboxProbeTimeout)
	defer icancel()
	out, err := m.runner()(ictx, env, append(append([]string{}, base...), "init")...)
	if err == nil {
		m.logger.Printf("[INFO] [offbox] initialized restic repo")
		return nil
	}
	if strings.Contains(string(out), "already initialized") || strings.Contains(string(out), "already exists") {
		return nil
	}
	return fmt.Errorf("offbox repo unreachable / init failed: %w: %s", err, truncate(out))
}

// RunOffboxBackup backs up every off-box-toggled app's recovery unit (recovery unit + DB dumps + volume
// tars) to the SFTP repo, then prunes per the retention policy. Single-flight + migration-guarded. A
// failure (incl. a fail-fast dead-NAS error) records status + alerts the operator. Returns the first error.
func (m *Manager) RunOffboxBackup(ctx context.Context) error {
	if !m.OffboxConfigured() {
		return fmt.Errorf("off-box backup not configured")
	}
	if m.migrationActive() {
		m.logger.Printf("[INFO] [offbox] skipped — migration in progress")
		return nil
	}
	if err := m.acquireRunning(); err != nil {
		m.logger.Printf("[INFO] [offbox] skipped — another backup is running")
		return nil // single-flight: don't race; the next scheduled run retries
	}
	defer m.releaseRunning()

	apps := m.settings.GetOffboxApps()
	t := m.settings.GetOffboxTarget()
	base, env := m.offboxBaseArgs(t)
	start := time.Now()
	_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.LastStatus = "running"; o.LastError = "" })

	backedUp, missing, runErr := m.runOffboxInternal(ctx, apps, base, env)

	// No-silent-success: apps were toggled but NOTHING was captured (every unit missing) → promote to a
	// hard error so the run reports "error" and the operator is alerted, instead of a misleading ok/0.
	if runErr == nil && len(apps) > 0 && backedUp == 0 {
		runErr = fmt.Errorf("off-box backup produced no snapshots: %d app(s) toggled but no recovery unit was found on any connected drive (missing: %s)",
			len(apps), strings.Join(missing, ", "))
	}

	dur := time.Since(start)
	snapshots := 0
	if runErr == nil {
		snapshots = m.offboxRecordStats(ctx, base, env)
	}
	_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
		o.LastRun = time.Now().UTC().Format(time.RFC3339)
		o.LastDuration = dur.Round(time.Second).String()
		if runErr != nil {
			o.LastStatus = "error"
			o.LastError = runErr.Error()
			o.LastWarning = ""
		} else {
			o.LastStatus = "ok"
			o.LastError = ""
			o.SnapshotCount = snapshots
			if len(missing) == 0 {
				o.LastWarning = ""
			} else {
				o.LastWarning = fmt.Sprintf("Figyelmeztetés: %d alkalmazásnak nincs elérhető mentése, ezek kimaradtak: %s",
					len(missing), strings.Join(missing, ", "))
			}
		}
	})
	if m.offboxNotify != nil {
		m.offboxNotify(dur, snapshots, runErr)
	}
	switch {
	case runErr != nil:
		m.logger.Printf("[ERROR] [offbox] backup failed after %s: %v", dur.Round(time.Second), runErr)
	case len(missing) > 0:
		m.logger.Printf("[INFO] [offbox] backup OK: %d app(s) backed up, %d skipped (no unit), %d snapshot(s), %s",
			backedUp, len(missing), snapshots, dur.Round(time.Second))
	default:
		m.logger.Printf("[INFO] [offbox] backup OK: %d app(s) backed up, %d snapshot(s), %s", backedUp, snapshots, dur.Round(time.Second))
	}
	return runErr
}

// offboxCandidateNSRoots is the durable, deployment-state-INDEPENDENT set of felhom-data namespace roots
// to search for a recovery unit: every registered SCHEDULABLE (non-decommissioned) storage path ∪ the
// system-data fallback drive, deduped by resolved nsRoot string. This deliberately does NOT consult
// GetAppDrivePath/AppNamespaceRoot — those read the app's LIVE app.yaml HDD_PATH and silently fall back
// to systemDataPath when the app isn't currently deployed, which made offbox look on the wrong drive
// (DIAG root cause). A disconnected drive's path simply isn't present on disk → os.Stat fails → the unit
// is "not here" (correct: a disconnected drive can't be offsited). Boundary: a decommissioned or
// non-schedulable drive is not searched (not an active managed backup location).
func (m *Manager) offboxCandidateNSRoots() []string {
	seen := map[string]bool{}
	var nsRoots []string
	add := func(nr string) {
		if nr != "" && !seen[nr] {
			seen[nr] = true
			nsRoots = append(nsRoots, nr)
		}
	}
	for _, sp := range m.settings.GetSchedulableStoragePaths() {
		add(m.namespaceRoot(sp.Path))
	}
	if m.systemDataPath != "" {
		add(m.namespaceRoot(m.systemDataPath))
	}
	return nsRoots
}

// discoverOffboxUnit locates an app's recovery unit (backups/primary/<app>) across the candidate nsRoots.
// Returns the src path + true when exactly one exists; when the SAME app's unit exists on more than one
// drive (drive churn / a stale copy left behind), it returns the NEWEST by manifest CreatedAt (falling
// back to the unit dir mtime) and WARNs about the others. Independent of the app's live deploy state.
func (m *Manager) discoverOffboxUnit(app string) (string, bool) {
	var foundSrc, foundManifest []string
	for _, nr := range m.offboxCandidateNSRoots() {
		p := RecoveryUnitPath(nr, app)
		if fi, err := os.Stat(p); err == nil && fi.IsDir() {
			foundSrc = append(foundSrc, p)
			foundManifest = append(foundManifest, RecoveryUnitManifestPath(nr, app))
		}
	}
	switch len(foundSrc) {
	case 0:
		return "", false
	case 1:
		return foundSrc[0], true
	default:
		best := 0
		bestT := offboxUnitTime(foundSrc[0], foundManifest[0])
		for i := 1; i < len(foundSrc); i++ {
			if t := offboxUnitTime(foundSrc[i], foundManifest[i]); t.After(bestT) {
				best, bestT = i, t
			}
		}
		var others []string
		for i, p := range foundSrc {
			if i != best {
				others = append(others, p)
			}
		}
		m.logger.Printf("[WARN] [offbox] %s: multiple recovery units found, using newest (%s); ignoring: %s",
			app, foundSrc[best], strings.Join(others, ", "))
		return foundSrc[best], true
	}
}

// offboxUnitTime returns a recovery unit's timestamp for the multi-copy tiebreak: the manifest's
// CreatedAt (RFC3339) if readable, else the unit dir's mtime (zero if neither is available).
func offboxUnitTime(src, manifestPath string) time.Time {
	if mf := readManifest(manifestPath); mf != nil {
		if t, err := time.Parse(time.RFC3339, mf.CreatedAt); err == nil {
			return t
		}
	}
	if fi, err := os.Stat(src); err == nil {
		return fi.ModTime()
	}
	return time.Time{}
}

// runOffboxInternal does the repo-ensure + per-app DISCOVER-then-backup + prune. Caller holds the running
// flag. Returns how many apps were actually backed up, which toggled apps had no discoverable unit
// (skipped), and the first hard error (repo-ensure or a restic backup exec failure).
func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []string) (backedUp int, missing []string, err error) {
	if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
		return 0, nil, rerr // fail fast (dead NAS surfaces here)
	}
	var firstErr error
	for _, stack := range apps {
		src, ok := m.discoverOffboxUnit(stack)
		if !ok {
			m.logger.Printf("[WARN] [offbox] %s: no recovery unit found on any connected drive — skipping", stack)
			missing = append(missing, stack)
			continue
		}
		bctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
		args := append(append([]string{}, base...), "backup", "--tag", "felhom-offbox", "--tag", stack, src)
		out, berr := m.runner()(bctx, env, args...)
		cancel()
		if berr != nil {
			m.logger.Printf("[ERROR] [offbox] backup %s failed: %v: %s", stack, berr, truncate(out))
			if firstErr == nil {
				firstErr = fmt.Errorf("offbox backup %s: %w", stack, berr)
			}
			continue
		}
		backedUp++
		m.logger.Printf("[INFO] [offbox] backed up %s (%s)", stack, src)
	}
	if firstErr != nil {
		return backedUp, missing, firstErr
	}
	// Retention: keep a sane window, prune the rest. Repo-wide (grouped by host+paths by default).
	fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
	defer cancel()
	fargs := append(append([]string{}, base...), "forget", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
	if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
		// A prune failure is non-fatal to the backup itself (data is safe) — log, don't fail the run.
		m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", ferr, truncate(out))
	}
	return backedUp, missing, nil
}

// offboxRecordStats reads the snapshot count (best-effort) for the UI; also fills repo size when stats works.
func (m *Manager) offboxRecordStats(ctx context.Context, base, env []string) int {
	sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
	defer cancel()
	out, err := m.runner()(sctx, env, append(append([]string{}, base...), "snapshots", "--json")...)
	if err != nil {
		return 0
	}
	var snaps []struct {
		ID string `json:"id"`
	}
	if json.Unmarshal(out, &snaps) != nil {
		return 0
	}
	// Repo size (best-effort, restore-size).
	if so, serr := m.runner()(sctx, env, append(append([]string{}, base...), "stats", "--json")...); serr == nil {
		var st struct {
			TotalSize int64 `json:"total_size"`
		}
		if json.Unmarshal(so, &st) == nil && st.TotalSize > 0 {
			human := humanizeBytes(st.TotalSize)
			_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.RepoSizeHuman = human })
		}
	}
	return len(snaps)
}

// RestoreOffbox restores an app's latest off-box snapshot to destDir (a scratch/verify location — it does
// NOT overwrite live data). Returns an error on failure (checks restic's own exit code).
func (m *Manager) RestoreOffbox(ctx context.Context, stackName, destDir string) error {
	if !m.OffboxConfigured() {
		return fmt.Errorf("off-box backup not configured")
	}
	if !isSafeStackName(stackName) {
		return fmt.Errorf("invalid stack name")
	}
	if err := os.MkdirAll(destDir, 0o755); err != nil {
		return fmt.Errorf("restore dir: %w", err)
	}
	t := m.settings.GetOffboxTarget()
	base, env := m.offboxBaseArgs(t)
	rctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
	defer cancel()
	args := append(append([]string{}, base...), "restore", "latest", "--tag", stackName, "--target", destDir)
	out, err := m.runner()(rctx, env, args...)
	if err != nil {
		return fmt.Errorf("offbox restore %s: %w: %s", stackName, err, truncate(out))
	}
	m.logger.Printf("[INFO] [offbox] restored %s → %s", stackName, destDir)
	return nil
}

// isSafeStackName guards a stack name used as a restic tag / path component.
func isSafeStackName(s string) bool {
	if s == "" || len(s) > 64 {
		return false
	}
	for _, c := range s {
		if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' {
			continue
		}
		return false
	}
	return true
}

// truncate caps subprocess output for a log line + strips a trailing newline.
func truncate(b []byte) string {
	s := strings.TrimSpace(string(b))
	if len(s) > 400 {
		return s[:400] + "…"
	}
	return s
}