admin
ddac21cd91
The QuotaGB hash change triggered a live re-apply that demoted the escrowed demo to pending and wiped its runtime status. ApplyOffsiteTarget now carries over EscrowState (custody tracks the preserved repo password, not the coords) + status fields; fresh guests still land pending. Red-proofed. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
905 lines
34 KiB
Go
905 lines
34 KiB
Go
package backup
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||
)
|
||
|
||
// newOffboxManager builds a Manager with a temp data dir + a configured + enabled off-box target and the
|
||
// 0600 secret files written, so OffboxConfigured() is true.
|
||
func newOffboxManager(t *testing.T) (*Manager, *settings.Settings) {
|
||
t.Helper()
|
||
logger := log.New(os.Stderr, "", 0)
|
||
dataDir := t.TempDir()
|
||
sett, err := settings.Load(filepath.Join(dataDir, "settings.json"), logger)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cfg := &config.Config{}
|
||
cfg.Paths.DataDir = dataDir
|
||
cfg.Paths.SystemDataPath = filepath.Join(dataDir, "sys")
|
||
m := NewManager(cfg, sett, logger)
|
||
if err := sett.SetOffboxTarget(&settings.OffboxTarget{
|
||
Enabled: true, Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo", Schedule: "daily",
|
||
EscrowState: "escrowed", // fork-4: default the harness to escrowed so behavioral run tests exercise the run path
|
||
}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := m.WriteOffboxSecrets("PRIVATE-KEY-MATERIAL", "nas.local ssh-ed25519 AAAAhostkey"); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return m, sett
|
||
}
|
||
|
||
// argsContainTimeout reports whether the restic arg vector carries the load-bearing ConnectTimeout.
|
||
func argsContainTimeout(args []string) bool {
|
||
return strings.Contains(strings.Join(args, " "), "-oConnectTimeout=")
|
||
}
|
||
|
||
// PINNED CROSS-REPO TEST VECTOR (SLICE 3): the same vector is asserted in felhom-agent's
|
||
// escrow.HashResticPassword test — if either hasher drifts (newline, encoding, trim), its half fails and
|
||
// the escrow auto-confirm can never silently mismatch. Convention: sha256 hex over the TRIMMED string.
|
||
func TestHashResticPassword_PinnedVector(t *testing.T) {
|
||
const vector = "cafef00ddeadbeef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||
const want = "dbfc02f987e1ac0c91911d5761267089b1144628745a3343e4d96194e43c08e4"
|
||
if got := HashResticPassword(vector); got != want {
|
||
t.Fatalf("pinned vector drift: got %s want %s", got, want)
|
||
}
|
||
if got := HashResticPassword(" " + vector + "\n"); got != want {
|
||
t.Fatalf("whitespace must not change the hash (trim convention), got %s", got)
|
||
}
|
||
}
|
||
|
||
// OffboxRepoPasswordHash: hashes the on-disk password file (the auto-confirm's local side); absent → ok=false.
|
||
func TestOffboxRepoPasswordHash(t *testing.T) {
|
||
// bare manager (no secrets written yet) → no password file → ok=false
|
||
logger := log.New(os.Stderr, "", 0)
|
||
dataDir := t.TempDir()
|
||
sett, err := settings.Load(filepath.Join(dataDir, "settings.json"), logger)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cfg := &config.Config{}
|
||
cfg.Paths.DataDir = dataDir
|
||
m := NewManager(cfg, sett, logger)
|
||
if _, ok := m.OffboxRepoPasswordHash(); ok {
|
||
t.Fatal("no password file → ok must be false")
|
||
}
|
||
const pw = "cafef00ddeadbeef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||
if err := m.InjectOffboxPassword(pw, false); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
got, ok := m.OffboxRepoPasswordHash()
|
||
if !ok || got != "dbfc02f987e1ac0c91911d5761267089b1144628745a3343e4d96194e43c08e4" {
|
||
t.Fatalf("hash of the injected password wrong: ok=%v got=%s", ok, got)
|
||
}
|
||
}
|
||
|
||
// TestOffbox_BaseArgsCarryConnectTimeout asserts the mandatory fail-fast + hardening args are present.
|
||
func TestOffbox_BaseArgsCarryConnectTimeout(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
base, env := m.offboxBaseArgs(sett.GetOffboxTarget())
|
||
joined := strings.Join(base, " ")
|
||
for _, want := range []string{
|
||
"-oConnectTimeout=10", // THE spike Q8 fail-fast knob
|
||
"-oStrictHostKeyChecking=yes", // no blind TOFU
|
||
"-oUserKnownHostsFile=", // pinned host key
|
||
"-oBatchMode=yes", // no interactive hang
|
||
"sftp:felhom@nas.local:/srv/repo",
|
||
} {
|
||
if !strings.Contains(joined, want) {
|
||
t.Errorf("base args missing %q: %s", want, joined)
|
||
}
|
||
}
|
||
if len(env) != 1 || !strings.HasPrefix(env[0], "RESTIC_PASSWORD_FILE=") {
|
||
t.Errorf("env must set RESTIC_PASSWORD_FILE only, got %v", env)
|
||
}
|
||
}
|
||
|
||
// failFastFake models the SSH transport's ConnectTimeout honoring: if the restic args carry
|
||
// -oConnectTimeout it returns a connect error promptly (fail-fast); WITHOUT it, it blocks until the ctx
|
||
// deadline (the dead-NAS multi-minute hang). This is the seam the ConnectTimeout companion exercises.
|
||
func failFastFake(_ *testing.T) offboxRunner {
|
||
return func(ctx context.Context, _ []string, args ...string) ([]byte, error) {
|
||
if argsContainTimeout(args) {
|
||
return []byte("dial tcp: connect: connection refused"), context.DeadlineExceeded // fast, bounded
|
||
}
|
||
<-ctx.Done() // no timeout arg → hang until the caller's deadline (the bug)
|
||
return nil, ctx.Err()
|
||
}
|
||
}
|
||
|
||
// TestOffbox_ConnectTimeoutIsLoadBearing is the §10 companion red-proof: WITH the arg the (fake) connect
|
||
// fails fast (well under the bound); WITHOUT it the connect blocks past the bound. A build that dropped
|
||
// the ConnectTimeout arg would take the slow path → this proves the arg is load-bearing.
|
||
func TestOffbox_ConnectTimeoutIsLoadBearing(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
realArgs, env := m.offboxBaseArgs(sett.GetOffboxTarget())
|
||
fake := failFastFake(t)
|
||
|
||
// WITH the arg: returns promptly (we model fast as an immediate error, not a ctx hang).
|
||
ctx1, c1 := context.WithTimeout(context.Background(), 2*time.Second)
|
||
defer c1()
|
||
start := time.Now()
|
||
_, err := fake(ctx1, env, append(append([]string{}, realArgs...), "cat", "config")...)
|
||
if elapsed := time.Since(start); elapsed > time.Second {
|
||
t.Fatalf("with ConnectTimeout the connect must fail fast, took %s", elapsed)
|
||
}
|
||
_ = err
|
||
|
||
// WITHOUT the arg (the bug): blocks until the ctx deadline.
|
||
stripped := stripConnectTimeout(realArgs)
|
||
if argsContainTimeout(stripped) {
|
||
t.Fatal("test setup: stripped args still contain the timeout")
|
||
}
|
||
ctx2, c2 := context.WithTimeout(context.Background(), 300*time.Millisecond)
|
||
defer c2()
|
||
start = time.Now()
|
||
_, err = fake(ctx2, env, append(append([]string{}, stripped...), "cat", "config")...)
|
||
if err != context.DeadlineExceeded {
|
||
t.Fatalf("without ConnectTimeout the connect should block to the deadline, got %v", err)
|
||
}
|
||
if elapsed := time.Since(start); elapsed < 250*time.Millisecond {
|
||
t.Fatalf("without ConnectTimeout it should have hung to the bound, only took %s", elapsed)
|
||
}
|
||
}
|
||
|
||
func stripConnectTimeout(args []string) []string {
|
||
out := make([]string, len(args))
|
||
for i, a := range args {
|
||
out[i] = strings.ReplaceAll(a, "-oConnectTimeout=10 ", "")
|
||
}
|
||
return out
|
||
}
|
||
|
||
// TestOffbox_RunFailsFastAndAlerts: a dead-NAS run returns an error promptly, records status=error, and
|
||
// fires the operator alert.
|
||
func TestOffbox_RunFailsFastAndAlerts(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
m.SetOffboxRunner(func(ctx context.Context, _ []string, args ...string) ([]byte, error) {
|
||
// dead NAS: every op (incl. the repo probe) errors fast.
|
||
return []byte("unable to open repository: connection refused"), context.DeadlineExceeded
|
||
})
|
||
var mu sync.Mutex
|
||
var gotErr error
|
||
var notified bool
|
||
m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { mu.Lock(); defer mu.Unlock(); notified = true; gotErr = err })
|
||
_ = sett.SetAppOffbox("rallly", true)
|
||
|
||
start := time.Now()
|
||
err := m.RunOffboxBackup(context.Background())
|
||
if err == nil {
|
||
t.Fatal("a dead NAS must produce a failed run")
|
||
}
|
||
if time.Since(start) > 5*time.Second {
|
||
t.Fatalf("run should fail fast, took %s", time.Since(start))
|
||
}
|
||
if !notified || gotErr == nil {
|
||
t.Fatal("a failed off-box run must alert the operator")
|
||
}
|
||
if st := sett.GetOffboxTarget(); st.LastStatus != "error" || st.LastError == "" {
|
||
t.Fatalf("status must record the failure, got %+v", st)
|
||
}
|
||
}
|
||
|
||
// TestOffbox_RepoIdempotent: when the repo exists (cat config succeeds), ensure must NOT init.
|
||
func TestOffbox_RepoIdempotent(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
var inits int
|
||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||
switch {
|
||
case contains(args, "cat") && contains(args, "config"):
|
||
return []byte(`{"version":2}`), nil // repo exists
|
||
case contains(args, "init"):
|
||
inits++
|
||
return nil, nil
|
||
}
|
||
return nil, nil
|
||
})
|
||
base, env := m.offboxBaseArgs(sett.GetOffboxTarget())
|
||
if err := m.ensureOffboxRepo(context.Background(), base, env); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if inits != 0 {
|
||
t.Fatalf("an existing repo must NOT be re-initialized, init called %d times", inits)
|
||
}
|
||
}
|
||
|
||
// TestOffbox_RestoreRoundTrip: backup a temp tree (fake records src per tag), restore (fake copies the
|
||
// recorded tree to target) → byte-identical. Exercises the orchestration without real restic.
|
||
func TestOffbox_RestoreRoundTrip(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
// Lay down an app's recovery-unit tree on disk (what RunOffboxBackup will back up).
|
||
nsRoot := m.AppNamespaceRoot("rallly")
|
||
src := RecoveryUnitPath(nsRoot, "rallly")
|
||
if err := os.MkdirAll(filepath.Join(src, "db-dumps"), 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
want := []byte("CREATE TABLE x; -- dump bytes")
|
||
if err := os.WriteFile(filepath.Join(src, "db-dumps", "rallly.sql"), want, 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
_ = sett.SetAppOffbox("rallly", true)
|
||
|
||
captured := map[string]string{} // tag → src path
|
||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||
switch {
|
||
case contains(args, "cat") && contains(args, "config"):
|
||
return []byte(`{}`), nil
|
||
case contains(args, "backup"):
|
||
captured[tagOf(args)] = args[len(args)-1] // last arg = src path
|
||
return nil, nil
|
||
case contains(args, "forget"):
|
||
return nil, nil
|
||
case contains(args, "restore"):
|
||
target := valAfter(args, "--target")
|
||
if err := copyTree(captured[tagOf(args)], target); err != nil {
|
||
return nil, err
|
||
}
|
||
return nil, nil
|
||
case contains(args, "snapshots"):
|
||
return []byte(`[{"id":"abc"}]`), nil
|
||
case contains(args, "stats"):
|
||
return []byte(`{"total_size":123}`), nil
|
||
}
|
||
return nil, nil
|
||
})
|
||
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("backup: %v", err)
|
||
}
|
||
dest := t.TempDir()
|
||
if err := m.RestoreOffbox(context.Background(), "rallly", dest); err != nil {
|
||
t.Fatalf("restore: %v", err)
|
||
}
|
||
got, err := os.ReadFile(filepath.Join(dest, "db-dumps", "rallly.sql"))
|
||
if err != nil {
|
||
t.Fatalf("restored file missing: %v", err)
|
||
}
|
||
if string(got) != string(want) {
|
||
t.Fatalf("restore not byte-identical: got %q want %q", got, want)
|
||
}
|
||
}
|
||
|
||
// Scenario A (SLICE 4) — over the soft quota: the BACKUP step is refused (Hungarian error + status),
|
||
// but the retention/prune step STILL RUNS (pruning is the customer's only way back under quota) and
|
||
// restore is untouched. The prune-still-runs assert is the red-proofed core.
|
||
func TestOffbox_QuotaRefusesBackupButPrunes(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
|
||
o.QuotaGB = 50
|
||
o.RepoSizeBytes = 51 << 30 // 51 GiB — over the 50 GiB soft quota
|
||
})
|
||
var backups, forgets, restores int
|
||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||
switch {
|
||
case contains(args, "cat") && contains(args, "config"):
|
||
return []byte(`{}`), nil
|
||
case contains(args, "backup"):
|
||
backups++
|
||
case contains(args, "forget"):
|
||
forgets++
|
||
case contains(args, "restore"):
|
||
restores++
|
||
case contains(args, "stats"):
|
||
return []byte(`{"total_size":123}`), nil
|
||
case contains(args, "snapshots"):
|
||
return []byte(`[]`), nil
|
||
}
|
||
return nil, nil
|
||
})
|
||
if err := m.RunOffboxBackup(context.Background()); err == nil {
|
||
t.Fatal("an over-quota run must report the refusal as an error")
|
||
}
|
||
if backups != 0 {
|
||
t.Fatalf("NEW backups must be refused over quota, got %d backup call(s)", backups)
|
||
}
|
||
if forgets != 1 {
|
||
t.Fatalf("prune MUST still run over quota (the only way back down — gating it deadlocks the customer), got %d", forgets)
|
||
}
|
||
tgt := sett.GetOffboxTarget()
|
||
if tgt.LastStatus != "error" || !strings.Contains(tgt.LastError, "túllépte a tárhelykeretet") ||
|
||
!strings.Contains(tgt.LastError, "51/50") {
|
||
t.Fatalf("Hungarian over-quota status wrong: status=%q err=%q", tgt.LastStatus, tgt.LastError)
|
||
}
|
||
// restore is NEVER quota-gated: it must reach the runner even over quota
|
||
_ = m.RestoreOffbox(context.Background(), "rallly", t.TempDir())
|
||
if restores != 1 {
|
||
t.Fatal("restore must NOT be quota-gated")
|
||
}
|
||
}
|
||
|
||
// Scenario B (SLICE 4) — approaching the quota (≥80%, <100%): the run proceeds OK and the Hungarian
|
||
// usage warning is set (visible on /backups).
|
||
func TestOffbox_QuotaWarnsAt80Percent(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.QuotaGB = 50 })
|
||
var backups int
|
||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||
switch {
|
||
case contains(args, "cat") && contains(args, "config"):
|
||
return []byte(`{}`), nil
|
||
case contains(args, "backup"):
|
||
backups++
|
||
case contains(args, "snapshots"):
|
||
return []byte(`[{"id":"s1"}]`), nil
|
||
case contains(args, "stats"):
|
||
return []byte(fmt.Sprintf(`{"total_size":%d}`, int64(42)<<30)), nil // 42 GiB of 50 = 84%
|
||
}
|
||
return nil, nil
|
||
})
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("an 84%% run must proceed, got %v", err)
|
||
}
|
||
tgt := sett.GetOffboxTarget()
|
||
if tgt.LastStatus != "ok" {
|
||
t.Fatalf("status must be ok at 84%%, got %q (%q)", tgt.LastStatus, tgt.LastError)
|
||
}
|
||
if !strings.Contains(tgt.LastWarning, "84%-át használja") || !strings.Contains(tgt.LastWarning, "42/50") {
|
||
t.Fatalf("the 80%%+ usage warning must be set, got %q", tgt.LastWarning)
|
||
}
|
||
if tgt.RepoSizeBytes != int64(42)<<30 {
|
||
t.Fatalf("RepoSizeBytes must persist from stats, got %d", tgt.RepoSizeBytes)
|
||
}
|
||
}
|
||
|
||
// Scenario C (SLICE 4) — quota 0 (dedicated/unset): no soft gate regardless of size.
|
||
func TestOffbox_QuotaZeroMeansNoGate(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
|
||
o.QuotaGB = 0
|
||
o.RepoSizeBytes = 900 << 30 // enormous — must not matter
|
||
})
|
||
var backups int
|
||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||
switch {
|
||
case contains(args, "cat") && contains(args, "config"):
|
||
return []byte(`{}`), nil
|
||
case contains(args, "backup"):
|
||
backups++
|
||
case contains(args, "snapshots"):
|
||
return []byte(`[]`), nil
|
||
case contains(args, "stats"):
|
||
return []byte(`{"total_size":123}`), nil
|
||
}
|
||
return nil, nil
|
||
})
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("quota-0 run must proceed, got %v", err)
|
||
}
|
||
if tgt := sett.GetOffboxTarget(); strings.Contains(tgt.LastWarning, "keret") || strings.Contains(tgt.LastError, "keret") {
|
||
t.Fatalf("quota 0 must produce no quota warning/refusal: warn=%q err=%q", tgt.LastWarning, tgt.LastError)
|
||
}
|
||
}
|
||
|
||
// SLICE 4 — the report carries the non-secret offsite object when enabled; nil when unconfigured
|
||
// (absent on the wire via omitempty → the hub checker skips, no false staleness on old/plain boxes).
|
||
func TestOffboxReportStatus(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
|
||
o.QuotaGB = 50
|
||
o.RepoSizeBytes = 45 << 30
|
||
o.LastStatus = "ok"
|
||
o.LastRun = "2026-07-09T20:00:00Z"
|
||
o.SnapshotCount = 7
|
||
})
|
||
st := m.OffboxReportStatus()
|
||
if st == nil || !st.Enabled || st.EscrowState != "escrowed" || st.QuotaGB != 50 ||
|
||
st.RepoSizeBytes != int64(45)<<30 || st.LastStatus != "ok" || st.SnapshotCount != 7 {
|
||
t.Fatalf("offsite report status wrong: %+v", st)
|
||
}
|
||
// unconfigured manager → nil
|
||
logger := log.New(os.Stderr, "", 0)
|
||
dataDir := t.TempDir()
|
||
sett2, err := settings.Load(filepath.Join(dataDir, "settings.json"), logger)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cfg := &config.Config{}
|
||
cfg.Paths.DataDir = dataDir
|
||
if got := NewManager(cfg, sett2, logger).OffboxReportStatus(); got != nil {
|
||
t.Fatalf("unconfigured offbox must report nil (absent on the wire), got %+v", got)
|
||
}
|
||
}
|
||
|
||
// v0.109.1 live finding — a RE-apply (quota bump / re-pin) must PRESERVE the existing target's escrow
|
||
// custody + runtime status (the repo password is preserved, so its escrow state carries over); a FRESH
|
||
// apply still lands pending.
|
||
func TestApplyOffsiteTarget_PreservesEscrowAndStatusOnReapply(t *testing.T) {
|
||
m, sett := newOffboxManager(t) // existing target: escrowed
|
||
_ = sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
|
||
o.LastRun = "2026-07-09T20:00:00Z"
|
||
o.LastStatus = "ok"
|
||
o.SnapshotCount = 5
|
||
o.RepoSizeBytes = 2 << 30
|
||
})
|
||
// the bridge re-applies with a freshly-built target (quota raised to 50)
|
||
fresh := &settings.OffboxTarget{Enabled: true, Host: "nas.local", User: "felhom", Port: 22, RepoPath: "/srv/repo", Schedule: "daily", QuotaGB: 50}
|
||
if err := m.ApplyOffsiteTarget(context.Background(), fresh, "KEY2", "nas.local ssh-ed25519 K2", nil); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
got := sett.GetOffboxTarget()
|
||
if got.EscrowState != "escrowed" {
|
||
t.Fatalf("a re-apply must NOT demote an escrowed target, got %q", got.EscrowState)
|
||
}
|
||
if got.QuotaGB != 50 {
|
||
t.Fatalf("the new quota must land, got %d", got.QuotaGB)
|
||
}
|
||
if got.LastRun != "2026-07-09T20:00:00Z" || got.SnapshotCount != 5 || got.RepoSizeBytes != 2<<30 {
|
||
t.Fatalf("runtime status must survive a re-apply (staleness/usage feed): %+v", got)
|
||
}
|
||
|
||
// fresh guest (no existing target) → pending, as before
|
||
logger := log.New(os.Stderr, "", 0)
|
||
dataDir := t.TempDir()
|
||
sett2, err := settings.Load(filepath.Join(dataDir, "settings.json"), logger)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cfg := &config.Config{}
|
||
cfg.Paths.DataDir = dataDir
|
||
m2 := NewManager(cfg, sett2, logger)
|
||
if err := m2.ApplyOffsiteTarget(context.Background(), &settings.OffboxTarget{Enabled: true, Host: "h", User: "u", Port: 23, RepoPath: "/r", Schedule: "daily"}, "K", "kh", nil); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if got2 := sett2.GetOffboxTarget(); got2.EscrowState != "pending" {
|
||
t.Fatalf("a fresh apply must land pending, got %q", got2.EscrowState)
|
||
}
|
||
}
|
||
|
||
// TestOffbox_SingleFlight: an off-box run while another backup holds m.running skips (no runner call).
|
||
func TestOffbox_SingleFlight(t *testing.T) {
|
||
m, _ := newOffboxManager(t)
|
||
called := false
|
||
m.SetOffboxRunner(func(context.Context, []string, ...string) ([]byte, error) { called = true; return nil, nil })
|
||
_ = m.acquireRunning() // simulate a concurrent backup holding the flag
|
||
defer m.releaseRunning()
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("single-flight skip should not error, got %v", err)
|
||
}
|
||
if called {
|
||
t.Fatal("off-box must not run (race) while another backup holds the flag")
|
||
}
|
||
}
|
||
|
||
// TestOffbox_SecretsAre0600: the SSH key + repo password files are 0600; the password is non-empty.
|
||
func TestOffbox_SecretsAre0600(t *testing.T) {
|
||
m, _ := newOffboxManager(t)
|
||
for _, p := range []string{m.offboxKeyPath(), m.offboxPwPath()} {
|
||
info, err := os.Stat(p)
|
||
if err != nil {
|
||
t.Fatalf("secret file missing: %v", err)
|
||
}
|
||
if runtimeIsUnix() && info.Mode().Perm()&0o077 != 0 {
|
||
t.Errorf("%s is group/other-readable (mode %v) — must be 0600", p, info.Mode().Perm())
|
||
}
|
||
}
|
||
pw, _ := os.ReadFile(m.offboxPwPath())
|
||
if len(strings.TrimSpace(string(pw))) < 32 {
|
||
t.Errorf("repo password too short / empty")
|
||
}
|
||
}
|
||
|
||
// --- tiny test helpers ---
|
||
|
||
// TestOffbox_ValidateRejectsInjection is the security companion: host/user/repo values that could inject
|
||
// an ssh option (leading '-' → e.g. -oProxyCommand) or a shell metacharacter must be REFUSED; a clean
|
||
// target is accepted. A build without this guard would let a hostile target reach the ssh exec → FAIL.
|
||
func TestOffbox_ValidateRejectsInjection(t *testing.T) {
|
||
ok := &settings.OffboxTarget{Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo"}
|
||
if err := ValidateOffboxTarget(ok); err != nil {
|
||
t.Fatalf("clean target rejected: %v", err)
|
||
}
|
||
bad := []settings.OffboxTarget{
|
||
{Host: "-oProxyCommand=touch /tmp/pwn", User: "felhom", RepoPath: "/srv/repo"}, // ssh option injection
|
||
{Host: "nas;rm -rf /", User: "felhom", RepoPath: "/srv/repo"}, // metacharacters
|
||
{Host: "nas.local", User: "-oProxyCommand=x", RepoPath: "/srv/repo"}, // user option injection
|
||
{Host: "nas.local", User: "felhom", RepoPath: "/srv/repo; evil"}, // path metacharacters
|
||
{Host: "nas.local", User: "felhom", RepoPath: "/srv/../etc"}, // traversal
|
||
{Host: "nas local", User: "felhom", RepoPath: "/srv/repo"}, // space
|
||
{Host: "nas.local", User: "felhom", RepoPath: "relative/path"}, // non-absolute
|
||
}
|
||
for i, b := range bad {
|
||
bb := b
|
||
if err := ValidateOffboxTarget(&bb); err == nil {
|
||
t.Errorf("case %d (%+v) must be rejected", i, bb)
|
||
}
|
||
}
|
||
}
|
||
|
||
// --- Off-box unit DISCOVERY + no-silent-success (§7 A–E) ---
|
||
|
||
// recordingOffboxRunner captures the exact `src` path of each restic `backup` call and returns success
|
||
// for the repo probe/init/snapshots/stats/forget. Per-stack `backup` errors are configurable.
|
||
type recordingOffboxRunner struct {
|
||
backupSrc []string // src path per backup call, in order (the load-bearing effect to assert)
|
||
backupErr map[string]error // stack (last --tag) → error to return from `backup`
|
||
}
|
||
|
||
func (rr *recordingOffboxRunner) run(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||
switch {
|
||
case contains(args, "cat") && contains(args, "config"):
|
||
return []byte(`{"version":2}`), nil // repo exists (no init)
|
||
case contains(args, "backup"):
|
||
rr.backupSrc = append(rr.backupSrc, args[len(args)-1]) // last arg = src path
|
||
if e := rr.backupErr[tagOf(args)]; e != nil {
|
||
return []byte("restic backup failed"), e
|
||
}
|
||
return nil, nil
|
||
case contains(args, "forget"):
|
||
return nil, nil
|
||
case contains(args, "snapshots"):
|
||
return []byte(`[{"id":"s1"}]`), nil
|
||
case contains(args, "stats"):
|
||
return []byte(`{"total_size":123}`), nil
|
||
}
|
||
return nil, nil
|
||
}
|
||
|
||
// addSchedulablePath registers a schedulable (non-decommissioned) storage path — a candidate drive.
|
||
func addSchedulablePath(t *testing.T, sett *settings.Settings, p string) {
|
||
t.Helper()
|
||
if err := sett.AddStoragePath(settings.StoragePath{Path: p, Schedulable: true, AddedAt: "2026-07-01T00:00:00Z"}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
// writeUnit lays a recovery unit (dir + a manifest with the given CreatedAt) at <nsRoot>/backups/primary/<app>.
|
||
func writeUnit(t *testing.T, nsRoot, app, createdAt string) {
|
||
t.Helper()
|
||
if err := os.MkdirAll(RecoveryUnitPath(nsRoot, app), 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
b, _ := json.Marshal(RecoveryManifest{AppName: app, CreatedAt: createdAt})
|
||
if err := os.WriteFile(RecoveryUnitManifestPath(nsRoot, app), b, 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
// A — undeployed-but-toggled app whose unit lives on a REGISTERED drive (not systemDataPath): discovery
|
||
// must find it there. The harness has no stackProvider, so the OLD AppNamespaceRoot path would resolve to
|
||
// systemDataPath (where no unit is) — the F1 bug. See the companion red-proof in REPORT.
|
||
func TestOffbox_DiscoversUnitOnRegisteredDrive(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
usb := t.TempDir()
|
||
addSchedulablePath(t, sett, usb)
|
||
usbNS := m.namespaceRoot(usb) // registered drive → in-guest → nsRoot == usb
|
||
writeUnit(t, usbNS, "audiobookshelf", "2026-07-01T00:00:00Z")
|
||
// prove the OLD resolution would have looked elsewhere (no unit at systemDataPath nsRoot)
|
||
if _, err := os.Stat(RecoveryUnitPath(m.namespaceRoot(m.systemDataPath), "audiobookshelf")); err == nil {
|
||
t.Fatal("setup: unit must NOT exist on systemDataPath for this repro")
|
||
}
|
||
_ = sett.SetAppOffbox("audiobookshelf", true)
|
||
|
||
rr := &recordingOffboxRunner{}
|
||
m.SetOffboxRunner(rr.run)
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("run: %v", err)
|
||
}
|
||
wantSrc := RecoveryUnitPath(usbNS, "audiobookshelf")
|
||
if len(rr.backupSrc) != 1 || rr.backupSrc[0] != wantSrc {
|
||
t.Fatalf("backup src = %v, want exactly [%s] (discovered on the registered drive)", rr.backupSrc, wantSrc)
|
||
}
|
||
if st := sett.GetOffboxTarget(); st.LastStatus != "ok" || st.LastWarning != "" {
|
||
t.Fatalf("status = %+v, want ok / no warning", st)
|
||
}
|
||
}
|
||
|
||
// B — a toggled app with NO recovery unit anywhere: the run must be a HARD ERROR (no silent ok/0), alert
|
||
// the operator, and record status=error naming the app. Companion red-proof in REPORT.
|
||
func TestOffbox_NoUnitAnywhereIsHardError(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
_ = sett.SetAppOffbox("ghost", true) // toggled; no unit created anywhere
|
||
rr := &recordingOffboxRunner{}
|
||
m.SetOffboxRunner(rr.run)
|
||
var notified bool
|
||
var notifiedErr error
|
||
m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { notified = true; notifiedErr = err })
|
||
|
||
err := m.RunOffboxBackup(context.Background())
|
||
if err == nil {
|
||
t.Fatal("0-of-N toggled apps backed up must ERROR (no silent success)")
|
||
}
|
||
if len(rr.backupSrc) != 0 {
|
||
t.Fatalf("no backup should have run, got %v", rr.backupSrc)
|
||
}
|
||
if !notified || notifiedErr == nil {
|
||
t.Fatal("the 0/N run must alert the operator with a non-nil err")
|
||
}
|
||
st := sett.GetOffboxTarget()
|
||
if st.LastStatus != "error" || st.LastError == "" {
|
||
t.Fatalf("status must be error, got %+v", st)
|
||
}
|
||
if !strings.Contains(st.LastError, "ghost") {
|
||
t.Fatalf("LastError should name the missing app, got %q", st.LastError)
|
||
}
|
||
}
|
||
|
||
// C — partial: one toggled app has a unit, another doesn't → the present one is backed up, status stays
|
||
// ok, notify err is nil, but LastWarning (Hungarian) names the missing app.
|
||
func TestOffbox_PartialRunWarnsNotErrors(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
usb := t.TempDir()
|
||
addSchedulablePath(t, sett, usb)
|
||
usbNS := m.namespaceRoot(usb)
|
||
writeUnit(t, usbNS, "present", "2026-07-01T00:00:00Z")
|
||
_ = sett.SetAppOffbox("present", true)
|
||
_ = sett.SetAppOffbox("gone", true) // no unit
|
||
rr := &recordingOffboxRunner{}
|
||
m.SetOffboxRunner(rr.run)
|
||
notifiedErr := errors.New("sentinel") // must be cleared to nil by a non-erroring run
|
||
m.SetOffboxNotify(func(_ time.Duration, _ int, err error) { notifiedErr = err })
|
||
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("a partial run must NOT error, got %v", err)
|
||
}
|
||
if len(rr.backupSrc) != 1 || rr.backupSrc[0] != RecoveryUnitPath(usbNS, "present") {
|
||
t.Fatalf("only 'present' should be backed up, got %v", rr.backupSrc)
|
||
}
|
||
if notifiedErr != nil {
|
||
t.Fatalf("partial run must notify with nil err, got %v", notifiedErr)
|
||
}
|
||
st := sett.GetOffboxTarget()
|
||
if st.LastStatus != "ok" {
|
||
t.Fatalf("status = %q, want ok", st.LastStatus)
|
||
}
|
||
if !strings.Contains(st.LastWarning, "gone") {
|
||
t.Fatalf("LastWarning must name the missing app 'gone', got %q", st.LastWarning)
|
||
}
|
||
}
|
||
|
||
// D — the same app's unit on TWO registered drives (drive churn): exactly ONE backup call, for the NEWER
|
||
// unit (by manifest CreatedAt).
|
||
func TestOffbox_MultipleUnitsPicksNewest(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
older, newer := t.TempDir(), t.TempDir()
|
||
addSchedulablePath(t, sett, older)
|
||
addSchedulablePath(t, sett, newer)
|
||
olderNS, newerNS := m.namespaceRoot(older), m.namespaceRoot(newer)
|
||
writeUnit(t, olderNS, "z", "2026-01-01T00:00:00Z")
|
||
writeUnit(t, newerNS, "z", "2026-07-01T00:00:00Z")
|
||
_ = sett.SetAppOffbox("z", true)
|
||
rr := &recordingOffboxRunner{}
|
||
m.SetOffboxRunner(rr.run)
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("run: %v", err)
|
||
}
|
||
wantSrc := RecoveryUnitPath(newerNS, "z")
|
||
if len(rr.backupSrc) != 1 || rr.backupSrc[0] != wantSrc {
|
||
t.Fatalf("must back up exactly the NEWER unit once; got %v want [%s]", rr.backupSrc, wantSrc)
|
||
}
|
||
}
|
||
|
||
// E — every toggled app present: all backed up, status ok, no warning, snapshot count reflects stats.
|
||
func TestOffbox_AllPresentHappyPath(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
usb := t.TempDir()
|
||
addSchedulablePath(t, sett, usb)
|
||
usbNS := m.namespaceRoot(usb)
|
||
writeUnit(t, usbNS, "a", "2026-07-01T00:00:00Z")
|
||
writeUnit(t, usbNS, "b", "2026-07-01T00:00:00Z")
|
||
_ = sett.SetAppOffbox("a", true)
|
||
_ = sett.SetAppOffbox("b", true)
|
||
rr := &recordingOffboxRunner{}
|
||
m.SetOffboxRunner(rr.run)
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("run: %v", err)
|
||
}
|
||
if len(rr.backupSrc) != 2 {
|
||
t.Fatalf("both apps must be backed up, got %v", rr.backupSrc)
|
||
}
|
||
st := sett.GetOffboxTarget()
|
||
if st.LastStatus != "ok" || st.LastWarning != "" {
|
||
t.Fatalf("happy path wants ok + no warning, got %+v", st)
|
||
}
|
||
if st.SnapshotCount != 1 {
|
||
t.Fatalf("snapshot count should reflect the stats fake (1), got %d", st.SnapshotCount)
|
||
}
|
||
}
|
||
|
||
// Edge — zero toggled apps is a clean no-op ok (no error, no backup call).
|
||
func TestOffbox_NoAppsToggledIsCleanOK(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
rr := &recordingOffboxRunner{}
|
||
m.SetOffboxRunner(rr.run)
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("zero toggled apps must be a clean no-op, got %v", err)
|
||
}
|
||
if len(rr.backupSrc) != 0 {
|
||
t.Fatalf("no backup should run with 0 toggled apps, got %v", rr.backupSrc)
|
||
}
|
||
if st := sett.GetOffboxTarget(); st.LastStatus != "ok" || st.LastError != "" {
|
||
t.Fatalf("status = %+v, want ok / no error", st)
|
||
}
|
||
}
|
||
|
||
// --- fork-4: atomicity gate + DR inject + coord ---
|
||
|
||
// setPending overrides the harness's escrowed default to pending.
|
||
func setEscrowState(t *testing.T, sett *settings.Settings, state string) {
|
||
t.Helper()
|
||
if err := sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.EscrowState = state }); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
// Scenario A — a toggled app with a present unit is NOT backed up while escrow is pending (atomicity).
|
||
func TestOffbox_PendingEscrowBlocksRun(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
setEscrowState(t, sett, "pending")
|
||
usb := t.TempDir()
|
||
addSchedulablePath(t, sett, usb)
|
||
writeUnit(t, m.namespaceRoot(usb), "app1", "2026-07-01T00:00:00Z")
|
||
_ = sett.SetAppOffbox("app1", true)
|
||
rr := &recordingOffboxRunner{}
|
||
m.SetOffboxRunner(rr.run)
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("pending escrow must be a clean skip, got %v", err)
|
||
}
|
||
if len(rr.backupSrc) != 0 {
|
||
t.Fatalf("NO offsite backup may run while escrow is pending, got %v", rr.backupSrc)
|
||
}
|
||
if !m.OffboxConfigured() {
|
||
t.Fatal("config must still be valid while pending (only RUNS are gated)")
|
||
}
|
||
if m.OffboxRunnable() {
|
||
t.Fatal("OffboxRunnable must be false while pending")
|
||
}
|
||
}
|
||
|
||
// Scenario B — confirming escrow flips to escrowed and the run then proceeds.
|
||
func TestOffbox_ConfirmEscrowEnablesRun(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
setEscrowState(t, sett, "pending")
|
||
usb := t.TempDir()
|
||
addSchedulablePath(t, sett, usb)
|
||
writeUnit(t, m.namespaceRoot(usb), "app1", "2026-07-01T00:00:00Z")
|
||
_ = sett.SetAppOffbox("app1", true)
|
||
rr := &recordingOffboxRunner{}
|
||
m.SetOffboxRunner(rr.run)
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil || len(rr.backupSrc) != 0 {
|
||
t.Fatalf("must be blocked while pending (err=%v src=%v)", err, rr.backupSrc)
|
||
}
|
||
setEscrowState(t, sett, "escrowed")
|
||
if !m.OffboxRunnable() {
|
||
t.Fatal("must be runnable after confirm")
|
||
}
|
||
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
||
t.Fatalf("run after confirm: %v", err)
|
||
}
|
||
if len(rr.backupSrc) != 1 {
|
||
t.Fatalf("must back up after confirm, got %v", rr.backupSrc)
|
||
}
|
||
}
|
||
|
||
// Scenario C — a pre-placed (DR-injected) recovered password is honored, not regenerated.
|
||
func TestOffbox_InjectPasswordPrePlaced(t *testing.T) {
|
||
m, _ := newOffboxManager(t)
|
||
_ = os.Remove(m.offboxPwPath()) // simulate a fresh controller (no password yet)
|
||
const recovered = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||
if err := m.InjectOffboxPassword(recovered, false); err != nil {
|
||
t.Fatalf("inject: %v", err)
|
||
}
|
||
if err := m.WriteOffboxSecrets("newkey", "nas.local ssh-ed25519 NEWKEY"); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
got, _ := os.ReadFile(m.offboxPwPath())
|
||
if string(got) != recovered {
|
||
t.Fatalf("injected password was overwritten (len now %d) — the existing repo would be unopenable", len(got))
|
||
}
|
||
// refuse to clobber an existing password without force
|
||
if err := m.InjectOffboxPassword("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", false); err == nil {
|
||
t.Fatal("inject must refuse to clobber an existing password without force")
|
||
}
|
||
// force overwrites
|
||
const forced = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
|
||
if err := m.InjectOffboxPassword(forced, true); err != nil {
|
||
t.Fatalf("force inject: %v", err)
|
||
}
|
||
if got2, _ := os.ReadFile(m.offboxPwPath()); string(got2) != forced {
|
||
t.Fatal("force inject must overwrite")
|
||
}
|
||
// invalid (non-hex / wrong length) rejected
|
||
if err := m.InjectOffboxPassword("not-a-valid-hex-password", false); err == nil {
|
||
t.Fatal("an invalid repo password must be rejected")
|
||
}
|
||
}
|
||
|
||
// Companion to C — WITHOUT inject, WriteOffboxSecrets generates a DIFFERENT password (so the recovered
|
||
// one is load-bearing: a fresh gen could never open the existing offsite repo).
|
||
func TestOffbox_NoInjectGeneratesDifferentPassword(t *testing.T) {
|
||
m, _ := newOffboxManager(t)
|
||
_ = os.Remove(m.offboxPwPath())
|
||
const recovered = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||
if err := m.WriteOffboxSecrets("k", "kh"); err != nil { // no inject → generates fresh
|
||
t.Fatal(err)
|
||
}
|
||
raw, rerr := os.ReadFile(m.offboxPwPath())
|
||
if rerr != nil {
|
||
t.Fatal(rerr)
|
||
}
|
||
got := strings.TrimSpace(string(raw))
|
||
if got == recovered {
|
||
t.Fatal("the freshly generated password coincided with the recovered one (impossible with 256-bit entropy)")
|
||
}
|
||
if len(got) != 64 {
|
||
t.Fatalf("generated repo password must be 64 hex, got %d", len(got))
|
||
}
|
||
}
|
||
|
||
// Scenario E (backup half) — OffboxCoord returns the non-secret coordinates, ok=false when unconfigured.
|
||
func TestOffbox_CoordForDR(t *testing.T) {
|
||
m, sett := newOffboxManager(t)
|
||
host, user, port, repo, ok := m.OffboxCoord()
|
||
if !ok || host != "nas.local" || user != "felhom" || port != 22 || repo != "/srv/repo" {
|
||
t.Fatalf("coord = %s/%s/%d/%s ok=%v", host, user, port, repo, ok)
|
||
}
|
||
if err := sett.SetOffboxTarget(&settings.OffboxTarget{}); err != nil { // empty target
|
||
t.Fatal(err)
|
||
}
|
||
if _, _, _, _, ok := m.OffboxCoord(); ok {
|
||
t.Fatal("an unconfigured target must yield ok=false")
|
||
}
|
||
}
|
||
|
||
func runtimeIsUnix() bool { return os.PathSeparator == '/' }
|
||
|
||
func contains(ss []string, want string) bool {
|
||
for _, s := range ss {
|
||
if s == want {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// tagOf returns the LAST --tag value (the per-stack tag; backup adds "felhom-offbox" then the stack).
|
||
func tagOf(args []string) string {
|
||
tag := ""
|
||
for i, a := range args {
|
||
if a == "--tag" && i+1 < len(args) {
|
||
tag = args[i+1]
|
||
}
|
||
}
|
||
return tag
|
||
}
|
||
|
||
func valAfter(args []string, flag string) string {
|
||
for i, a := range args {
|
||
if a == flag && i+1 < len(args) {
|
||
return args[i+1]
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func copyTree(src, dst string) error {
|
||
return filepath.Walk(src, func(p string, info os.FileInfo, err error) error {
|
||
if err != nil {
|
||
return err
|
||
}
|
||
rel, _ := filepath.Rel(src, p)
|
||
target := filepath.Join(dst, rel)
|
||
if info.IsDir() {
|
||
return os.MkdirAll(target, 0o755)
|
||
}
|
||
b, rerr := os.ReadFile(p)
|
||
if rerr != nil {
|
||
return rerr
|
||
}
|
||
return os.WriteFile(target, b, 0o644)
|
||
})
|
||
}
|