Files
felhom-controller/controller/internal/backup/shares_test.go
T
admin 85b76e0fc3 feat(shares): R-7b Part 3 — offsite shares leg (Model B') + B' isolation proof
ONE additional restic call tagged [felhom-offbox, _shares] carrying the payload
staging dir + every mandatory share folder. Hooked into runOffboxInternal AFTER the
per-app loop and BEFORE retention, so forget --group-by host,tags covers the _shares
group with no flag change. Reuses resticStep, the caller's repo-ensure and
single-flight, and the SAME enlargement-gate arithmetic.

- quota gate degrades the push to MANIFEST-ONLY, never to nothing
- EnlargedBlocked keeps the RAW _shares key (templates index by it); the display
  mapping applies only at the notification + Hungarian-prose boundaries
- OffboxTarget gains SharesLastRun/Status/Count for per-tier page truth
- zero-toggle notice suppressed when the shares leg provided coverage
- reserved-name defense: an app keyed _shares is excluded from the run loudly

RED-PROOFS RUN AND REVERTED (all fired):
  1. shares leg appends into the app's argv -> isolation test FAILS
  2. mandatory->offsite mapping inverted    -> Scenario A + B FAIL
  3. manifest-only degradation dropped      -> Scenario C FAILS
2026-07-18 12:51:46 +02:00

456 lines
17 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package backup
import (
"encoding/json"
"io"
"log"
"os"
"path/filepath"
"sort"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// --- shared harness -------------------------------------------------------------------------------
// sharesEnv is a Manager wired for the R-7b shares source: a real settings store, real temp drives,
// and FAKE copy/exec seams so nothing shells out to rsync, du, docker or restic.
type sharesEnv struct {
m *Manager
// mirrored records every tier2Mirror call as "src=>dst" in call order.
mirrored []string
// drives maps a label to the temp dir standing in for that registered drive.
drives map[string]string
sett *settings.Settings
tmp string
}
// newSharesEnv registers the named drives, turns sharing on, and installs the fake seams. Shares are
// added afterwards with addShare so each test states exactly the registry it needs.
func newSharesEnv(t *testing.T, driveNames ...string) *sharesEnv {
t.Helper()
tmp := t.TempDir()
sett, err := settings.Load(filepath.Join(tmp, "settings.json"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
env := &sharesEnv{drives: map[string]string{}, sett: sett, tmp: tmp}
for _, name := range driveNames {
p := filepath.Join(tmp, name)
if err := os.MkdirAll(p, 0o755); err != nil {
t.Fatal(err)
}
if err := sett.AddStoragePath(settings.StoragePath{Path: p, Label: name, Schedulable: true}); err != nil {
t.Fatal(err)
}
env.drives[name] = p
}
if err := sett.SetSMBEnabled(true); err != nil {
t.Fatal(err)
}
cfg := &config.Config{}
cfg.Paths.DataDir = filepath.Join(tmp, "data")
sysDrive := filepath.Join(tmp, "sysdrive")
if err := os.MkdirAll(sysDrive, 0o755); err != nil {
t.Fatal(err)
}
cfg.Paths.SystemDataPath = sysDrive
env.m = &Manager{
cfg: cfg, settings: sett, logger: log.New(io.Discard, "", 0), systemDataPath: sysDrive,
// Fake mirror: records the call AND actually copies, so restore tests have real bytes.
tier2Mirror: func(src, dst string) error {
env.mirrored = append(env.mirrored, src+"=>"+dst)
return copyTreeForTest(src, dst)
},
sharesPassdbCapture: func() ([]byte, error) { return []byte("FAKE-PASSDB-TAR"), nil },
}
return env
}
// addShare registers a share, creating its folder with one file so it is "available".
func (e *sharesEnv) addShare(t *testing.T, drive, name string, offsite bool) string {
t.Helper()
p := filepath.Join(e.drives[drive], name)
if err := os.MkdirAll(p, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(p, name+".txt"), []byte("content-of-"+name), 0o644); err != nil {
t.Fatal(err)
}
if err := e.sett.AddSMBShare(settings.SMBShare{Name: name, Path: p, Offsite: offsite, CreatedAt: "2026-07-18T00:00:00Z"}); err != nil {
t.Fatal(err)
}
return p
}
// copyTreeForTest is a minimal recursive copy standing in for rsync in the mirror seam.
func copyTreeForTest(src, dst string) error {
fi, err := os.Stat(src)
if err != nil {
return err
}
if !fi.IsDir() {
b, rErr := os.ReadFile(src)
if rErr != nil {
return rErr
}
if mErr := os.MkdirAll(filepath.Dir(dst), 0o755); mErr != nil {
return mErr
}
return os.WriteFile(dst, b, 0o644)
}
if err := os.MkdirAll(dst, 0o755); err != nil {
return err
}
entries, err := os.ReadDir(src)
if err != nil {
return err
}
for _, e := range entries {
if err := copyTreeForTest(filepath.Join(src, e.Name()), filepath.Join(dst, e.Name())); err != nil {
return err
}
}
return nil
}
// mirroredDsts returns just the destination side of every recorded mirror call.
func (e *sharesEnv) mirroredDsts() []string {
var out []string
for _, s := range e.mirrored {
out = append(out, s[strings.Index(s, "=>")+2:])
}
return out
}
// --- Part 1: payload ------------------------------------------------------------------------------
// The manifest must carry the share DEFINITIONS verbatim — that is the whole point of the payload:
// after a DR rebuild the customer gets their files AND their „Megosztás" page back, not an empty one.
func TestSharesPayloadManifestMatchesRegistry(t *testing.T) {
env := newSharesEnv(t, "hdd_1")
env.addShare(t, "hdd_1", "dokumentumok", true)
env.addShare(t, "hdd_1", "filmek", false)
dir, passdbOK, err := env.m.buildSharesPayload()
if err != nil {
t.Fatalf("buildSharesPayload: %v", err)
}
if !passdbOK {
t.Fatal("expected the (faked) passdb capture to succeed")
}
blob, err := os.ReadFile(filepath.Join(dir, sharesManifestName))
if err != nil {
t.Fatalf("read manifest: %v", err)
}
var mf SharesManifest
if err := json.Unmarshal(blob, &mf); err != nil {
t.Fatalf("manifest is not valid JSON: %v", err)
}
if mf.Version != sharesManifestVersion {
t.Errorf("manifest version = %d, want %d", mf.Version, sharesManifestVersion)
}
if mf.ServerName != settings.DefaultSMBServerName {
t.Errorf("server name = %q, want %q", mf.ServerName, settings.DefaultSMBServerName)
}
want := env.sett.GetSMBShares()
sort.Slice(want, func(i, j int) bool { return want[i].Name < want[j].Name })
if len(mf.Shares) != len(want) {
t.Fatalf("manifest has %d shares, registry has %d", len(mf.Shares), len(want))
}
for i := range want {
if mf.Shares[i] != want[i] {
t.Errorf("share %d: manifest %+v != registry %+v", i, mf.Shares[i], want[i])
}
}
// The credential copy must be present and 0600 — it is secret-bearing.
st, err := os.Stat(filepath.Join(dir, sharesPassdbName))
if err != nil {
t.Fatalf("passdb archive missing: %v", err)
}
if runtimeIsPOSIX() && st.Mode().Perm() != 0o600 {
t.Errorf("passdb archive mode = %o, want 600 (secret-bearing)", st.Mode().Perm())
}
}
// Determinism: an unchanged registry must marshal to IDENTICAL bytes regardless of the order shares
// were added in, so a no-op nightly run gives the tier-2 mirror nothing to rewrite.
func TestSharesPayloadManifestIsDeterministic(t *testing.T) {
// Built from the registry only (no filesystem), so the two runs differ ONLY in insertion order —
// otherwise the temp-dir paths baked into each share would make any comparison meaningless.
read := func(order []string) []byte {
sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
for _, n := range order {
if err := sett.AddSMBShare(settings.SMBShare{
Name: n, Path: "/mnt/hdd_1/" + n, Offsite: true, CreatedAt: "2026-07-18T00:00:00Z",
}); err != nil {
t.Fatal(err)
}
}
m := &Manager{settings: sett, logger: log.New(io.Discard, "", 0)}
b, err := json.MarshalIndent(m.buildSharesManifest(), "", " ")
if err != nil {
t.Fatal(err)
}
return b
}
a := read([]string{"alpha", "zulu", "mike"})
b := read([]string{"zulu", "mike", "alpha"})
if string(a) != string(b) {
t.Errorf("manifest is order-dependent:\n%s\n---\n%s", a, b)
}
}
// Scenario: samba container down. The payload must still be produced (manifest-only) and must NOT
// error — re-setting the SMB password is cheap; losing the definitions is not.
func TestSharesPayloadPassdbAbsentIsManifestOnly(t *testing.T) {
env := newSharesEnv(t, "hdd_1")
env.addShare(t, "hdd_1", "dokumentumok", true)
env.m.sharesPassdbCapture = func() ([]byte, error) { return nil, os.ErrNotExist }
dir, passdbOK, err := env.m.buildSharesPayload()
if err != nil {
t.Fatalf("a down container must not fail the payload: %v", err)
}
if passdbOK {
t.Error("passdbOK must be false when the capture failed and no prior copy exists")
}
if _, err := os.Stat(filepath.Join(dir, sharesManifestName)); err != nil {
t.Errorf("manifest must still be written: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, sharesPassdbName)); !os.IsNotExist(err) {
t.Error("no passdb archive should exist when the capture failed and none was staged before")
}
}
// A previously captured credential copy must SURVIVE a later failed capture — a stale credential is
// strictly better for DR than none, and the manifest carries no password of its own to contradict it.
func TestSharesPayloadKeepsPriorPassdbOnCaptureFailure(t *testing.T) {
env := newSharesEnv(t, "hdd_1")
env.addShare(t, "hdd_1", "dokumentumok", true)
dir, _, err := env.m.buildSharesPayload()
if err != nil {
t.Fatal(err)
}
env.m.sharesPassdbCapture = func() ([]byte, error) { return nil, os.ErrNotExist }
_, passdbOK, err := env.m.buildSharesPayload()
if err != nil {
t.Fatal(err)
}
if !passdbOK {
t.Error("a previously staged passdb copy must keep passdbOK true")
}
b, err := os.ReadFile(filepath.Join(dir, sharesPassdbName))
if err != nil || string(b) != "FAKE-PASSDB-TAR" {
t.Errorf("prior passdb copy was lost: %q, %v", b, err)
}
}
// The reserved key must never reach a Hungarian surface raw.
func TestDisplayStackNameMapsReservedKey(t *testing.T) {
if got := DisplayStackName(SharesPseudoStack); got != SharesDisplayName {
t.Errorf("DisplayStackName(%q) = %q, want %q", SharesPseudoStack, got, SharesDisplayName)
}
if got := DisplayStackName("immich"); got != "immich" {
t.Errorf("DisplayStackName must be identity for ordinary stacks, got %q", got)
}
}
// --- Part 2: tier-2 shares job --------------------------------------------------------------------
// Scenario B: an OPTIONAL (Felhőmentés-off) share is tier-2'd like any other — the tier-2 tier is the
// local copy every share gets; only the offsite tier discriminates.
func TestSharesTier2MirrorsBothClasses(t *testing.T) {
env := newSharesEnv(t, "hdd_1", "hdd_2")
env.addShare(t, "hdd_1", "dokumentumok", true)
env.addShare(t, "hdd_1", "filmek", false)
if err := env.m.RunSharesTier2(); err != nil {
t.Fatalf("RunSharesTier2: %v", err)
}
key := sharesDriveKey(env.drives["hdd_1"])
destBase := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", SharesPseudoStack)
for _, share := range []string{"dokumentumok", "filmek"} {
want := filepath.Join(destBase, key, share)
if _, err := os.Stat(filepath.Join(want, share+".txt")); err != nil {
t.Errorf("share %s was not mirrored to %s: %v", share, want, err)
}
}
// The payload rides the target so it is independently restorable.
if _, err := os.Stat(filepath.Join(destBase, sharesPayloadDestRel, sharesManifestName)); err != nil {
t.Errorf("payload manifest missing from the tier-2 target: %v", err)
}
// Marker LAST: its presence means every leg + reconcile completed.
b, err := os.ReadFile(filepath.Join(destBase, tier2LayoutMarker))
if err != nil || string(b) != tier2LayoutVersion {
t.Errorf("layout marker missing/wrong: %q, %v", b, err)
}
// Status is recorded under the reserved key, and renders as „Megosztások".
cd := env.m.SharesTier2Status()
if cd == nil || cd.LastStatus != "ok" {
t.Fatalf("shares tier-2 status = %+v, want LastStatus=ok", cd)
}
}
// The target may NEVER be the leg's own source drive — that would be a same-disk copy pretending to
// be a second backup. With shares on two drives each group must land on the other one.
func TestSharesTier2NeverTargetsItsOwnSourceDrive(t *testing.T) {
env := newSharesEnv(t, "hdd_1", "hdd_2")
env.addShare(t, "hdd_1", "egy", true)
env.addShare(t, "hdd_2", "ketto", true)
if err := env.m.RunSharesTier2(); err != nil {
t.Fatalf("RunSharesTier2: %v", err)
}
for _, call := range env.mirrored {
src, dst, _ := strings.Cut(call, "=>")
if strings.HasPrefix(src, env.drives["hdd_1"]) && strings.HasPrefix(dst, env.drives["hdd_1"]) {
t.Errorf("leg mirrored onto its own source drive: %s", call)
}
if strings.HasPrefix(src, env.drives["hdd_2"]) && strings.HasPrefix(dst, env.drives["hdd_2"]) {
t.Errorf("leg mirrored onto its own source drive: %s", call)
}
}
}
// Scenario F: a share whose drive is marked away is skipped LOUDLY and its path reaches NO mirror
// argument — mirroring a dead mountpoint would copy an empty dir over a good backup.
func TestSharesTier2SkipsDeadMountAndContinues(t *testing.T) {
env := newSharesEnv(t, "hdd_1", "hdd_2", "hdd_3")
env.addShare(t, "hdd_1", "elo", true)
deadPath := env.addShare(t, "hdd_3", "halott", true)
if err := env.sett.SetDisconnected(env.drives["hdd_3"], true, nil); err != nil {
t.Fatal(err)
}
if err := env.m.RunSharesTier2(); err != nil {
t.Fatalf("the run must continue for healthy shares: %v", err)
}
for _, call := range env.mirrored {
if strings.Contains(call, deadPath) {
t.Errorf("dead-mount share reached a mirror call: %s", call)
}
}
var sawLive bool
for _, dst := range env.mirroredDsts() {
if strings.HasSuffix(dst, string(filepath.Separator)+"elo") {
sawLive = true
}
}
if !sawLive {
t.Error("the healthy share must still be mirrored")
}
}
// Zero shares / sharing disabled: a clean no-op — no dest tree, no status record, no empty dirs.
func TestSharesTier2ZeroSharesIsCleanNoOp(t *testing.T) {
env := newSharesEnv(t, "hdd_1", "hdd_2")
if err := env.m.RunSharesTier2(); err != nil {
t.Fatalf("RunSharesTier2: %v", err)
}
if len(env.mirrored) != 0 {
t.Errorf("expected no mirror calls, got %v", env.mirrored)
}
if cd := env.m.SharesTier2Status(); cd != nil {
t.Errorf("expected no status record for a feature nobody uses, got %+v", cd)
}
destBase := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", SharesPseudoStack)
if _, err := os.Stat(destBase); !os.IsNotExist(err) {
t.Errorf("expected no _shares dest tree, stat err = %v", err)
}
}
// A share removed from the registry must stop occupying the secondary drive within ONE run — the
// reconcile pass prunes it, reusing the same destBase-bounded removal guard the app path uses.
func TestSharesTier2ReconcilePrunesRemovedShare(t *testing.T) {
env := newSharesEnv(t, "hdd_1", "hdd_2")
env.addShare(t, "hdd_1", "marad", true)
env.addShare(t, "hdd_1", "torolt", true)
if err := env.m.RunSharesTier2(); err != nil {
t.Fatal(err)
}
key := sharesDriveKey(env.drives["hdd_1"])
destBase := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", SharesPseudoStack)
stale := filepath.Join(destBase, key, "torolt")
if _, err := os.Stat(stale); err != nil {
t.Fatalf("precondition: %s should exist after the first run: %v", stale, err)
}
if err := env.sett.RemoveSMBShare("torolt"); err != nil {
t.Fatal(err)
}
if err := env.m.RunSharesTier2(); err != nil {
t.Fatal(err)
}
if _, err := os.Stat(stale); !os.IsNotExist(err) {
t.Errorf("removed share's dest dir was not pruned: stat err = %v", err)
}
if _, err := os.Stat(filepath.Join(destBase, key, "marad")); err != nil {
t.Errorf("surviving share was pruned by mistake: %v", err)
}
}
// The tier-2 half of the B isolation proof: the shares job must write ONLY under
// backups/secondary/_shares. A per-app dest tree standing beside it must come out byte-for-byte
// untouched — same contents, same bytes — and no mirror call may target it. Red-proof: point the
// shares destBase at backups/secondary/<share> (dropping the _shares segment) and this fails.
func TestSharesTier2LeavesPerAppTreeUntouched(t *testing.T) {
env := newSharesEnv(t, "hdd_1", "hdd_2")
env.addShare(t, "hdd_1", "dokumentumok", true)
// A pre-existing per-app tier-2 dest with a sentinel payload.
appDest := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", "immich")
if err := os.MkdirAll(filepath.Join(appDest, "recovery-unit"), 0o755); err != nil {
t.Fatal(err)
}
sentinel := filepath.Join(appDest, "recovery-unit", "manifest.json")
if err := os.WriteFile(sentinel, []byte(`{"app":"immich"}`), 0o644); err != nil {
t.Fatal(err)
}
if err := env.m.RunSharesTier2(); err != nil {
t.Fatal(err)
}
b, err := os.ReadFile(sentinel)
if err != nil || string(b) != `{"app":"immich"}` {
t.Errorf("B INVARIANT VIOLATED — the per-app tier-2 tree was modified: %q, %v", b, err)
}
for _, call := range env.mirrored {
if strings.Contains(call, appDest) {
t.Errorf("B INVARIANT VIOLATED — a shares mirror targeted the per-app dest: %s", call)
}
}
// And everything it DID write lives under the reserved subtree.
sharesRoot := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", SharesPseudoStack)
for _, dst := range env.mirroredDsts() {
if !strings.HasPrefix(dst, sharesRoot+string(filepath.Separator)) {
t.Errorf("shares job wrote outside its reserved subtree: %s", dst)
}
}
}
// Drive keys must be collision-free across drives that share a basename.
func TestSharesDriveKeyIsCollisionFree(t *testing.T) {
if sharesDriveKey("/mnt/a/data") == sharesDriveKey("/mnt/b/data") {
t.Error("drives with the same basename must not map to the same dest key")
}
for _, bad := range []string{"/", "\x00", " "} {
if k := sharesDriveKey(bad); strings.ContainsAny(k, `/\`) {
t.Errorf("sharesDriveKey(%q) = %q — must be a single safe path segment", bad, k)
}
}
}
// runtimeIsPOSIX reports whether file-mode assertions are meaningful on this host (Windows reports
// synthesised permissions, so mode checks there are noise rather than signal).
func runtimeIsPOSIX() bool { return os.PathSeparator == '/' }