85b76e0fc3
ONE additional restic call tagged [felhom-offbox, _shares] carrying the payload staging dir + every mandatory share folder. Hooked into runOffboxInternal AFTER the per-app loop and BEFORE retention, so forget --group-by host,tags covers the _shares group with no flag change. Reuses resticStep, the caller's repo-ensure and single-flight, and the SAME enlargement-gate arithmetic. - quota gate degrades the push to MANIFEST-ONLY, never to nothing - EnlargedBlocked keeps the RAW _shares key (templates index by it); the display mapping applies only at the notification + Hungarian-prose boundaries - OffboxTarget gains SharesLastRun/Status/Count for per-tier page truth - zero-toggle notice suppressed when the shares leg provided coverage - reserved-name defense: an app keyed _shares is excluded from the run loudly RED-PROOFS RUN AND REVERTED (all fired): 1. shares leg appends into the app's argv -> isolation test FAILS 2. mandatory->offsite mapping inverted -> Scenario A + B FAIL 3. manifest-only degradation dropped -> Scenario C FAILS
456 lines
17 KiB
Go
456 lines
17 KiB
Go
package backup
|
||
|
||
import (
|
||
"encoding/json"
|
||
"io"
|
||
"log"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"testing"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||
)
|
||
|
||
// --- shared harness -------------------------------------------------------------------------------
|
||
|
||
// sharesEnv is a Manager wired for the R-7b shares source: a real settings store, real temp drives,
|
||
// and FAKE copy/exec seams so nothing shells out to rsync, du, docker or restic.
|
||
type sharesEnv struct {
|
||
m *Manager
|
||
// mirrored records every tier2Mirror call as "src=>dst" in call order.
|
||
mirrored []string
|
||
// drives maps a label to the temp dir standing in for that registered drive.
|
||
drives map[string]string
|
||
sett *settings.Settings
|
||
tmp string
|
||
}
|
||
|
||
// newSharesEnv registers the named drives, turns sharing on, and installs the fake seams. Shares are
|
||
// added afterwards with addShare so each test states exactly the registry it needs.
|
||
func newSharesEnv(t *testing.T, driveNames ...string) *sharesEnv {
|
||
t.Helper()
|
||
tmp := t.TempDir()
|
||
sett, err := settings.Load(filepath.Join(tmp, "settings.json"), log.New(io.Discard, "", 0))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
env := &sharesEnv{drives: map[string]string{}, sett: sett, tmp: tmp}
|
||
for _, name := range driveNames {
|
||
p := filepath.Join(tmp, name)
|
||
if err := os.MkdirAll(p, 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := sett.AddStoragePath(settings.StoragePath{Path: p, Label: name, Schedulable: true}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
env.drives[name] = p
|
||
}
|
||
if err := sett.SetSMBEnabled(true); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cfg := &config.Config{}
|
||
cfg.Paths.DataDir = filepath.Join(tmp, "data")
|
||
sysDrive := filepath.Join(tmp, "sysdrive")
|
||
if err := os.MkdirAll(sysDrive, 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cfg.Paths.SystemDataPath = sysDrive
|
||
|
||
env.m = &Manager{
|
||
cfg: cfg, settings: sett, logger: log.New(io.Discard, "", 0), systemDataPath: sysDrive,
|
||
// Fake mirror: records the call AND actually copies, so restore tests have real bytes.
|
||
tier2Mirror: func(src, dst string) error {
|
||
env.mirrored = append(env.mirrored, src+"=>"+dst)
|
||
return copyTreeForTest(src, dst)
|
||
},
|
||
sharesPassdbCapture: func() ([]byte, error) { return []byte("FAKE-PASSDB-TAR"), nil },
|
||
}
|
||
return env
|
||
}
|
||
|
||
// addShare registers a share, creating its folder with one file so it is "available".
|
||
func (e *sharesEnv) addShare(t *testing.T, drive, name string, offsite bool) string {
|
||
t.Helper()
|
||
p := filepath.Join(e.drives[drive], name)
|
||
if err := os.MkdirAll(p, 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.WriteFile(filepath.Join(p, name+".txt"), []byte("content-of-"+name), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := e.sett.AddSMBShare(settings.SMBShare{Name: name, Path: p, Offsite: offsite, CreatedAt: "2026-07-18T00:00:00Z"}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return p
|
||
}
|
||
|
||
// copyTreeForTest is a minimal recursive copy standing in for rsync in the mirror seam.
|
||
func copyTreeForTest(src, dst string) error {
|
||
fi, err := os.Stat(src)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !fi.IsDir() {
|
||
b, rErr := os.ReadFile(src)
|
||
if rErr != nil {
|
||
return rErr
|
||
}
|
||
if mErr := os.MkdirAll(filepath.Dir(dst), 0o755); mErr != nil {
|
||
return mErr
|
||
}
|
||
return os.WriteFile(dst, b, 0o644)
|
||
}
|
||
if err := os.MkdirAll(dst, 0o755); err != nil {
|
||
return err
|
||
}
|
||
entries, err := os.ReadDir(src)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, e := range entries {
|
||
if err := copyTreeForTest(filepath.Join(src, e.Name()), filepath.Join(dst, e.Name())); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// mirroredDsts returns just the destination side of every recorded mirror call.
|
||
func (e *sharesEnv) mirroredDsts() []string {
|
||
var out []string
|
||
for _, s := range e.mirrored {
|
||
out = append(out, s[strings.Index(s, "=>")+2:])
|
||
}
|
||
return out
|
||
}
|
||
|
||
// --- Part 1: payload ------------------------------------------------------------------------------
|
||
|
||
// The manifest must carry the share DEFINITIONS verbatim — that is the whole point of the payload:
|
||
// after a DR rebuild the customer gets their files AND their „Megosztás" page back, not an empty one.
|
||
func TestSharesPayloadManifestMatchesRegistry(t *testing.T) {
|
||
env := newSharesEnv(t, "hdd_1")
|
||
env.addShare(t, "hdd_1", "dokumentumok", true)
|
||
env.addShare(t, "hdd_1", "filmek", false)
|
||
|
||
dir, passdbOK, err := env.m.buildSharesPayload()
|
||
if err != nil {
|
||
t.Fatalf("buildSharesPayload: %v", err)
|
||
}
|
||
if !passdbOK {
|
||
t.Fatal("expected the (faked) passdb capture to succeed")
|
||
}
|
||
blob, err := os.ReadFile(filepath.Join(dir, sharesManifestName))
|
||
if err != nil {
|
||
t.Fatalf("read manifest: %v", err)
|
||
}
|
||
var mf SharesManifest
|
||
if err := json.Unmarshal(blob, &mf); err != nil {
|
||
t.Fatalf("manifest is not valid JSON: %v", err)
|
||
}
|
||
if mf.Version != sharesManifestVersion {
|
||
t.Errorf("manifest version = %d, want %d", mf.Version, sharesManifestVersion)
|
||
}
|
||
if mf.ServerName != settings.DefaultSMBServerName {
|
||
t.Errorf("server name = %q, want %q", mf.ServerName, settings.DefaultSMBServerName)
|
||
}
|
||
want := env.sett.GetSMBShares()
|
||
sort.Slice(want, func(i, j int) bool { return want[i].Name < want[j].Name })
|
||
if len(mf.Shares) != len(want) {
|
||
t.Fatalf("manifest has %d shares, registry has %d", len(mf.Shares), len(want))
|
||
}
|
||
for i := range want {
|
||
if mf.Shares[i] != want[i] {
|
||
t.Errorf("share %d: manifest %+v != registry %+v", i, mf.Shares[i], want[i])
|
||
}
|
||
}
|
||
// The credential copy must be present and 0600 — it is secret-bearing.
|
||
st, err := os.Stat(filepath.Join(dir, sharesPassdbName))
|
||
if err != nil {
|
||
t.Fatalf("passdb archive missing: %v", err)
|
||
}
|
||
if runtimeIsPOSIX() && st.Mode().Perm() != 0o600 {
|
||
t.Errorf("passdb archive mode = %o, want 600 (secret-bearing)", st.Mode().Perm())
|
||
}
|
||
}
|
||
|
||
// Determinism: an unchanged registry must marshal to IDENTICAL bytes regardless of the order shares
|
||
// were added in, so a no-op nightly run gives the tier-2 mirror nothing to rewrite.
|
||
func TestSharesPayloadManifestIsDeterministic(t *testing.T) {
|
||
// Built from the registry only (no filesystem), so the two runs differ ONLY in insertion order —
|
||
// otherwise the temp-dir paths baked into each share would make any comparison meaningless.
|
||
read := func(order []string) []byte {
|
||
sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), log.New(io.Discard, "", 0))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, n := range order {
|
||
if err := sett.AddSMBShare(settings.SMBShare{
|
||
Name: n, Path: "/mnt/hdd_1/" + n, Offsite: true, CreatedAt: "2026-07-18T00:00:00Z",
|
||
}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
m := &Manager{settings: sett, logger: log.New(io.Discard, "", 0)}
|
||
b, err := json.MarshalIndent(m.buildSharesManifest(), "", " ")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return b
|
||
}
|
||
a := read([]string{"alpha", "zulu", "mike"})
|
||
b := read([]string{"zulu", "mike", "alpha"})
|
||
if string(a) != string(b) {
|
||
t.Errorf("manifest is order-dependent:\n%s\n---\n%s", a, b)
|
||
}
|
||
}
|
||
|
||
// Scenario: samba container down. The payload must still be produced (manifest-only) and must NOT
|
||
// error — re-setting the SMB password is cheap; losing the definitions is not.
|
||
func TestSharesPayloadPassdbAbsentIsManifestOnly(t *testing.T) {
|
||
env := newSharesEnv(t, "hdd_1")
|
||
env.addShare(t, "hdd_1", "dokumentumok", true)
|
||
env.m.sharesPassdbCapture = func() ([]byte, error) { return nil, os.ErrNotExist }
|
||
|
||
dir, passdbOK, err := env.m.buildSharesPayload()
|
||
if err != nil {
|
||
t.Fatalf("a down container must not fail the payload: %v", err)
|
||
}
|
||
if passdbOK {
|
||
t.Error("passdbOK must be false when the capture failed and no prior copy exists")
|
||
}
|
||
if _, err := os.Stat(filepath.Join(dir, sharesManifestName)); err != nil {
|
||
t.Errorf("manifest must still be written: %v", err)
|
||
}
|
||
if _, err := os.Stat(filepath.Join(dir, sharesPassdbName)); !os.IsNotExist(err) {
|
||
t.Error("no passdb archive should exist when the capture failed and none was staged before")
|
||
}
|
||
}
|
||
|
||
// A previously captured credential copy must SURVIVE a later failed capture — a stale credential is
|
||
// strictly better for DR than none, and the manifest carries no password of its own to contradict it.
|
||
func TestSharesPayloadKeepsPriorPassdbOnCaptureFailure(t *testing.T) {
|
||
env := newSharesEnv(t, "hdd_1")
|
||
env.addShare(t, "hdd_1", "dokumentumok", true)
|
||
dir, _, err := env.m.buildSharesPayload()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
env.m.sharesPassdbCapture = func() ([]byte, error) { return nil, os.ErrNotExist }
|
||
_, passdbOK, err := env.m.buildSharesPayload()
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !passdbOK {
|
||
t.Error("a previously staged passdb copy must keep passdbOK true")
|
||
}
|
||
b, err := os.ReadFile(filepath.Join(dir, sharesPassdbName))
|
||
if err != nil || string(b) != "FAKE-PASSDB-TAR" {
|
||
t.Errorf("prior passdb copy was lost: %q, %v", b, err)
|
||
}
|
||
}
|
||
|
||
// The reserved key must never reach a Hungarian surface raw.
|
||
func TestDisplayStackNameMapsReservedKey(t *testing.T) {
|
||
if got := DisplayStackName(SharesPseudoStack); got != SharesDisplayName {
|
||
t.Errorf("DisplayStackName(%q) = %q, want %q", SharesPseudoStack, got, SharesDisplayName)
|
||
}
|
||
if got := DisplayStackName("immich"); got != "immich" {
|
||
t.Errorf("DisplayStackName must be identity for ordinary stacks, got %q", got)
|
||
}
|
||
}
|
||
|
||
// --- Part 2: tier-2 shares job --------------------------------------------------------------------
|
||
|
||
// Scenario B: an OPTIONAL (Felhőmentés-off) share is tier-2'd like any other — the tier-2 tier is the
|
||
// local copy every share gets; only the offsite tier discriminates.
|
||
func TestSharesTier2MirrorsBothClasses(t *testing.T) {
|
||
env := newSharesEnv(t, "hdd_1", "hdd_2")
|
||
env.addShare(t, "hdd_1", "dokumentumok", true)
|
||
env.addShare(t, "hdd_1", "filmek", false)
|
||
|
||
if err := env.m.RunSharesTier2(); err != nil {
|
||
t.Fatalf("RunSharesTier2: %v", err)
|
||
}
|
||
key := sharesDriveKey(env.drives["hdd_1"])
|
||
destBase := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", SharesPseudoStack)
|
||
|
||
for _, share := range []string{"dokumentumok", "filmek"} {
|
||
want := filepath.Join(destBase, key, share)
|
||
if _, err := os.Stat(filepath.Join(want, share+".txt")); err != nil {
|
||
t.Errorf("share %s was not mirrored to %s: %v", share, want, err)
|
||
}
|
||
}
|
||
// The payload rides the target so it is independently restorable.
|
||
if _, err := os.Stat(filepath.Join(destBase, sharesPayloadDestRel, sharesManifestName)); err != nil {
|
||
t.Errorf("payload manifest missing from the tier-2 target: %v", err)
|
||
}
|
||
// Marker LAST: its presence means every leg + reconcile completed.
|
||
b, err := os.ReadFile(filepath.Join(destBase, tier2LayoutMarker))
|
||
if err != nil || string(b) != tier2LayoutVersion {
|
||
t.Errorf("layout marker missing/wrong: %q, %v", b, err)
|
||
}
|
||
// Status is recorded under the reserved key, and renders as „Megosztások".
|
||
cd := env.m.SharesTier2Status()
|
||
if cd == nil || cd.LastStatus != "ok" {
|
||
t.Fatalf("shares tier-2 status = %+v, want LastStatus=ok", cd)
|
||
}
|
||
}
|
||
|
||
// The target may NEVER be the leg's own source drive — that would be a same-disk copy pretending to
|
||
// be a second backup. With shares on two drives each group must land on the other one.
|
||
func TestSharesTier2NeverTargetsItsOwnSourceDrive(t *testing.T) {
|
||
env := newSharesEnv(t, "hdd_1", "hdd_2")
|
||
env.addShare(t, "hdd_1", "egy", true)
|
||
env.addShare(t, "hdd_2", "ketto", true)
|
||
|
||
if err := env.m.RunSharesTier2(); err != nil {
|
||
t.Fatalf("RunSharesTier2: %v", err)
|
||
}
|
||
for _, call := range env.mirrored {
|
||
src, dst, _ := strings.Cut(call, "=>")
|
||
if strings.HasPrefix(src, env.drives["hdd_1"]) && strings.HasPrefix(dst, env.drives["hdd_1"]) {
|
||
t.Errorf("leg mirrored onto its own source drive: %s", call)
|
||
}
|
||
if strings.HasPrefix(src, env.drives["hdd_2"]) && strings.HasPrefix(dst, env.drives["hdd_2"]) {
|
||
t.Errorf("leg mirrored onto its own source drive: %s", call)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Scenario F: a share whose drive is marked away is skipped LOUDLY and its path reaches NO mirror
|
||
// argument — mirroring a dead mountpoint would copy an empty dir over a good backup.
|
||
func TestSharesTier2SkipsDeadMountAndContinues(t *testing.T) {
|
||
env := newSharesEnv(t, "hdd_1", "hdd_2", "hdd_3")
|
||
env.addShare(t, "hdd_1", "elo", true)
|
||
deadPath := env.addShare(t, "hdd_3", "halott", true)
|
||
if err := env.sett.SetDisconnected(env.drives["hdd_3"], true, nil); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
if err := env.m.RunSharesTier2(); err != nil {
|
||
t.Fatalf("the run must continue for healthy shares: %v", err)
|
||
}
|
||
for _, call := range env.mirrored {
|
||
if strings.Contains(call, deadPath) {
|
||
t.Errorf("dead-mount share reached a mirror call: %s", call)
|
||
}
|
||
}
|
||
var sawLive bool
|
||
for _, dst := range env.mirroredDsts() {
|
||
if strings.HasSuffix(dst, string(filepath.Separator)+"elo") {
|
||
sawLive = true
|
||
}
|
||
}
|
||
if !sawLive {
|
||
t.Error("the healthy share must still be mirrored")
|
||
}
|
||
}
|
||
|
||
// Zero shares / sharing disabled: a clean no-op — no dest tree, no status record, no empty dirs.
|
||
func TestSharesTier2ZeroSharesIsCleanNoOp(t *testing.T) {
|
||
env := newSharesEnv(t, "hdd_1", "hdd_2")
|
||
if err := env.m.RunSharesTier2(); err != nil {
|
||
t.Fatalf("RunSharesTier2: %v", err)
|
||
}
|
||
if len(env.mirrored) != 0 {
|
||
t.Errorf("expected no mirror calls, got %v", env.mirrored)
|
||
}
|
||
if cd := env.m.SharesTier2Status(); cd != nil {
|
||
t.Errorf("expected no status record for a feature nobody uses, got %+v", cd)
|
||
}
|
||
destBase := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", SharesPseudoStack)
|
||
if _, err := os.Stat(destBase); !os.IsNotExist(err) {
|
||
t.Errorf("expected no _shares dest tree, stat err = %v", err)
|
||
}
|
||
}
|
||
|
||
// A share removed from the registry must stop occupying the secondary drive within ONE run — the
|
||
// reconcile pass prunes it, reusing the same destBase-bounded removal guard the app path uses.
|
||
func TestSharesTier2ReconcilePrunesRemovedShare(t *testing.T) {
|
||
env := newSharesEnv(t, "hdd_1", "hdd_2")
|
||
env.addShare(t, "hdd_1", "marad", true)
|
||
env.addShare(t, "hdd_1", "torolt", true)
|
||
if err := env.m.RunSharesTier2(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
key := sharesDriveKey(env.drives["hdd_1"])
|
||
destBase := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", SharesPseudoStack)
|
||
stale := filepath.Join(destBase, key, "torolt")
|
||
if _, err := os.Stat(stale); err != nil {
|
||
t.Fatalf("precondition: %s should exist after the first run: %v", stale, err)
|
||
}
|
||
|
||
if err := env.sett.RemoveSMBShare("torolt"); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := env.m.RunSharesTier2(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := os.Stat(stale); !os.IsNotExist(err) {
|
||
t.Errorf("removed share's dest dir was not pruned: stat err = %v", err)
|
||
}
|
||
if _, err := os.Stat(filepath.Join(destBase, key, "marad")); err != nil {
|
||
t.Errorf("surviving share was pruned by mistake: %v", err)
|
||
}
|
||
}
|
||
|
||
// The tier-2 half of the B′ isolation proof: the shares job must write ONLY under
|
||
// backups/secondary/_shares. A per-app dest tree standing beside it must come out byte-for-byte
|
||
// untouched — same contents, same bytes — and no mirror call may target it. Red-proof: point the
|
||
// shares destBase at backups/secondary/<share> (dropping the _shares segment) and this fails.
|
||
func TestSharesTier2LeavesPerAppTreeUntouched(t *testing.T) {
|
||
env := newSharesEnv(t, "hdd_1", "hdd_2")
|
||
env.addShare(t, "hdd_1", "dokumentumok", true)
|
||
|
||
// A pre-existing per-app tier-2 dest with a sentinel payload.
|
||
appDest := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", "immich")
|
||
if err := os.MkdirAll(filepath.Join(appDest, "recovery-unit"), 0o755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
sentinel := filepath.Join(appDest, "recovery-unit", "manifest.json")
|
||
if err := os.WriteFile(sentinel, []byte(`{"app":"immich"}`), 0o644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
if err := env.m.RunSharesTier2(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
b, err := os.ReadFile(sentinel)
|
||
if err != nil || string(b) != `{"app":"immich"}` {
|
||
t.Errorf("B′ INVARIANT VIOLATED — the per-app tier-2 tree was modified: %q, %v", b, err)
|
||
}
|
||
for _, call := range env.mirrored {
|
||
if strings.Contains(call, appDest) {
|
||
t.Errorf("B′ INVARIANT VIOLATED — a shares mirror targeted the per-app dest: %s", call)
|
||
}
|
||
}
|
||
// And everything it DID write lives under the reserved subtree.
|
||
sharesRoot := filepath.Join(NamespaceRoot(env.drives["hdd_2"], true), "backups", "secondary", SharesPseudoStack)
|
||
for _, dst := range env.mirroredDsts() {
|
||
if !strings.HasPrefix(dst, sharesRoot+string(filepath.Separator)) {
|
||
t.Errorf("shares job wrote outside its reserved subtree: %s", dst)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Drive keys must be collision-free across drives that share a basename.
|
||
func TestSharesDriveKeyIsCollisionFree(t *testing.T) {
|
||
if sharesDriveKey("/mnt/a/data") == sharesDriveKey("/mnt/b/data") {
|
||
t.Error("drives with the same basename must not map to the same dest key")
|
||
}
|
||
for _, bad := range []string{"/", "\x00", " "} {
|
||
if k := sharesDriveKey(bad); strings.ContainsAny(k, `/\`) {
|
||
t.Errorf("sharesDriveKey(%q) = %q — must be a single safe path segment", bad, k)
|
||
}
|
||
}
|
||
}
|
||
|
||
// runtimeIsPOSIX reports whether file-mode assertions are meaningful on this host (Windows reports
|
||
// synthesised permissions, so mode checks there are noise rather than signal).
|
||
func runtimeIsPOSIX() bool { return os.PathSeparator == '/' }
|