From 68f0e0cf5c3db386ba9933070d3209adcc1676be Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Tue, 14 Jul 2026 17:45:53 +0200 Subject: [PATCH] F-S2 + F-S3: compose-derived appdata dir resolution (v0.131.0) The controller assumed an app's HDD appdata dir is always appdata/. paperless-ngx writes appdata/paperless (stack paperless-ngx), so every consumer keying by stack name silently missed it via a stat-and-skip. One canonical resolver appbackup.AppDataDirNames derives the real dir name(s) from the app's compose ${HDD_PATH} binds; all consumers use it. - F-S2 (tier-2): RunTier2 mirrors the resolved appdata/ (paperless docs got NO tier-2 copy before). Tier2Info size + RestoreTier2Files live dir use it. WARN when a declared appdata dir is absent. New tier2Mirror seam. - F-S3 (migrate, NEW): all six per-app appdata legs (collision/size/copy/verify/ cleanup/skip-set) now loop resolved names. scope="app" migration of paperless previously copied nothing and left an empty media dir (scope="all" was saved by the merge walk). WARN on missing declared dir in the copy leg. - Multi-dir (N>1) refusal: tier-2 backup/info/restore refuse loudly (Hungarian); migrate supports N. No catalog app hits it today; lifted by Task 3. - Display: storage page sums resolved dirs. - Truth repair: the v0.130.0 "tier-2 copies the namespace wholesale" claim is false; corrected in CHANGELOG + main.go export-adapter comment. +9 tests; red-proofs RP-1..RP-5 all confirmed. Controller-only, no agent/hub coupling. Task 1 of the backup-classification-redesign arc. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01A45Qop8YY8tS94bz63LFne --- CHANGELOG.md | 43 ++++ CONTEXT.md | 23 +- REUSE.md | 5 +- controller/README.md | 16 +- controller/cmd/controller/main.go | 7 +- .../appbackup/appdatadirnames_test.go | 104 +++++++++ controller/internal/appbackup/paths.go | 62 ++++- .../internal/backup/appbackup_bridge.go | 8 + controller/internal/backup/backup.go | 5 + controller/internal/backup/tier2.go | 86 ++++++- .../internal/backup/tier2_appdata_test.go | 215 ++++++++++++++++++ controller/internal/backup/tier2_restore.go | 12 +- .../internal/backup/tier2_restore_test.go | 3 +- controller/internal/stacks/migrate.go | 84 +++++-- .../internal/stacks/migrate_fs3_test.go | 104 +++++++++ controller/internal/web/handlers.go | 32 ++- 16 files changed, 765 insertions(+), 44 deletions(-) create mode 100644 controller/internal/appbackup/appdatadirnames_test.go create mode 100644 controller/internal/backup/tier2_appdata_test.go create mode 100644 controller/internal/stacks/migrate_fs3_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 01e2db2..3f8b896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,48 @@ ## Changelog +### v0.131.0 — F-S2 + F-S3: compose-derived appdata dir resolution (paperless-ngx → appdata/paperless) (2026-07-14) + +The controller assumed an app's HDD appdata dir is always `appdata/`. paperless-ngx binds +`${HDD_PATH}/appdata/paperless/...` — stack `paperless-ngx`, dir `paperless` — so every consumer that +keyed by stack name silently missed it via a stat-and-skip. One canonical resolver +(`appbackup.AppDataDirNames`) now derives the real dir name(s) from the app's compose `${HDD_PATH}` +binds, and all consumers use it. Task 1 of the backup-classification-redesign arc +(`felhom.eu/documentation/audits/SPIKE-backup-classification-2026-07-14.md`), deliberately independent +of the classification schema. + +- **F-S2 (spike-proven live) — tier-2 backup/info/restore.** `RunTier2` now mirrors the resolved + `appdata/` dir, so paperless documents get their off-drive copy (previously: the appdata leg's + `os.Stat` gate skipped `appdata/paperless-ngx`, which never existed — **silent, no copy**). + `Tier2Info`'s size + the SSD-headroom guard use the resolved dir. `RestoreTier2Files` targets the + resolved live dir (was restoring into a wrong/empty `appdata/paperless-ngx`). A `[WARN]` now fires + when the compose DECLARES an appdata dir but it is absent on disk (the silence that hid F-S2). Every + rsync leg goes through a new `tier2Mirror` seam (prod behavior unchanged). +- **F-S3 (NEW, found this session) — scope="app" migration.** `migrate.go` keyed all six per-app + appdata legs (collision check, source-size, copy, verify, cleanup, skip-set) by stack name. For + **scope="app"** there is no merge walk, so migrating paperless-ngx copied nothing, "verified" + vacuously, flipped `HDD_PATH`, and the app came up with an **empty media dir**. (scope="all" was + saved by the merge walk — data safe, accounting off.) All six legs now loop the resolved name(s); + the copy leg WARNs on a missing declared dir. +- **Multi-dir refusal (defensive; no catalog app hits it today).** An app resolving to N>1 distinct + appdata dirs is refused loudly by tier-2 backup/info (honest `no_target` status + + `"az alkalmazáshoz több adatkönyvtár tartozik — a 2. mentés jelenleg alkalmazásonként egy könyvtárat + támogat"`) and tier-2 file-restore (refused BEFORE the app is stopped). Migrate supports N naturally. + This limitation is lifted by the tier-policy engine (Task 3). +- **Display.** The storage-detail page sums the resolved appdata dir(s), so paperless-ngx shows a + non-empty size. +- **Truth repair.** The v0.130.0 entry below states "The scheduled/tier-2 backup path was NOT affected + (it copies the felhom-data namespace wholesale)" — **that sentence is false** and is left in place + only as the historical record it corrects here: tier-2 copies the recovery unit + the resolved + `appdata/` dir(s) ONLY (never the userdata tree — F-S1, unaddressed here — and never the + namespace wholesale). The `main.go` export-adapter comment that repeated the claim is fixed in code. + +Scope guards: destination layout unchanged (`/appdata` stays flat); no userdata copying at +any tier (F-S1 is the classification redesign's, not this task's); `ExportDataMounts` / `.fab` / +offbox untouched. Tests: +9 (resolver table incl. dedupe/foreign-drive/whole-root; RunTier2 +paperless/legacy/multi-dir; Tier2Info + restore refusals + resolved live dir; scope="app" paperless +migration). Red-proofs RP-1..RP-5 all confirmed (resolver, RunTier2 leg, restore dst, migrate copy +leg, N>1 guard). Controller-only; no agent/hub coupling; MinAgent unchanged. + ### v0.130.0 — CRITICAL C6B-F1: hollow .fab export (three compounding defects) + C6B-F2 share-removal guard (2026-07-14) CAMPAIGN-6B surfaced that `.fab` export produced a **config-only, data-free bundle** for 12/13 diff --git a/CONTEXT.md b/CONTEXT.md index c74eb68..3f6c801 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -7,7 +7,28 @@ > > Ask Claude Code: "Please update CONTEXT.md with what we did today" -Last updated: 2026-07-14 (v0.130.0 — CRITICAL C6B-F1 hollow-.fab-export fix + C6B-F2 share-removal guard) +Last updated: 2026-07-14 (v0.131.0 — F-S2 + F-S3 compose-derived appdata dir resolution) + +> **2026-07-14 — v0.131.0: F-S2 + F-S3 (compose-derived appdata dir resolution).** Task 1 of the +> backup-classification-redesign arc (spike: `felhom.eu/documentation/audits/SPIKE-backup-classification-2026-07-14.md`). +> The controller assumed `appdata/`; paperless-ngx writes `appdata/paperless` (stack +> `paperless-ngx`). ONE canonical resolver `appbackup.AppDataDirNames(hddPath, stackName, mounts)` +> derives the real dir name(s) from compose `${HDD_PATH}` binds (deduped/sorted; fallback `[stackName]`); +> all consumers use it. **F-S2** (spike-proven): `RunTier2`/`Tier2Info`/`RestoreTier2Files` now hit the +> resolved dir (paperless documents got NO tier-2 copy before — the appdata leg stat-skipped a dir that +> never existed). **F-S3 (NEW, found this session):** `migrate.go` keyed all six per-app appdata legs by +> stack name; **scope="app"** has no merge walk, so migrating paperless-ngx copied nothing, verified +> vacuously, flipped HDD_PATH → **empty media dir** (scope="all" was saved by the merge walk — data safe, +> accounting off). All six legs now loop resolved names. **Multi-dir (N>1) refusal** is defensive (no +> catalog app hits it today: immich/nextcloud/romm match, paperless mismatches, each app = exactly ONE +> dir): tier-2 backup/info/restore refuse loudly (Hungarian); **migrate supports N naturally**. This +> limitation is **deferred to Task 3 (tier-policy engine)**, which owns the destination layout. Storage +> page sums resolved dirs. Truth repairs: the v0.130.0 CHANGELOG/CONTEXT "tier-2 copies the namespace +> wholesale" claim is FALSE — corrected in the v0.131.0 CHANGELOG entry + `main.go` export-adapter +> comment; tier-2 copies the recovery unit + resolved `appdata/` ONLY (NOT userdata — F-S1, +> unaddressed here). New seam `tier2Mirror`; `migSeams.resolveNames`. +9 tests, RP-1..RP-5 all +> confirmed. Controller-only, no agent/hub coupling. **NOT live-validated here:** scope="app" migration +> of a real app between drives (F-S3 live proof — supervised leg, Viktor's session). > **2026-07-14 — v0.130.0: CRITICAL C6B-F1 (hollow .fab export) + C6B-F2 (share-removal guard).** > CAMPAIGN-6B proved `.fab` export shipped **config-only, data-free bundles** for 12/13 `needs_hdd` diff --git a/REUSE.md b/REUSE.md index f0caa1c..607b025 100644 --- a/REUSE.md +++ b/REUSE.md @@ -12,7 +12,8 @@ |---|---|---|---|---| | `NamespaceRoot` | controller/internal/appbackup/paths.go | `(drivePath string, inGuestDrive bool) string` | Resolve felhom-data root for a drive | `inGuestDrive=true` returns path AS-IS (Model A: guest mount IS the ns root); false appends `felhom-data`. Never double-nest | | `PrimaryBackupPath` / `RecoveryUnitPath` / `RecoveryUnitComposePath` / `RecoveryUnitManifestPath` | controller/internal/appbackup/paths.go | `(nsRoot[, stackName]) string` | All backup dir layout | Take the NAMESPACE ROOT, not a bare drive path | -| `AppDBDumpPath` / `AppVolumeDumpPath` / `AppDataDir` | controller/internal/appbackup/paths.go | `(nsRoot, stackName) string` | Per-app dump/data dirs | Same nsRoot contract | +| `AppDBDumpPath` / `AppVolumeDumpPath` / `AppDataDir` | controller/internal/appbackup/paths.go | `(nsRoot, stackName) string` | Per-app dump/data dirs | Same nsRoot contract. `AppDataDir`'s final segment is the app's real appdata dir NAME — NOT always the stack name (paperless-ngx → `paperless`); resolve via `AppDataDirNames` first (F-S2/F-S3) | +| `AppDataDirNames` / `AppDataBindsPresent` | controller/internal/appbackup/paths.go | `(hddPath, stackName string, hddMounts []string) []string` / `(hddPath, hddMounts) bool` | Resolve the real `appdata/` dir(s) from compose `${HDD_PATH}` binds (F-S2/F-S3) | `hddMounts` = ParseComposeHDDMounts shape. Deduped+sorted; falls back to `[stackName]` when no appdata bind. Tier-2 (`backup.Manager.tier2AppDataName`) refuses N>1; migrate (`stacks.Manager.ResolveAppDataDirNames`) loops N. `BindsPresent` drives the WARN-on-missing-declared-dir | | `UserdataDir` / `EnsureUserdataSkeleton` / `EnsureDirOwned` | controller/internal/appbackup/userdata.go | `(nsRoot)` / `(path, gid int)` | userdata/ tree w/ 2775 setgid gid-1000 convention | Linux-only chown via build-tag twin userdata_linux.go | | `HumanizeBytes` | controller/internal/appbackup/appdata.go | `(b int64) string` | Human byte sizes | Exported canonical; private clones exist (§6) | | `stablePathForName` / `agentWhere` | controller/internal/web/intermediary.go | `(name/registeredPath) string` | Map registry stable path `/mnt/felhom-drives/` ↔ raw agent mount | Registry stores STABLE path; agent ops take the RAW mount — always convert | @@ -199,6 +200,8 @@ | `dumpVolumesSafe` (func seam) | controller/internal/backup/backup.go | nil → real `DumpAppVolumesSafe` | injected in controller/internal/backup/volume_dumps_test.go (gating tests without Docker) | | `generateSecret` (func seam) | controller/internal/backup/backup.go | `stacks.Manager.GenerateSecretForField` via `SetSecretGenerator` (main.go) | injected in controller/internal/backup/restore_secrets_gen_test.go | | `restoreFilesCopier` (func seam) | controller/internal/backup/backup.go | nil → real `rsyncRestoreMissing` | injected in controller/internal/backup/tier2_restore_test.go (orchestration without rsync) | +| `tier2Mirror` (func seam) | controller/internal/backup/backup.go | nil → real `rsyncMirror` | both RunTier2 rsync legs; injected in controller/internal/backup/tier2_appdata_test.go (resolve→mirror without rsync) | +| `migSeams.resolveNames` (func seam) | controller/internal/stacks/migrate.go | nil → real `ResolveAppDataDirNames` (compose-derived) | injected in controller/internal/stacks/migrate_fs3_test.go (F-S3 appdata dir-name resolution) | Cross-repo edges: - `controller/internal/agentapi/client.go` ↔ **felhom-agent** local API (`/storage`, `/disks*`, `/backup*`, `/netstorage*`, `/guest/*`): pinned leaf SHA-256 + per-guest bearer token from bootstrap.json. diff --git a/controller/README.md b/controller/README.md index fe0d726..f0bda42 100644 --- a/controller/README.md +++ b/controller/README.md @@ -476,7 +476,7 @@ Path computation is centralized in `backup/paths.go` via the `FelhomDataDir = "f - `PrimaryResticRepoPath(drivePath)` → `/felhom-data/backups/primary/restic/` - `AppDBDumpPath(drivePath, stackName)` → `/felhom-data/backups/primary//db-dumps/` - `AppVolumeDumpPath(drivePath, stackName)` → `/felhom-data/backups/primary//volume-dumps/` -- `AppDataDir(drivePath, stackName)` → `/felhom-data/appdata//` +- `AppDataDir(drivePath, name)` → `/felhom-data/appdata//` (final segment is the app's real appdata dir NAME, resolved via `AppDataDirNames` from compose binds — NOT always the stack name; F-S2) - `SecondaryResticRepoPath(drivePath)` → `/felhom-data/backups/secondary/restic/` - `AppSecondaryRsyncPath(drivePath, stackName)` → `/felhom-data/backups/secondary//rsync/` - `SecondaryInfraPath(drivePath)` → `/felhom-data/backups/secondary/_infra/` @@ -529,9 +529,14 @@ backups/primary// #### Tier 2 — off-drive copy (Phase 3, v0.55.x) For every HDD app, Tier 2 (`internal/backup/tier2.go`) rsync-mirrors the recovery unit -(`backups/primary//`) + the app's `appdata//` to `/backups/secondary//` on a -**different physical disk** — the only off-drive protection bind-mounted HDD userdata can get (PBS can't -reach bind mounts). Auto-targeted: **prefer another registered user-data drive** (off-disk via +(`backups/primary//`) + the app's resolved `appdata//` to `/backups/secondary//` +on a **different physical disk** — the only off-drive protection bind-mounted HDD app data can get (PBS +can't reach bind mounts). **The appdata dir NAME is derived from the app's compose `${HDD_PATH}` binds, +not assumed to be the stack name** (F-S2, v0.131.0: paperless-ngx writes `appdata/paperless`; +`tier2AppDataName` → `appbackup.AppDataDirNames`); an app resolving to >1 distinct appdata dir is +refused loudly. This copies the recovery unit + `appdata/` ONLY — **not** the browsable +`userdata/` tree (F-S1, owned by the backup-classification redesign) and **not** the namespace +wholesale. Auto-targeted: **prefer another registered user-data drive** (off-disk via `system.SamePhysicalDevice`); else the **internal SSD for small units only**, behind a size-aware **rootfs-headroom guard** (`tier2FitsHeadroom`) that **refuses rather than fills** the ~8 GB guest rootfs (reserve = `max(2 GB, 20%)`), recording an honest "needs a 2nd HDD" status. Status persists via @@ -541,7 +546,8 @@ reach bind mounts). Auto-targeted: **prefer another registered user-data drive** **In-place file restore from the Tier-2 copy (C2, v0.100.0 — closes drill finding F2)** — `POST /backup/tier2/restore` (`backup.RestoreTier2Files`, `internal/backup/tier2_restore.go`) + the **"Fájlok visszaállítása"** button on the healthy Tier-2 layer row. **Additive-only** semantics -(`rsyncRestoreMissing`: `rsync -a --ignore-existing`): files missing from the live `appdata/` +(`rsyncRestoreMissing`: `rsync -a --ignore-existing`): files missing from the live resolved +`appdata/` dir (F-S2 — compose-derived, not the stack name) are copied back from the RECORDED Tier-2 copy; existing live files are **never overwritten** (a customer edit after the last copy wins) and **nothing is ever deleted** — this exactly serves the "I deleted my files" scenario with zero risk to newer data. Source = the recorded diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index d887180..28b1c0c 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -1357,8 +1357,11 @@ func (a *exportAdapter) GetStackHDDMounts(name string) []string { // C6B-F1 (v0.130.0): union ${HDD_PATH} binds + the ${USERDATA_PATH} root. The old // ParseComposeHDDMounts-only call was blind to the standard userdata convention, so // 12/13 needs_hdd catalog apps exported hollow (config-only) bundles. The backup-side - // stackAdapter is intentionally NOT changed — the scheduled/tier-2 path copies the - // felhom-data namespace wholesale and its mount list is size/inventory display only. + // stackAdapter is intentionally NOT changed here — the scheduled/tier-2 path copies the + // recovery unit + the app's resolved appdata/ dir(s) only (NOT the userdata tree — + // F-S1; NOT the namespace wholesale), and derives that dir name from the compose binds + // (F-S2, see backup.tier2AppDataName). Spec: + // felhom.eu/documentation/audits/SPIKE-backup-classification-2026-07-14.md. return stacks.ExportDataMounts(s.ComposePath, appCfg.Env["HDD_PATH"]) } return nil diff --git a/controller/internal/appbackup/appdatadirnames_test.go b/controller/internal/appbackup/appdatadirnames_test.go new file mode 100644 index 0000000..381f914 --- /dev/null +++ b/controller/internal/appbackup/appdatadirnames_test.go @@ -0,0 +1,104 @@ +package appbackup + +import ( + "path/filepath" + "reflect" + "testing" +) + +// fp joins the elements under an HDD path with OS separators — mounts in the ParseComposeHDDMounts +// shape are already filepath.Clean'd, so tests build them the same way. +func fp(elems ...string) string { return filepath.Join(elems...) } + +// TestAppDataDirNames is the pure derivation table (Group A). Every case asserts the RESOLVED name +// list, never mere absence of error. Companion RP-1: a resolver that ignores mounts and returns +// []string{stackName} fails the paperless, two-name, and dedupe cases. +func TestAppDataDirNames(t *testing.T) { + const hdd = "/mnt/felhom-usb" + cases := []struct { + name string + stack string + mounts []string + want []string + }{ + { + // paperless shape: stack "paperless-ngx", dir "paperless" (F-S2/F-S3 core). + name: "paperless mismatch", + stack: "paperless-ngx", + mounts: []string{ + fp(hdd, "appdata", "paperless", "media"), + fp(hdd, "appdata", "paperless", "export"), + }, + want: []string{"paperless"}, // media+export dedupe to one name + }, + { + // match shape: dir name == stack name (immich/nextcloud/romm). + name: "matching name", + stack: "nextcloud", + mounts: []string{fp(hdd, "appdata", "nextcloud")}, + want: []string{"nextcloud"}, + }, + { + // two DISTINCT names → both, sorted (no catalog app does this today). + name: "two distinct names sorted", + stack: "weird", + mounts: []string{ + fp(hdd, "appdata", "zebra", "x"), + fp(hdd, "appdata", "alpha", "y"), + }, + want: []string{"alpha", "zebra"}, + }, + { + // non-appdata HDD binds + a foreign-drive mount are filtered → fallback. + name: "non-appdata and foreign filtered", + stack: "romm", + mounts: []string{ + fp(hdd, "roms"), // under HDD but not appdata/ + fp("/mnt/other-drive", "appdata", "ghost"), // foreign drive — wrong prefix + }, + want: []string{"romm"}, + }, + { + // whole-appdata-root bind (no name derivable) → ignored → fallback. + name: "whole appdata root bind", + stack: "root-binder", + mounts: []string{fp(hdd, "appdata")}, + want: []string{"root-binder"}, + }, + { + name: "empty mounts fallback", + stack: "vaultwarden", + mounts: nil, + want: []string{"vaultwarden"}, + }, + { + // unclean paths still resolve (Clean applied both sides). + name: "unclean path", + stack: "paperless-ngx", + mounts: []string{hdd + "/appdata/paperless/../paperless/media"}, + want: []string{"paperless"}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := AppDataDirNames(hdd, tc.stack, tc.mounts) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("AppDataDirNames(%q, %q, %v) = %v, want %v", hdd, tc.stack, tc.mounts, got, tc.want) + } + }) + } +} + +// TestAppDataBindsPresent pins the WARN predicate: true only when a mount sits under appdata/. +func TestAppDataBindsPresent(t *testing.T) { + const hdd = "/mnt/felhom-usb" + if !AppDataBindsPresent(hdd, []string{fp(hdd, "appdata", "paperless", "media")}) { + t.Error("declared appdata bind should report present") + } + if AppDataBindsPresent(hdd, []string{fp(hdd, "roms")}) { + t.Error("non-appdata bind should NOT report present") + } + if AppDataBindsPresent(hdd, nil) { + t.Error("no mounts should NOT report present") + } +} diff --git a/controller/internal/appbackup/paths.go b/controller/internal/appbackup/paths.go index aa0ecfe..ad5e961 100644 --- a/controller/internal/appbackup/paths.go +++ b/controller/internal/appbackup/paths.go @@ -5,7 +5,11 @@ // cross-drive, or drive-mount code in the backup package. package appbackup -import "path/filepath" +import ( + "path/filepath" + "sort" + "strings" +) // FelhomDataDir is the namespace directory on storage drives for all felhom-managed data. const FelhomDataDir = "felhom-data" @@ -64,7 +68,61 @@ func AppVolumeDumpPath(nsRoot, stackName string) string { return filepath.Join(RecoveryUnitPath(nsRoot, stackName), "volume-dumps") } -// AppDataDir returns the app data directory under a felhom-data namespace root. +// AppDataDir returns the app data directory under a felhom-data namespace root. The final segment +// is the app's real appdata dir NAME — usually the stack name, but NOT always: paperless-ngx writes +// appdata/paperless (F-S2/F-S3). Callers that key by stack name silently miss such apps; use +// AppDataDirNames to resolve the real name(s) from the app's compose binds and pass them here. func AppDataDir(nsRoot, stackName string) string { return filepath.Join(nsRoot, "appdata", stackName) } + +// AppDataDirNames returns the app's real directory name(s) under /appdata, derived from its +// compose HDD bind mounts (F-S2/F-S3: the dir name is NOT always the stack name — paperless-ngx +// writes appdata/paperless). hddMounts are resolved host paths in the ParseComposeHDDMounts shape +// (each is itself or a subpath, filepath.Clean'd). The first path element under +// /appdata/ is taken as the dir name; results are deduped and sorted. Falls back to +// []string{stackName} when no appdata-prefixed mount is derivable (no HDD appdata binds, unreadable +// compose, nil provider) — the exact legacy behavior. +// +// Today every catalog app resolves to exactly ONE name (immich→immich, nextcloud→nextcloud, +// romm→romm, paperless-ngx→paperless). The N>1 return is defensive: tier-2 refuses it loudly, +// migrate handles it naturally. +func AppDataDirNames(hddPath, stackName string, hddMounts []string) []string { + prefix := filepath.Clean(hddPath) + string(filepath.Separator) + "appdata" + string(filepath.Separator) + seen := make(map[string]bool) + var names []string + for _, mnt := range hddMounts { + cm := filepath.Clean(mnt) + if !strings.HasPrefix(cm, prefix) { + continue // not under appdata/ (a whole-root bind, a different subtree, a foreign drive) + } + rem := strings.TrimPrefix(cm, prefix) + first := strings.Split(rem, string(filepath.Separator))[0] + if first == "" { + continue + } + if !seen[first] { + seen[first] = true + names = append(names, first) + } + } + if len(names) == 0 { + return []string{stackName} + } + sort.Strings(names) + return names +} + +// AppDataBindsPresent reports whether any of the app's resolved HDD mounts sits under +// /appdata/ — i.e. the compose actually DECLARES an appdata bind. Callers use it to +// distinguish "no appdata to back up" (silent skip is correct) from "declared appdata dir missing +// on disk" (the silence that hid F-S2 — worth a WARN). Same prefix rule as AppDataDirNames. +func AppDataBindsPresent(hddPath string, hddMounts []string) bool { + prefix := filepath.Clean(hddPath) + string(filepath.Separator) + "appdata" + string(filepath.Separator) + for _, mnt := range hddMounts { + if strings.HasPrefix(filepath.Clean(mnt), prefix) { + return true + } + } + return false +} diff --git a/controller/internal/backup/appbackup_bridge.go b/controller/internal/backup/appbackup_bridge.go index 5399d4e..60f0c59 100644 --- a/controller/internal/backup/appbackup_bridge.go +++ b/controller/internal/backup/appbackup_bridge.go @@ -133,3 +133,11 @@ func RecoveryUnitManifestPath(nsRoot, stackName string) string { func AppDataDir(nsRoot, stackName string) string { return appbackup.AppDataDir(nsRoot, stackName) } + +func AppDataDirNames(hddPath, stackName string, hddMounts []string) []string { + return appbackup.AppDataDirNames(hddPath, stackName, hddMounts) +} + +func AppDataBindsPresent(hddPath string, hddMounts []string) bool { + return appbackup.AppDataBindsPresent(hddPath, hddMounts) +} diff --git a/controller/internal/backup/backup.go b/controller/internal/backup/backup.go index faa4560..0ddc426 100644 --- a/controller/internal/backup/backup.go +++ b/controller/internal/backup/backup.go @@ -65,6 +65,11 @@ type Manager struct { // the orchestration never shells out. Nil → the real rsyncRestoreMissing (additive-only). restoreFilesCopier func(src, dst string) (filesRestored int, err error) + // tier2Mirror (F-S2) — the Tier-2 backup mirror seam (both rsync legs in RunTier2), overridable + // so the resolve→mirror→record flow is unit-testable without rsync. Nil → the real rsyncMirror + // (`-a --delete`, contents-of-src semantics). + tier2Mirror func(src, dst string) error + // migrationRunning, if set, reports whether a data migration is in progress. The scheduled // backup paths skip when it returns true (Change 3 — backup ↔ migration mutual exclusion), so a // nightly dump/Tier-2 can't race a migration copy/cleanup on the same drive. diff --git a/controller/internal/backup/tier2.go b/controller/internal/backup/tier2.go index beac6b1..9a4fd71 100644 --- a/controller/internal/backup/tier2.go +++ b/controller/internal/backup/tier2.go @@ -14,21 +14,59 @@ import ( "gitea.dooplex.hu/admin/felhom-controller/internal/system" ) -// Tier 2 = an off-drive (different physical disk) copy of an HDD app's recovery unit + bulk userdata. -// It is the ONLY off-drive protection that browsable HDD userdata can get — PBS can't reach bind -// mounts. Auto-enabled for every HDD app; the target is auto-picked: prefer another registered -// user-data drive (can hold bulk), else the internal SSD for SMALL units only — and the SSD is the -// guest rootfs (~8 GB), so we REFUSE rather than fill it (a size-aware headroom guard). When no -// off-drive target fits, we record an honest "needs a 2nd HDD" status instead of silently doing -// nothing useful. +// Tier 2 = an off-drive (different physical disk) copy of an HDD app's recovery unit + its resolved +// appdata/ dir(s). It does NOT copy the browsable userdata tree (F-S1: userdata is not backed +// up at any tier yet — that gap is owned by the classification redesign, see +// felhom.eu/documentation/audits/SPIKE-backup-classification-2026-07-14.md). The appdata dir NAME is +// derived from the app's compose binds, NOT assumed to be the stack name (F-S2: paperless-ngx writes +// appdata/paperless); see tier2AppDataName. Auto-enabled for every HDD app; the target is auto-picked: +// prefer another registered user-data drive (can hold bulk), else the internal SSD for SMALL units +// only — and the SSD is the guest rootfs (~8 GB), so we REFUSE rather than fill it (a size-aware +// headroom guard). When no off-drive target fits, we record an honest "needs a 2nd HDD" status +// instead of silently doing nothing useful. const gibibyte = 1024 * 1024 * 1024 var ( errNoOffDiskTarget = errors.New("no off-drive target (single drive, app already on the system disk)") errSSDNoHeadroom = errors.New("the internal SSD lacks headroom for this app's data — a 2nd drive is required for off-drive backup") + // errTier2MultiDir is raised when an app's compose resolves to MORE THAN ONE distinct appdata + // dir under /appdata (no catalog app does today). Tier 2's destination layout is flat + // (/appdata), so it refuses rather than silently collapse two source dirs into one. + errTier2MultiDir = errors.New("az alkalmazáshoz több adatkönyvtár tartozik — a 2. mentés jelenleg alkalmazásonként egy könyvtárat támogat") ) +// appDataDirNames resolves the app's real appdata dir name(s) under hddPath from its compose HDD +// binds, via the stack provider (nil provider → legacy [stackName] fallback). See +// appbackup.AppDataDirNames. +func (m *Manager) appDataDirNames(stackName, hddPath string) []string { + var mounts []string + if m.stackProvider != nil { + mounts = m.stackProvider.GetStackHDDMounts(stackName) + } + return AppDataDirNames(hddPath, stackName, mounts) +} + +// tier2AppDataName resolves the SINGLE appdata dir name for tier-2's flat destination. N>1 distinct +// names → errTier2MultiDir (the one place the tier-2 multi-dir refusal is built). It always returns +// at least one name from appDataDirNames' fallback, so name is meaningful only when err == nil. +func (m *Manager) tier2AppDataName(stackName, hddPath string) (string, error) { + names := m.appDataDirNames(stackName, hddPath) + if len(names) > 1 { + return "", errTier2MultiDir + } + return names[0], nil +} + +// tier2AppDataBindsPresent reports whether the app's compose declares an appdata bind (drives the +// WARN-on-missing-declared-dir rule; nil provider → false). +func (m *Manager) tier2AppDataBindsPresent(stackName, hddPath string) bool { + if m.stackProvider == nil { + return false + } + return AppDataBindsPresent(hddPath, m.stackProvider.GetStackHDDMounts(stackName)) +} + // Tier2Target is a resolved off-drive destination for an app's Tier 2 copy. type Tier2Target struct { NamespaceRoot string // felhom-data namespace root on the target drive @@ -143,7 +181,16 @@ func (m *Manager) RunTier2(stackName string) error { } sourceNsRoot := m.namespaceRoot(sourceDrive) unitDir := RecoveryUnitPath(sourceNsRoot, stackName) - appDataDir := AppDataDir(sourceNsRoot, stackName) + // F-S2: resolve the app's REAL appdata dir name from its compose binds (paperless-ngx writes + // appdata/paperless, not appdata/paperless-ngx). For an HDD app HDD_PATH == nsRoot (Model A), so + // the mounts (resolved against HDD_PATH) share the nsRoot prefix. N>1 distinct names → refuse. + appDataName, resErr := m.tier2AppDataName(stackName, sourceNsRoot) + if resErr != nil { + m.recordTier2NoTarget(stackName, resErr.Error()) + m.logger.Printf("[ERROR] [backup] Tier 2 for %s refused: %v", stackName, resErr) + return nil + } + appDataDir := AppDataDir(sourceNsRoot, appDataName) if _, err := os.Stat(unitDir); err != nil { return nil // no recovery unit yet — nothing to copy } @@ -166,7 +213,12 @@ func (m *Manager) RunTier2(stackName string) error { destBase := filepath.Join(target.NamespaceRoot, "backups", "secondary", stackName) start := time.Now() - if err := rsyncMirror(unitDir, filepath.Join(destBase, "recovery-unit")); err != nil { + mirror := m.tier2Mirror + if mirror == nil { + mirror = rsyncMirror + } + + if err := mirror(unitDir, filepath.Join(destBase, "recovery-unit")); err != nil { m.recordTier2Failure(stackName, target, err) if m.tier2Notify != nil { m.tier2Notify(stackName, target.Label, time.Since(start), err) @@ -174,13 +226,18 @@ func (m *Manager) RunTier2(stackName string) error { return fmt.Errorf("tier2 rsync unit for %s: %w", stackName, err) } if _, e := os.Stat(appDataDir); e == nil { - if err := rsyncMirror(appDataDir, filepath.Join(destBase, "appdata")); err != nil { + if err := mirror(appDataDir, filepath.Join(destBase, "appdata")); err != nil { m.recordTier2Failure(stackName, target, err) if m.tier2Notify != nil { m.tier2Notify(stackName, target.Label, time.Since(start), err) } return fmt.Errorf("tier2 rsync appdata for %s: %w", stackName, err) } + } else if m.tier2AppDataBindsPresent(stackName, sourceNsRoot) { + // F-S2: the compose DECLARES an appdata bind but the dir is missing on disk. Skipping is kept + // (nothing to copy) but the silence that hid F-S2 for months is now a loud WARN. + m.logger.Printf("[WARN] [backup] Tier 2 for %s: compose declares appdata dir %q but it is absent at %s — appdata leg skipped", + stackName, appDataName, appDataDir) } dur := time.Since(start) @@ -279,8 +336,15 @@ func (m *Manager) Tier2Info(stackName string) Tier2Info { } // Resolve what the runner WOULD pick right now (real unit size feeds the SSD headroom guard). + // F-S2: N>1 distinct appdata dirs → the same honest refusal the runner records. sourceNsRoot := m.namespaceRoot(source) - unitSize := dirSizeBytes(RecoveryUnitPath(sourceNsRoot, stackName)) + dirSizeBytes(AppDataDir(sourceNsRoot, stackName)) + appDataName, resErr := m.tier2AppDataName(stackName, sourceNsRoot) + if resErr != nil { + info.NoTarget = true + info.NoTargetReason = resErr.Error() + return info + } + unitSize := dirSizeBytes(RecoveryUnitPath(sourceNsRoot, stackName)) + dirSizeBytes(AppDataDir(sourceNsRoot, appDataName)) target, err := m.selectTier2Target(stackName, unitSize) if err != nil { info.NoTarget = true diff --git a/controller/internal/backup/tier2_appdata_test.go b/controller/internal/backup/tier2_appdata_test.go new file mode 100644 index 0000000..adf0bd4 --- /dev/null +++ b/controller/internal/backup/tier2_appdata_test.go @@ -0,0 +1,215 @@ +package backup + +import ( + "errors" + "io" + "log" + "path/filepath" + "testing" + + "gitea.dooplex.hu/admin/felhom-controller/internal/config" + "gitea.dooplex.hu/admin/felhom-controller/internal/settings" +) + +// newRunTier2Manager builds a Manager wired to a fake provider, a real temp-dir source drive holding +// a recovery unit + an appdata/ dir (created iff appDataName != ""), and a (non-existent) +// schedulable off-drive target so selectTier2Target resolves a real target without touching the disk +// (SamePhysicalDevice returns false for an unstattable path on both Linux and Windows). Compose +// mounts default to appdata//media + /export; callers override fake.mounts for the +// legacy/multi-dir shapes. The tier2Mirror seam captures the (src,dst) of each leg. +func newRunTier2Manager(t *testing.T, stack, appDataName string) (m *Manager, src string, captured *[][2]string) { + t.Helper() + tmp := t.TempDir() + src = filepath.Join(tmp, "usb") // the source drive == HDD_PATH == namespace root (Model A) + sysPath := filepath.Join(tmp, "sys") + target := filepath.Join(tmp, "off-drive-target") // never created → treated as a different device + + sett, err := settings.Load(filepath.Join(tmp, "settings.json"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + if err := sett.AddStoragePath(settings.StoragePath{Path: target, Label: "off", Schedulable: true}); err != nil { + t.Fatal(err) + } + + // A recovery unit (gates RunTier2) + the resolved appdata dir, both with real bytes. + mustWrite(t, filepath.Join(RecoveryUnitPath(src, stack), "manifest.json"), "{}") + var mounts []string + if appDataName != "" { + mustWrite(t, filepath.Join(AppDataDir(src, appDataName), "media", "a.jpg"), "JPEGDATA") + mounts = []string{ + filepath.Join(src, "appdata", appDataName, "media"), + filepath.Join(src, "appdata", appDataName, "export"), + } + } + + fake := &t2rFakeProvider{hdd: src, mounts: mounts} + cfg := &config.Config{} + cfg.Paths.SystemDataPath = sysPath + m = NewManager(cfg, sett, log.New(io.Discard, "", 0)) + m.stackProvider = fake + m.systemDataPath = sysPath + + pairs := &[][2]string{} + m.tier2Mirror = func(s, d string) error { + *pairs = append(*pairs, [2]string{s, d}) + return nil + } + return m, src, pairs +} + +// TestRunTier2_PaperlessShape (Scenario A / RP-2): the appdata leg is mirrored from the REAL +// compose-derived dir (appdata/paperless), and the recorded size includes its bytes. Companion +// RP-2: reverting the L146 site to AppDataDir(nsRoot, stackName) makes the appdata leg mirror a +// non-existent dir — the appdata capture below (dst ".../appdata") never fires. +func TestRunTier2_PaperlessShape(t *testing.T) { + m, srcDrive, captured := newRunTier2Manager(t, "paperless-ngx", "paperless") + + if err := m.RunTier2("paperless-ngx"); err != nil { + t.Fatalf("RunTier2: %v", err) + } + + // The appdata leg must mirror appdata/paperless → /appdata. + wantSrc := AppDataDir(srcDrive, "paperless") + var appdataLeg *[2]string + for i := range *captured { + if filepath.Base((*captured)[i][1]) == "appdata" { + appdataLeg = &(*captured)[i] + } + } + if appdataLeg == nil { + t.Fatalf("appdata leg was never mirrored (F-S2 regression); captured=%v", *captured) + } + if appdataLeg[0] != wantSrc { + t.Errorf("appdata mirror src = %q, want the resolved dir %q", appdataLeg[0], wantSrc) + } + if filepath.Base(appdataLeg[1]) != "appdata" { + t.Errorf("appdata mirror dst = %q, want flat /appdata", appdataLeg[1]) + } + // Recorded success size must include the paperless bytes (non-empty, > 0). + cd := m.settings.GetCrossDriveConfig("paperless-ngx") + if cd == nil || cd.LastStatus != "ok" { + t.Fatalf("expected recorded ok status, got %+v", cd) + } + if cd.LastSizeHuman == "" || cd.LastSizeHuman == "0 B" { + t.Errorf("recorded size = %q, want it to include the appdata bytes", cd.LastSizeHuman) + } +} + +// TestRunTier2_LegacyShape (Scenario B): a match-name app AND a no-binds app both mirror the exact +// same src the pre-fix code keyed by stack name — byte-identical behavior. +func TestRunTier2_LegacyShape(t *testing.T) { + t.Run("matching name (nextcloud)", func(t *testing.T) { + m, srcDrive, captured := newRunTier2Manager(t, "nextcloud", "nextcloud") + m.stackProvider.(*t2rFakeProvider).mounts = []string{filepath.Join(srcDrive, "appdata", "nextcloud")} + if err := m.RunTier2("nextcloud"); err != nil { + t.Fatalf("RunTier2: %v", err) + } + assertAppdataSrc(t, captured, AppDataDir(srcDrive, "nextcloud")) + }) + t.Run("no appdata binds (fallback == stack name)", func(t *testing.T) { + // The appdata dir is created under the stack name; mounts are cleared → resolver falls back. + m, srcDrive, captured := newRunTier2Manager(t, "vaultwarden", "vaultwarden") + m.stackProvider.(*t2rFakeProvider).mounts = nil + if err := m.RunTier2("vaultwarden"); err != nil { + t.Fatalf("RunTier2: %v", err) + } + assertAppdataSrc(t, captured, AppDataDir(srcDrive, "vaultwarden")) + }) +} + +func assertAppdataSrc(t *testing.T, captured *[][2]string, want string) { + t.Helper() + for _, p := range *captured { + if filepath.Base(p[1]) == "appdata" { + if p[0] != want { + t.Errorf("appdata mirror src = %q, want %q", p[0], want) + } + return + } + } + t.Fatalf("appdata leg not mirrored; captured=%v", *captured) +} + +// TestRunTier2_MultiDirRefusal (Scenario C / RP-5): two distinct appdata dirs → an honest no_target +// status with the EXACT Hungarian reason, an [ERROR] log, and NO mirror call. Companion RP-5: +// deleting the N>1 guard in tier2AppDataName makes the mirror fire (call-count > 0). +func TestRunTier2_MultiDirRefusal(t *testing.T) { + m, srcDrive, captured := newRunTier2Manager(t, "twodir", "alpha") + m.stackProvider.(*t2rFakeProvider).mounts = []string{ + filepath.Join(srcDrive, "appdata", "alpha", "x"), + filepath.Join(srcDrive, "appdata", "beta", "y"), + } + if err := m.RunTier2("twodir"); err != nil { + t.Fatalf("RunTier2 must record a status, not error: %v", err) + } + if n := len(*captured); n != 0 { + t.Errorf("mirror was called %d time(s) on a multi-dir refusal — want 0", n) + } + cd := m.settings.GetCrossDriveConfig("twodir") + if cd == nil || cd.LastStatus != "no_target" { + t.Fatalf("expected no_target status, got %+v", cd) + } + if cd.LastError != errTier2MultiDir.Error() { + t.Errorf("reason = %q, want %q", cd.LastError, errTier2MultiDir.Error()) + } +} + +// TestTier2Info_MultiDirRefusal (Scenario C, info tier): the config-panel view surfaces the same +// refusal reason. +func TestTier2Info_MultiDirRefusal(t *testing.T) { + m, srcDrive, _ := newRunTier2Manager(t, "twodir", "alpha") + m.stackProvider.(*t2rFakeProvider).mounts = []string{ + filepath.Join(srcDrive, "appdata", "alpha", "x"), + filepath.Join(srcDrive, "appdata", "beta", "y"), + } + info := m.Tier2Info("twodir") + if !info.NoTarget { + t.Fatal("expected NoTarget on a multi-dir app") + } + if info.NoTargetReason != errTier2MultiDir.Error() { + t.Errorf("reason = %q, want %q", info.NoTargetReason, errTier2MultiDir.Error()) + } +} + +// TestRestoreTier2Files_ResolvedLiveDir (Scenario E / RP-3): the copier's dst is the RESOLVED live +// dir (appdata/paperless), not appdata/. Companion RP-3: reverting liveDir to stack-name +// keying makes dst appdata/paperless-ngx and this fails. +func TestRestoreTier2Files_ResolvedLiveDir(t *testing.T) { + m, fake, liveDrive, _ := newT2RManager(t) + fake.mounts = []string{ + filepath.Join(liveDrive, "appdata", "paperless", "media"), + } + var gotDst string + m.restoreFilesCopier = func(_, dst string) (int, error) { gotDst = dst; return 0, nil } + + if _, err := m.RestoreTier2Files("app"); err != nil { + t.Fatalf("restore: %v", err) + } + if want := AppDataDir(liveDrive, "paperless"); gotDst != want { + t.Errorf("restore dst = %q, want resolved live dir %q", gotDst, want) + } +} + +// TestRestoreTier2Files_MultiDirRefusal (Scenario C, restore tier): two dirs → errTier2MultiDirRestore +// BEFORE the app is stopped (effect assertion: StopStack never invoked). +func TestRestoreTier2Files_MultiDirRefusal(t *testing.T) { + m, fake, liveDrive, _ := newT2RManager(t) + fake.mounts = []string{ + filepath.Join(liveDrive, "appdata", "alpha", "x"), + filepath.Join(liveDrive, "appdata", "beta", "y"), + } + called := false + m.restoreFilesCopier = func(string, string) (int, error) { called = true; return 0, nil } + + _, err := m.RestoreTier2Files("app") + if !errors.Is(err, errTier2MultiDirRestore) { + t.Fatalf("err = %v, want errTier2MultiDirRestore", err) + } + if len(fake.stopped) != 0 { + t.Errorf("app was STOPPED on a multi-dir refusal: %v", fake.stopped) + } + if called { + t.Error("copier invoked on a refusal") + } +} diff --git a/controller/internal/backup/tier2_restore.go b/controller/internal/backup/tier2_restore.go index 38dcfbc..c15f17d 100644 --- a/controller/internal/backup/tier2_restore.go +++ b/controller/internal/backup/tier2_restore.go @@ -29,6 +29,9 @@ var ( errTier2DriveGone = errors.New("a másodlagos meghajtó nincs csatlakoztatva") errLiveDriveGone = errors.New("az alkalmazás meghajtója nincs csatlakoztatva") errLiveDriveDecommed = errors.New("az alkalmazás meghajtója le van szerelve") + // errTier2MultiDirRestore (F-S2): the app resolves to more than one appdata dir, which the flat + // tier-2 copy layout does not represent — refused BEFORE the app is stopped. + errTier2MultiDirRestore = errors.New("az alkalmazáshoz több adatkönyvtár tartozik — a fájl-visszaállítás jelenleg nem támogatott") ) // RestoreTier2Files restores the app's MISSING user files in place from its recorded Tier-2 copy @@ -60,7 +63,14 @@ func (m *Manager) RestoreTier2Files(stackName string) (filesRestored int, err er return 0, fmt.Errorf("%w (%s)", errLiveDriveDecommed, drive) } } - liveDir := AppDataDir(m.namespaceRoot(drive), stackName) + // F-S2: the live appdata dir is the app's REAL compose-derived dir (paperless-ngx → paperless), + // not the stack name. N>1 distinct dirs → refuse here, BEFORE the app is stopped. + liveNsRoot := m.namespaceRoot(drive) + appDataName, resErr := m.tier2AppDataName(stackName, liveNsRoot) + if resErr != nil { + return 0, errTier2MultiDirRestore + } + liveDir := AppDataDir(liveNsRoot, appDataName) // Source side: the RECORDED Tier-2 copy must exist and its drive must be connected. var srcDir string diff --git a/controller/internal/backup/tier2_restore_test.go b/controller/internal/backup/tier2_restore_test.go index bf29669..02fc17f 100644 --- a/controller/internal/backup/tier2_restore_test.go +++ b/controller/internal/backup/tier2_restore_test.go @@ -17,6 +17,7 @@ import ( // the NON-effect (never stopped) and the happy path asserts the stop→copy→start order. type t2rFakeProvider struct { hdd string + mounts []string // F-S2: configurable compose-derived HDD mounts (drives appdata dir-name resolution) stopped []string started []string order []string // interleaved event log: "stop", "copy" (appended by the copier seam), "start" @@ -24,7 +25,7 @@ type t2rFakeProvider struct { func (f *t2rFakeProvider) GetStackComposePath(string) (string, bool) { return "", false } func (f *t2rFakeProvider) ListDeployedStacks() []StackSummary { return nil } -func (f *t2rFakeProvider) GetStackHDDMounts(string) []string { return nil } +func (f *t2rFakeProvider) GetStackHDDMounts(string) []string { return f.mounts } func (f *t2rFakeProvider) GetStackHDDPath(string) string { return f.hdd } func (f *t2rFakeProvider) GetDockerVolumes(string) []string { return nil } func (f *t2rFakeProvider) StopStack(name string) error { diff --git a/controller/internal/stacks/migrate.go b/controller/internal/stacks/migrate.go index 0617a40..bd1499e 100644 --- a/controller/internal/stacks/migrate.go +++ b/controller/internal/stacks/migrate.go @@ -139,6 +139,7 @@ type migSeams struct { verify func(ctx context.Context, src, dst string) error stop func(name string) error flipRedeploy func(name, target string) error + resolveNames func(app string) []string // F-S3: override appdata dir-name resolution in tests } // SetMigrationDeps wires the registry + the backup-running check (mutual exclusion, Change 3). @@ -291,6 +292,39 @@ func (m *Manager) appSourceNS(cfg *AppConfig) (src, ns string) { return src, appbackup.NamespaceRoot(src, m.inGuest(src)) } +// resolveAppDataDirNames returns an app's real appdata dir name(s) under its HDD_PATH, derived from +// its compose binds (F-S2/F-S3: paperless-ngx writes appdata/paperless, not appdata/paperless-ngx). +// Mirrors the stackAdapter Priority-1 shape (app's own HDD_PATH + ParseComposeHDDMounts); the +// Priority-2 multi-storage union is NOT needed here — migration/tier-2 only apply to real-HDD apps. +// A test seam (testSeams.resolveNames) overrides it so the phase machine can be driven without a +// live stack registry. Falls back to []string{app} when nothing is derivable (legacy behavior). +// ResolveAppDataDirNames returns an app's real appdata dir name(s) under its HDD_PATH (F-S2/F-S3), +// for callers outside the migrate flow (the storage-page size display). Exported thin wrapper. +func (m *Manager) ResolveAppDataDirNames(app string) []string { + names, _ := m.resolveAppDataDirs(app) + return names +} + +// resolveAppDataDirs also reports whether the app's compose DECLARES an appdata bind (declared) — +// migCopy uses it to WARN when a declared dir is missing on disk (the silence that hid F-S2), while +// distinguishing that from an app that genuinely has no appdata (fallback, no WARN). +func (m *Manager) resolveAppDataDirs(app string) (names []string, declared bool) { + if m.testSeams != nil && m.testSeams.resolveNames != nil { + return m.testSeams.resolveNames(app), true + } + stack, ok := m.GetStack(app) + if !ok { + return []string{app}, false + } + cfg := LoadAppConfig(filepath.Dir(stack.ComposePath)) + if cfg == nil || cfg.Env["HDD_PATH"] == "" { + return []string{app}, false + } + hddPath := cfg.Env["HDD_PATH"] + mounts := ParseComposeHDDMounts(stack.ComposePath, hddPath) + return appbackup.AppDataDirNames(hddPath, app, mounts), appbackup.AppDataBindsPresent(hddPath, mounts) +} + // appsOnDrive returns the names of deployed apps whose HDD_PATH equals sourcePath. func (m *Manager) appsOnDrive(sourcePath string) []string { var out []string @@ -322,11 +356,14 @@ func (m *Manager) migValidate(j *MigrationJob) error { if m.settings == nil || !m.settings.IsStoragePathSchedulable(j.Target) { return fmt.Errorf("a céltároló nem elérhető vagy nem választható") } - // App-dir collision: refuse if the same app dir already exists at the target. + // App-dir collision: refuse if any of the app's resolved appdata dir(s) already exists at target. var collide []string for _, app := range j.Apps { - if pathExists(appbackup.AppDataDir(j.TargetNS, app)) { - collide = append(collide, app) + for _, name := range m.ResolveAppDataDirNames(app) { + if pathExists(appbackup.AppDataDir(j.TargetNS, name)) { + collide = append(collide, app) + break + } } } if len(collide) > 0 { @@ -351,7 +388,9 @@ func (m *Manager) migSourceSize(j *MigrationJob) int64 { } var total int64 for _, app := range j.Apps { - total += dirBytes(appbackup.AppDataDir(j.SourceNS, app)) + for _, name := range m.ResolveAppDataDirNames(app) { + total += dirBytes(appbackup.AppDataDir(j.SourceNS, name)) + } total += dirBytes(appbackup.RecoveryUnitPath(j.SourceNS, app)) } return total @@ -445,10 +484,21 @@ func (m *Manager) migCopy(ctx context.Context, j *MigrationJob) error { } j.CurrentApp = app _ = m.persistJob(j) - // appdata subtree (collision-free post-validate) - if err := m.copySubtree(ctx, j, appbackup.AppDataDir(j.SourceNS, app), appbackup.AppDataDir(j.TargetNS, app)); err != nil { - u.Error = err.Error() - return fmt.Errorf("másolás sikertelen (%s appdata): %w", app, err) + // appdata subtree(s) — the app's REAL compose-derived dir name(s) (F-S3: paperless-ngx writes + // appdata/paperless, not appdata/paperless-ngx); collision-free post-validate. + names, declared := m.resolveAppDataDirs(app) + for _, name := range names { + src := appbackup.AppDataDir(j.SourceNS, name) + if declared && !pathExists(src) { + // The compose DECLARES this appdata dir but it is absent on disk — WARN instead of the + // silent no-op that hid F-S2. copySubtree still no-ops safely below. + m.logger.Printf("[WARN] [migrate] %s: compose declares appdata dir %q but it is absent at %s — nothing to copy", + app, name, src) + } + if err := m.copySubtree(ctx, j, src, appbackup.AppDataDir(j.TargetNS, name)); err != nil { + u.Error = err.Error() + return fmt.Errorf("másolás sikertelen (%s appdata): %w", app, err) + } } // the app's recovery unit (db-dumps + volume-dumps + compose + manifest) if err := m.copySubtree(ctx, j, appbackup.RecoveryUnitPath(j.SourceNS, app), appbackup.RecoveryUnitPath(j.TargetNS, app)); err != nil { @@ -488,9 +538,11 @@ func (m *Manager) migVerify(ctx context.Context, j *MigrationJob) error { if stateRank(u.State) >= stateRank(UnitVerified) { continue } - if err := m.verifySubtree(ctx, appbackup.AppDataDir(j.SourceNS, app), appbackup.AppDataDir(j.TargetNS, app)); err != nil { - u.Error = err.Error() - return fmt.Errorf("ellenőrzés sikertelen (%s appdata): %w", app, err) + for _, name := range m.ResolveAppDataDirNames(app) { + if err := m.verifySubtree(ctx, appbackup.AppDataDir(j.SourceNS, name), appbackup.AppDataDir(j.TargetNS, name)); err != nil { + u.Error = err.Error() + return fmt.Errorf("ellenőrzés sikertelen (%s appdata): %w", app, err) + } } if err := m.verifySubtree(ctx, appbackup.RecoveryUnitPath(j.SourceNS, app), appbackup.RecoveryUnitPath(j.TargetNS, app)); err != nil { u.Error = err.Error() @@ -553,8 +605,10 @@ func (m *Manager) migCleanup(j *MigrationJob) error { if u.State == UnitCleaned { continue } - if err := os.RemoveAll(appbackup.AppDataDir(j.SourceNS, app)); err != nil { - return fmt.Errorf("forrás törlése sikertelen (%s appdata): %w", app, err) + for _, name := range m.ResolveAppDataDirNames(app) { + if err := os.RemoveAll(appbackup.AppDataDir(j.SourceNS, name)); err != nil { + return fmt.Errorf("forrás törlése sikertelen (%s appdata): %w", app, err) + } } if err := os.RemoveAll(appbackup.RecoveryUnitPath(j.SourceNS, app)); err != nil { return fmt.Errorf("forrás törlése sikertelen (%s mentés): %w", app, err) @@ -602,7 +656,9 @@ func (m *Manager) migCleanupAllowed(j *MigrationJob) error { func (m *Manager) appDataSkipSet(j *MigrationJob) map[string]bool { skip := map[string]bool{} for _, app := range j.Apps { - skip[filepath.Clean(appbackup.AppDataDir(j.SourceNS, app))] = true + for _, name := range m.ResolveAppDataDirNames(app) { + skip[filepath.Clean(appbackup.AppDataDir(j.SourceNS, name))] = true + } } return skip } diff --git a/controller/internal/stacks/migrate_fs3_test.go b/controller/internal/stacks/migrate_fs3_test.go new file mode 100644 index 0000000..f3e7656 --- /dev/null +++ b/controller/internal/stacks/migrate_fs3_test.go @@ -0,0 +1,104 @@ +package stacks + +import ( + "context" + "path/filepath" + "testing" + + "gitea.dooplex.hu/admin/felhom-controller/internal/appbackup" +) + +// newAppJob builds a scope="app" job (no non-app merge unit — app scope has no merge walk). +func newAppJob(srcNS, dstNS, source, target string, apps ...string) *MigrationJob { + j := &MigrationJob{ + Scope: "app", Phase: PhaseStop, Source: source, Target: target, + SourceNS: srcNS, TargetNS: dstNS, Apps: apps, Units: map[string]*MigUnit{}, + } + for _, a := range apps { + j.Units[a] = &MigUnit{App: a, State: UnitPending} + } + return j +} + +// TestMigrate_PaperlessShape_ScopeApp is Scenario D / RP-4: a scope="app" migration of paperless-ngx +// (stack name) must copy, verify, and clean the REAL appdata dir (paperless), never appdata/paperless-ngx. +// Every leg is asserted by captured src/dst pairs and on-disk effects — never mere absence of error. +// Companion RP-4: dropping the resolveNames loop (keying by stack name) makes the copy seam receive +// appdata/paperless-ngx and every assertion below fails. +func TestMigrate_PaperlessShape_ScopeApp(t *testing.T) { + m := newMigManager(t, "/mnt/target") + srcNS := t.TempDir() + dstNS := t.TempDir() + // Real source data under the REAL dir name (paperless), plus the recovery unit. + writeFile(t, filepath.Join(appbackup.AppDataDir(srcNS, "paperless"), "media", "doc.pdf"), "PDF") + writeFile(t, filepath.Join(appbackup.RecoveryUnitPath(srcNS, "paperless-ngx"), "manifest.json"), "{}") + + type pair struct{ src, dst string } + var copied, verified []pair + m.testSeams = &migSeams{ + resolveNames: func(app string) []string { + if app == "paperless-ngx" { + return []string{"paperless"} + } + return []string{app} + }, + stop: func(string) error { return nil }, + copy: func(_ context.Context, s, d string, _ func(int64)) error { + copied = append(copied, pair{s, d}) + return nil + }, + verify: func(_ context.Context, s, d string) error { + verified = append(verified, pair{s, d}) + return nil + }, + flipRedeploy: func(string, string) error { return nil }, + } + + // --- direct-call assertions (accounting legs) before the destructive run --- + wantSrc := appbackup.AppDataDir(srcNS, "paperless") + // migSourceSize must probe appdata/paperless (non-zero: the PDF exists there). + if got := m.migSourceSize(newAppJob(srcNS, dstNS, "/s", "/t", "paperless-ngx")); got == 0 { + t.Error("migSourceSize = 0 — it probed appdata/paperless-ngx (empty) instead of appdata/paperless") + } + // appDataSkipSet must contain the resolved source dir. + skip := m.appDataSkipSet(newAppJob(srcNS, dstNS, "/s", "/t", "paperless-ngx")) + if !skip[filepath.Clean(wantSrc)] { + t.Errorf("skip-set = %v, want it to contain %q", skip, filepath.Clean(wantSrc)) + } + // Collision check must probe appdata/paperless at the target. Target must be the schedulable path + // so migValidate reaches the collision check (TargetNS stays the temp dir where the dir exists). + writeFile(t, filepath.Join(appbackup.AppDataDir(dstNS, "paperless"), "x"), "x") + cj := newAppJob(srcNS, dstNS, "/s", "/mnt/target", "paperless-ngx") + if err := m.migValidate(cj); err == nil || !contains(err.Error(), "paperless-ngx") { + t.Errorf("collision must fire on the resolved target dir, got %v", err) + } + + // --- full pipeline run (copy → verify → cleanup) --- + j := newAppJob(srcNS, dstNS, "/s", "/t", "paperless-ngx") + m.runJobSync(j) + if j.Phase != PhaseDone { + t.Fatalf("phase = %s (err=%s), want done", j.Phase, j.Error) + } + + wantDst := appbackup.AppDataDir(dstNS, "paperless") + assertHasPair := func(name string, got []pair) { + t.Helper() + for _, p := range got { + if p.src == wantSrc && p.dst == wantDst { + return + } + } + t.Errorf("%s seam never saw appdata/paperless: src=%q dst=%q, got %v", name, wantSrc, wantDst, got) + } + assertHasPair("copy", copied) + assertHasPair("verify", verified) + + // Cleanup removed the REAL source dir. + if pathExists(wantSrc) { + t.Errorf("cleanup left the source appdata/paperless behind: %s", wantSrc) + } + // And it never probed / created appdata/paperless-ngx. + if pathExists(appbackup.AppDataDir(srcNS, "paperless-ngx")) { + t.Errorf("stale appdata/paperless-ngx dir should never exist") + } +} diff --git a/controller/internal/web/handlers.go b/controller/internal/web/handlers.go index accde10..64973b8 100644 --- a/controller/internal/web/handlers.go +++ b/controller/internal/web/handlers.go @@ -1675,18 +1675,28 @@ func (s *Server) appDetailsForPath(storagePath string) []StorageAppDetail { Name: stack.Meta.DisplayName, Stack: stack.Meta.Slug, } - // Try to get data size from the storage subdirectory - appDataDir := backup.AppDataDir(storagePath, stack.Name) - if fi, err := os.Stat(appDataDir); err == nil && fi.IsDir() { - detail.SizeHuman = dirSizeHuman(appDataDir) + // Try to get data size from the storage subdirectory. F-S2: the app's real appdata dir name is + // NOT always the stack name (paperless-ngx writes appdata/paperless) — sum the resolved dir(s). + // Here hddPath == storagePath (the drive's in-guest mount is the namespace root, Model A). + var total int64 + var any bool + for _, name := range s.stackMgr.ResolveAppDataDirNames(stack.Name) { + d := backup.AppDataDir(storagePath, name) + if fi, err := os.Stat(d); err == nil && fi.IsDir() { + total += dirSizeBytesWalk(d) + any = true + } + } + if any { + detail.SizeHuman = humanizeDirBytes(total) } details = append(details, detail) } return details } -// dirSizeHuman returns a human-readable size for a directory. -func dirSizeHuman(path string) string { +// dirSizeBytesWalk returns the total size in bytes of the regular files under path (0 if absent). +func dirSizeBytesWalk(path string) int64 { var total int64 filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { @@ -1695,6 +1705,11 @@ func dirSizeHuman(path string) string { total += info.Size() return nil }) + return total +} + +// humanizeDirBytes formats a byte count as B/KB/MB/GB (the storage-page display convention). +func humanizeDirBytes(total int64) string { const ( KB = 1024 MB = KB * 1024 @@ -1712,6 +1727,11 @@ func dirSizeHuman(path string) string { } } +// dirSizeHuman returns a human-readable size for a single directory. +func dirSizeHuman(path string) string { + return humanizeDirBytes(dirSizeBytesWalk(path)) +} + func formatFreeSpace(gb float64) string { if gb >= 1000 { return fmt.Sprintf("%.1f TB", gb/1024)