v0.172.0 — R-75: canonical import root, catalog-derived skeleton, import surfaces

${IMPORT_PATH} = <system namespace root>/userdata/import — ONE drop-zone per box,
on the system drive, injected at BOTH compose-env builders with NO per-drive
fallback (unresolvable leaves it unset so compose fails loudly rather than
quietly building a second, dead drop-zone).

Third BindRoot (RootImport) + Import list in BackupSpec, extended through
ValidateBackupSpec/ClassifyBinds. Load-bearing: a stale `userdata: import/<app>`
entry against the moved bind would be a WHOLE-BLOCK reject, taking the app's
mandatory hdd classification with it.

Exhaustive-root audit: resolveAbs/structuralGuard/ComputeCaptureSet/
ComputeFabBuckets now take importRoot explicitly (an import bind resolved
against hddPath would name a directory on the wrong drive); unresolvable is
refused loudly into Skipped. GetImportRoot added to both provider interfaces.

Catalog-derived skeleton: UserdataSkeleton() -> UserdataSkeletonCarry() +
BuildUserdataSkeleton(), SORTED. The carry-list makes zero-removals true by
construction (`documents` is in no catalog app but on both boxes) and is the
fresh-box floor. The sort is not tidiness: the naive map-order derivation
measured 20 distinct outputs from 20 identical runs, which with fbNeedsRecreate
is a fleet-wide FileBrowser restart loop.

One authoritative compose parser: ParseComposeUserdataMounts now delegates to
ParseComposeClassifiableBinds. Import root excluded from per-app migration.

Surfaces: FileBrowser /srv/beolvasas source; app-page "Hova tegyem a fajlokat?"
with PathEscape deep links (never QueryEscape) and class-driven copy;
data_paths: annotation with the Fork-3 asymmetry; system-owned beolvasas SMB
share refused server-side at handler AND store, button omitted in template.

Caught on the way: the sharing template's row struct was function-local, so
adding {{if .System}} would have 500'd every share row. ShareRow is now
package-level and the render test uses the handler's own type.

Tests 915 -> 949, all green. MinAgent unchanged.
This commit is contained in:
2026-07-26 08:12:57 +02:00
parent 3b672ba74c
commit 2958946517
57 changed files with 2228 additions and 151 deletions
+5 -1
View File
@@ -19,7 +19,11 @@ type StackDataProvider interface {
GetStackComposePath(name string) (composePath string, ok bool)
ListDeployedStacks() []StackSummary
GetStackHDDMounts(name string) []string
GetStackHDDPath(name string) string // raw HDD_PATH from app.yaml (empty if no HDD)
GetStackHDDPath(name string) string // raw HDD_PATH from app.yaml (empty if no HDD)
// GetImportRoot returns the CANONICAL drop-zone root (R-75): <system namespace root>/userdata/import.
// It is app-INDEPENDENT and lives on the SYSTEM drive, so ${IMPORT_PATH} binds cannot be resolved
// from GetStackHDDPath. Empty when unresolvable — structuralGuard refuses such binds loudly.
GetImportRoot() string
GetDockerVolumes(name string) []string // full Docker volume names (project-prefixed)
StopStack(name string) error
StartStack(name string) error
+35 -13
View File
@@ -59,6 +59,8 @@ const (
reasonEscape = "path escapes the drive root"
reasonBareRoot = "bare drive-root bind would capture the backups tree"
reasonReserved = "path inside the reserved backups zone"
// reasonNoImportRoot: a ${IMPORT_PATH} bind with no resolvable system namespace root (R-75).
reasonNoImportRoot = "canonical import root unresolvable (system_data_path unconfigured)"
)
// ComputeCaptureSet resolves an app's classified binds into the tier-filtered absolute capture set.
@@ -71,16 +73,17 @@ const (
// so the engines' no-block branch stays byte-identical to today (the SQ5 cost-regression guard).
//
// Resolution: RootHDD → path.Join(hddPath, relPath); RootUserdata → path.Join(hddPath, "userdata",
// relPath). Guards run AFTER the tier filter, so Skipped means exactly "would have been captured by
// this tier, refused for structural safety".
func ComputeCaptureSet(binds []ClassifiedBind, hasClassification bool, tier CaptureTier, hddPath string) CaptureSet {
// relPath); RootImport → path.Join(importRoot, relPath) — the SYSTEM drive, never hddPath (R-75).
// Guards run AFTER the tier filter, so Skipped means exactly "would have been captured by this tier,
// refused for structural safety".
func ComputeCaptureSet(binds []ClassifiedBind, hasClassification bool, tier CaptureTier, hddPath, importRoot string) CaptureSet {
if !hasClassification {
return CaptureSet{HasClassification: false}
}
cs := CaptureSet{HasClassification: true}
// Stages 13: tier filter → structural guards → equal-Abs collapse (shared with ComputeFabBuckets).
uniq, skipped := resolveGuardCollapse(binds, hddPath, func(c BindClass) bool { return tierKeeps(tier, c) })
uniq, skipped := resolveGuardCollapse(binds, hddPath, importRoot, func(c BindClass) bool { return tierKeeps(tier, c) })
cs.Skipped = skipped
// Stage 4: containment dedup — drop any path whose ancestor is already present (keep the ancestor).
@@ -107,18 +110,18 @@ func ComputeCaptureSet(binds []ClassifiedBind, hasClassification bool, tier Capt
// collapse (mandatory > optional > excluded; ties by smaller Root/RelPath). It does NOT apply
// containment dedup — the caller decides (ComputeCaptureSet does; ComputeFabBuckets must not, so a
// mandatory child inside an excluded parent stays independently addressable).
func resolveGuardCollapse(binds []ClassifiedBind, hddPath string, keep func(BindClass) bool) (uniq []CapturePath, skipped []SkippedPath) {
func resolveGuardCollapse(binds []ClassifiedBind, hddPath, importRoot string, keep func(BindClass) bool) (uniq []CapturePath, skipped []SkippedPath) {
var resolved []CapturePath
for _, b := range binds {
if !keep(b.Class) {
continue
}
if reason, bad := structuralGuard(b.Root, b.RelPath); bad {
if reason, bad := structuralGuard(b.Root, b.RelPath, importRoot); bad {
skipped = append(skipped, SkippedPath{Root: b.Root, RelPath: b.RelPath, Class: b.Class, Reason: reason})
continue
}
resolved = append(resolved, CapturePath{
Abs: resolveAbs(hddPath, b.Root, b.RelPath), Root: b.Root, RelPath: b.RelPath, Class: b.Class,
Abs: resolveAbs(hddPath, importRoot, b.Root, b.RelPath), Root: b.Root, RelPath: b.RelPath, Class: b.Class,
})
}
byAbs := make(map[string]CapturePath, len(resolved))
@@ -153,12 +156,12 @@ type FabBuckets struct {
// selection UI + plan. Same resolution + structural guards + equal-Abs collapse as ComputeCaptureSet
// (via resolveGuardCollapse), bucketed by class, no cross-bucket containment dedup. Each bucket is
// Abs-sorted (deterministic).
func ComputeFabBuckets(binds []ClassifiedBind, hasClassification bool, hddPath string) FabBuckets {
func ComputeFabBuckets(binds []ClassifiedBind, hasClassification bool, hddPath, importRoot string) FabBuckets {
if !hasClassification {
return FabBuckets{HasClassification: false}
}
fb := FabBuckets{HasClassification: true}
uniq, skipped := resolveGuardCollapse(binds, hddPath, func(BindClass) bool { return true })
uniq, skipped := resolveGuardCollapse(binds, hddPath, importRoot, func(BindClass) bool { return true })
fb.Skipped = skipped
for _, cp := range uniq {
switch cp.Class {
@@ -201,10 +204,19 @@ func tierKeeps(tier CaptureTier, class BindClass) bool {
// there (ParseComposeClassifiableBinds path.Cleans; ValidateBackupSpec vets only SPEC entries), so an
// unlisted writable "${HDD_PATH}/../x" bind reaches here classed mandatory — this guard is
// load-bearing security, not defence-in-depth.
func structuralGuard(root BindRoot, relPath string) (reason string, bad bool) {
func structuralGuard(root BindRoot, relPath, importRoot string) (reason string, bad bool) {
if relPathEscapes(relPath) {
return reasonEscape, true
}
// RootImport (R-75) resolves against the SYSTEM drive, not hddPath. If that root is unresolvable
// (system_data_path unconfigured) the bind cannot be placed at all — refuse it LOUDLY into Skipped
// rather than let resolveAbs join onto "" and produce a relative, wrong-drive path. The other two
// roots cannot hit this: hddPath is checked by their own callers.
if root == RootImport && importRoot == "" {
return reasonNoImportRoot, true
}
// A bare ${IMPORT_PATH} bind is allowed: it resolves to <sysNS>/userdata/import, which nests no
// backups/ tree (backups live at <sysNS>/backups, a sibling of userdata).
if root == RootHDD {
if relPath == "" {
return reasonBareRoot, true // bare ${HDD_PATH} would nest <hddPath>/backups into the capture
@@ -233,11 +245,21 @@ func relPathEscapes(relPath string) bool {
}
// resolveAbs maps a guarded (root, relPath) to its in-container absolute path via slash algebra.
func resolveAbs(hddPath string, root BindRoot, relPath string) string {
if root == RootUserdata {
//
// RootImport is the one root that does NOT resolve against hddPath: the canonical drop-zone lives on
// the SYSTEM drive (R-75), so importRoot is supplied separately by the caller. Resolving it against
// hddPath would silently name a directory on the WRONG DRIVE — a .fab opt-in would then capture (or
// on restore, write) somewhere that merely looks plausible. An empty importRoot is the unresolvable
// case and is refused upstream by structuralGuard, never silently joined.
func resolveAbs(hddPath, importRoot string, root BindRoot, relPath string) string {
switch root {
case RootUserdata:
return path.Join(hddPath, "userdata", relPath)
case RootImport:
return path.Join(importRoot, relPath)
default:
return path.Join(hddPath, relPath)
}
return path.Join(hddPath, relPath)
}
// strongerCapture picks the winner of an equal-Abs collision: mandatory beats optional; on equal
@@ -38,12 +38,12 @@ func TestComputeCaptureSet_PerTierSplit(t *testing.T) {
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media/photos", ReadOnly: true}, Class: ClassOptional, Origin: OriginExplicit},
}
off := ComputeCaptureSet(binds, true, TierOffsite, drv)
off := ComputeCaptureSet(binds, true, TierOffsite, drv, "")
if got, want := absList(off), []string{hdd("appdata/immich")}; !reflect.DeepEqual(got, want) {
t.Errorf("offsite Paths = %v, want %v (mandatory only — the :ro optional must NOT ship offsite)", got, want)
}
sec := ComputeCaptureSet(binds, true, TierSecondary, drv)
sec := ComputeCaptureSet(binds, true, TierSecondary, drv, "")
want := []string{hdd("appdata/immich"), udat("media/photos")}
if got := absList(sec); !reflect.DeepEqual(got, want) {
t.Errorf("secondary Paths = %v, want %v (sorted)", got, want)
@@ -72,7 +72,7 @@ func TestComputeCaptureSet_LegacyInert(t *testing.T) {
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "appdata/sonarr"}, Origin: OriginLegacy},
}
for _, tier := range []CaptureTier{TierOffsite, TierSecondary} {
cs := ComputeCaptureSet(binds, false, tier, drv)
cs := ComputeCaptureSet(binds, false, tier, drv, "")
if cs.HasClassification {
t.Errorf("%s: HasClassification=true for a legacy app", tier)
}
@@ -94,7 +94,7 @@ func TestComputeCaptureSet_ExcludedInvisible(t *testing.T) {
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "import/paperless"}, Class: ClassExcluded, Origin: OriginExplicit},
}
for _, tier := range []CaptureTier{TierOffsite, TierSecondary} {
cs := ComputeCaptureSet(binds, true, tier, drv)
cs := ComputeCaptureSet(binds, true, tier, drv, "")
if got, want := absList(cs), []string{hdd("appdata/paperless/media")}; !reflect.DeepEqual(got, want) {
t.Errorf("%s Paths = %v, want %v (excluded filtered)", tier, got, want)
}
@@ -113,7 +113,7 @@ func TestComputeCaptureSet_StructuralGuards(t *testing.T) {
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "backups/primary/x"}, Class: ClassMandatory, Origin: OriginDefaultWritable}, // d3 reserved zone
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: ""}, Class: ClassMandatory, Origin: OriginDefaultWritable}, // d4 bare userdata — ALLOWED
}
cs := ComputeCaptureSet(binds, true, TierOffsite, drv)
cs := ComputeCaptureSet(binds, true, TierOffsite, drv, "")
// Paths: ONLY d4's userdata root — no escaped root, no backups/ anywhere.
if got, want := absList(cs), []string{udat("")}; !reflect.DeepEqual(got, want) {
@@ -156,7 +156,7 @@ func TestComputeCaptureSet_LegitDotDotName(t *testing.T) {
binds := []ClassifiedBind{
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "appdata/a..b"}, Class: ClassMandatory, Origin: OriginExplicit},
}
cs := ComputeCaptureSet(binds, true, TierOffsite, drv)
cs := ComputeCaptureSet(binds, true, TierOffsite, drv, "")
if got, want := absList(cs), []string{hdd("appdata/a..b")}; !reflect.DeepEqual(got, want) {
t.Errorf("Paths = %v, want %v (a..b is a legit name, not traversal)", got, want)
}
@@ -174,7 +174,7 @@ func TestComputeCaptureSet_ContainmentAndCollision(t *testing.T) {
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "userdata/media"}, Class: ClassOptional, Origin: OriginExplicit}, // Abs collides with next
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media"}, Class: ClassMandatory, Origin: OriginExplicit}, // same Abs, mandatory
}
cs := ComputeCaptureSet(binds, true, TierSecondary, drv)
cs := ComputeCaptureSet(binds, true, TierSecondary, drv, "")
want := []string{hdd("appdata/paperless"), udat("media")}
if got := absList(cs); !reflect.DeepEqual(got, want) {
@@ -186,7 +186,7 @@ func TestComputeCaptureSet_ContainmentAndCollision(t *testing.T) {
}
// determinism: recompute and compare full struct
cs2 := ComputeCaptureSet(binds, true, TierSecondary, drv)
cs2 := ComputeCaptureSet(binds, true, TierSecondary, drv, "")
if !reflect.DeepEqual(cs, cs2) {
t.Error("ComputeCaptureSet is non-deterministic across runs")
}
+54 -16
View File
@@ -34,12 +34,23 @@ type BindRoot string
const (
RootUserdata BindRoot = "userdata" // relative to ${USERDATA_PATH}
RootHDD BindRoot = "hdd" // relative to ${HDD_PATH}
// RootImport is relative to ${IMPORT_PATH} — the CANONICAL drop-zone root (R-75). Unlike the
// other two it does NOT resolve against the app's own drive: it lives on the system drive's
// namespace, so every app's ingest folder is in one place. Resolvers therefore need the import
// root passed in separately; they cannot derive it from hddPath.
RootImport BindRoot = "import"
)
// BackupSpec is the .felhom.yml `backup:` block. Paths are forward-slash, relative, path.Clean'd.
type BackupSpec struct {
Userdata []BindSpec `yaml:"userdata,omitempty" json:"userdata,omitempty"`
HDD []BindSpec `yaml:"hdd,omitempty" json:"hdd,omitempty"`
// Import classifies ${IMPORT_PATH}-relative binds (R-75). An app whose ingest bind moved from
// ${USERDATA_PATH}/import/<app> to ${IMPORT_PATH}/<app> MUST move its backup entry here in the
// same change: ValidateBackupSpec rejects an entry matching no compose bind, and the rejection is
// WHOLE-BLOCK, so a stale `userdata: import/<app>` would discard the app's OTHER classifications
// (e.g. an hdd appdata path classed mandatory) and silently degrade it to legacy.
Import []BindSpec `yaml:"import,omitempty" json:"import,omitempty"`
}
// BindSpec is one classified entry in a BackupSpec.
@@ -86,6 +97,42 @@ func validClass(c BindClass) bool {
}
}
// ValidateRelPath is THE path-safety refusal set for every ${VAR}-relative catalog path — the
// `backup:` block and `data_paths:` both run through it, so there is exactly ONE definition of what
// a safe relative path is. Refuses: empty, backslash, absolute, non-path.Clean'd, and any leading
// ".." escape. It deliberately does NOT check "matches a compose bind" — that rule needs the bind
// list and differs per caller (whole-block reject for backup:, per-entry for data_paths:).
func ValidateRelPath(root BindRoot, p string) error {
where := fmt.Sprintf("%s[%q]", root, p)
if p == "" {
return fmt.Errorf("%s: empty path", where)
}
if strings.ContainsRune(p, '\\') {
return fmt.Errorf("%s: backslash in path (paths are forward-slash relative)", where)
}
if path.IsAbs(p) {
return fmt.Errorf("%s: absolute path (must be relative to the %s root)", where, root)
}
if p != path.Clean(p) {
return fmt.Errorf("%s: non-clean path (want %q)", where, path.Clean(p))
}
// path.Clean has run — ".." can only survive as a leading "../" segment.
if p == ".." || strings.HasPrefix(p, "../") {
return fmt.Errorf("%s: path escapes the root (..)", where)
}
return nil
}
// ValidRoot reports whether r is one of the three known bind roots.
func ValidRoot(r BindRoot) bool {
switch r {
case RootUserdata, RootHDD, RootImport:
return true
default:
return false
}
}
// ValidateBackupSpec checks a parsed backup block against the app's actual compose binds and returns
// the FIRST defect (whole-block semantics — the caller rejects the ENTIRE block on any error, so the
// app degrades to legacy rather than partially classifying). A nil spec is vacuously valid (legacy).
@@ -113,21 +160,8 @@ func ValidateBackupSpec(spec *BackupSpec, binds []ComposeBind) error {
if !validClass(e.Class) {
return fmt.Errorf("%s: invalid class %q (want mandatory|optional|excluded)", where, e.Class)
}
if e.Path == "" {
return fmt.Errorf("%s: empty path", where)
}
if strings.ContainsRune(e.Path, '\\') {
return fmt.Errorf("%s: backslash in path (paths are forward-slash relative)", where)
}
if path.IsAbs(e.Path) {
return fmt.Errorf("%s: absolute path (must be relative to the %s root)", where, root)
}
if e.Path != path.Clean(e.Path) {
return fmt.Errorf("%s: non-clean path (want %q)", where, path.Clean(e.Path))
}
// path.Clean has run — ".." can only survive as a leading "../" segment.
if e.Path == ".." || strings.HasPrefix(e.Path, "../") {
return fmt.Errorf("%s: path escapes the root (..)", where)
if err := ValidateRelPath(root, e.Path); err != nil {
return err
}
key := string(root) + "\x00" + e.Path
if seen[key] {
@@ -143,7 +177,10 @@ func ValidateBackupSpec(spec *BackupSpec, binds []ComposeBind) error {
if err := check(RootUserdata, spec.Userdata); err != nil {
return err
}
return check(RootHDD, spec.HDD)
if err := check(RootHDD, spec.HDD); err != nil {
return err
}
return check(RootImport, spec.Import)
}
// ClassifyBinds resolves every compose bind to a class + origin, applying the two-level default. The
@@ -181,6 +218,7 @@ func ClassifyBinds(spec *BackupSpec, binds []ComposeBind) (classified []Classifi
}
add(RootUserdata, spec.Userdata)
add(RootHDD, spec.HDD)
add(RootImport, spec.Import)
for _, b := range binds {
cb := ClassifiedBind{ComposeBind: b}
@@ -21,7 +21,7 @@ func TestComputeFabBuckets_Classified(t *testing.T) {
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media/movies"}, Class: ClassExcluded},
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "appdata/app"}, Class: ClassMandatory},
}
fb := ComputeFabBuckets(binds, true, drv)
fb := ComputeFabBuckets(binds, true, drv, "")
if !fb.HasClassification {
t.Fatal("HasClassification must be true")
}
@@ -39,7 +39,7 @@ func TestComputeFabBuckets_Classified(t *testing.T) {
// legacy (no block) → empty buckets (the full-root capture stays out of the classified plan).
func TestComputeFabBuckets_LegacyEmpty(t *testing.T) {
binds := []ClassifiedBind{{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media/tv"}, Origin: OriginLegacy}}
fb := ComputeFabBuckets(binds, false, drv)
fb := ComputeFabBuckets(binds, false, drv, "")
if fb.HasClassification || fb.Mandatory != nil || fb.Optional != nil || fb.Excluded != nil {
t.Errorf("legacy app must yield empty buckets, got %+v", fb)
}
@@ -52,7 +52,7 @@ func TestComputeFabBuckets_GuardsAllClasses(t *testing.T) {
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "../evil"}, Class: ClassExcluded},
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "appdata/ok"}, Class: ClassMandatory},
}
fb := ComputeFabBuckets(binds, true, drv)
fb := ComputeFabBuckets(binds, true, drv, "")
for _, b := range [][]CapturePath{fb.Mandatory, fb.Optional, fb.Excluded} {
for _, p := range b {
if p.RelPath == "../evil" {
@@ -71,7 +71,7 @@ func TestComputeFabBuckets_NoCrossBucketContainment(t *testing.T) {
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media"}, Class: ClassExcluded},
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media/books"}, Class: ClassMandatory},
}
fb := ComputeFabBuckets(binds, true, drv)
fb := ComputeFabBuckets(binds, true, drv, "")
if got, want := bucketAbs(fb.Mandatory), []string{udat("media/books")}; !reflect.DeepEqual(got, want) {
t.Errorf("mandatory child must survive independently: Mandatory = %v, want %v", got, want)
}
@@ -86,7 +86,7 @@ func TestComputeFabBuckets_EqualAbsMandatoryWins(t *testing.T) {
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "userdata/media"}, Class: ClassOptional},
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media"}, Class: ClassMandatory},
}
fb := ComputeFabBuckets(binds, true, drv)
fb := ComputeFabBuckets(binds, true, drv, "")
if got, want := bucketAbs(fb.Mandatory), []string{udat("media")}; !reflect.DeepEqual(got, want) {
t.Errorf("collapsed path must land in Mandatory, got Mandatory=%v", got)
}
@@ -0,0 +1,134 @@
package appbackup
import (
"os"
"path/filepath"
"slices"
"testing"
)
// R-75 Scenario C — DETERMINISM. This is the P6 gate.
//
// The spike measured the naive map-order derivation producing 20 DISTINCT outputs from 20 identical
// runs. fbNeedsRecreate force-recreates FileBrowser on ANY byte difference in the generated config,
// and SyncFileBrowserMounts has ~14 call sites — so a non-deterministic skeleton is a fleet-wide
// FileBrowser restart loop, the v0.151-class bug. 20 identical generations or this fails.
func TestScenarioC_SkeletonDeterminism(t *testing.T) {
// Deliberately UNSORTED input, with duplicates and a deep path, so the function has real work to
// normalise. A sort applied only to the input would not save a map-ordered implementation.
derived := []string{
"media/podcasts", "roms", "media/books", "downloads", "media",
"media/photos", "media/books", "a/b/c/d",
}
const n = 20
first := BuildUserdataSkeleton(derived)
for i := 1; i < n; i++ {
got := BuildUserdataSkeleton(derived)
if !slices.Equal(got, first) {
t.Fatalf("generation %d/%d differs — a non-deterministic skeleton force-recreates FileBrowser on every sync pass\n first: %v\n got: %v",
i+1, n, first, got)
}
}
if !slices.IsSorted(first) {
t.Errorf("skeleton must be sorted, got %v", first)
}
// Ancestor expansion: a deep derived path implies its whole chain.
for _, want := range []string{"a", "a/b", "a/b/c", "a/b/c/d"} {
if !slices.Contains(first, want) {
t.Errorf("ancestor chain incomplete: %q missing from %v", want, first)
}
}
// Dedup: "media/books" appeared twice in the input and "media" both derived and as an ancestor.
for _, d := range []string{"media", "media/books"} {
if c := countOf(first, d); c != 1 {
t.Errorf("%q appears %d times, want exactly 1", d, c)
}
}
}
func countOf(xs []string, want string) int {
n := 0
for _, x := range xs {
if x == want {
n++
}
}
return n
}
// R-75 Scenario D — ZERO REMOVALS, proven by construction.
//
// The derived set drops `documents` (implied by no catalog app) and, after the R-75 move, the two
// import/* entries. The carry-list is what keeps them. This asserts the merged set is a strict
// SUPERSET of the historical hardcoded skeleton for any derived input — including the empty one, the
// fresh-box case where the catalog has not synced yet.
func TestScenarioD_SkeletonNeverDropsACarriedDir(t *testing.T) {
for _, derived := range [][]string{
nil, // fresh box, catalog not yet synced
{"media/podcasts"}, // the one genuinely new entry
{"roms", "downloads", "media/photos"}, // a partial catalog
} {
got := BuildUserdataSkeleton(derived)
for _, carried := range UserdataSkeletonCarry() {
if !slices.Contains(got, carried) {
t.Errorf("derived=%v: carried dir %q was DROPPED — zero-removals violated", derived, carried)
}
}
}
// And the new entry really is added when the catalog implies it.
if !slices.Contains(BuildUserdataSkeleton([]string{"media/podcasts"}), "media/podcasts") {
t.Error("media/podcasts must be added when the catalog implies it")
}
// `documents` is the specific entry the spike flagged: in the carry-list, in no catalog app.
if !slices.Contains(BuildUserdataSkeleton([]string{"media/podcasts"}), "documents") {
t.Error("`documents` must survive — it exists on both demo boxes and may hold customer files")
}
}
// A traversal or absolute entry reaching the skeleton would make EnsureUserdataSkeleton create a
// directory outside the userdata root. The derived set comes from a compose parser, so this is a
// guard on untrusted-ish catalog input, not defence in depth.
func TestSkeletonRefusesEscapes(t *testing.T) {
got := BuildUserdataSkeleton([]string{"../escape", "..", "", "/abs/path", "ok/dir"})
for _, bad := range []string{"../escape", "..", "", "/abs/path"} {
if slices.Contains(got, bad) {
t.Errorf("escape entry %q must not reach the skeleton: %v", bad, got)
}
}
for _, d := range got {
if filepath.IsAbs(d) || d == ".." || len(d) > 3 && d[:3] == "../" {
t.Errorf("unsafe skeleton entry %q", d)
}
}
if !slices.Contains(got, "ok/dir") {
t.Error("a legitimate entry alongside bad ones must still be kept")
}
// "/abs/path" is not dropped outright — it is normalised to a relative path and kept, which is
// safe (it lands under the userdata root). Pin that so the behaviour is a decision, not a guess.
if !slices.Contains(got, "abs/path") {
t.Errorf("an absolute entry should be normalised to relative, got %v", got)
}
}
// EnsureUserdataSkeleton creates every dir it is given and NOTHING ELSE, and never removes.
func TestEnsureUserdataSkeletonCreatesOnly(t *testing.T) {
ns := t.TempDir()
// A pre-existing customer dir that no catalog app implies and the carry-list does not contain.
stray := filepath.Join(UserdataDir(ns), "sajat-mappa")
if err := os.MkdirAll(stray, 0o755); err != nil {
t.Fatal(err)
}
dirs := BuildUserdataSkeleton([]string{"media/podcasts"})
if err := EnsureUserdataSkeleton(ns, dirs); err != nil {
// chown to gid 1000 fails for a non-root test user; the dirs are still created.
t.Logf("EnsureUserdataSkeleton returned %v (expected when not running as root)", err)
}
for _, d := range dirs {
if fi, err := os.Stat(filepath.Join(UserdataDir(ns), d)); err != nil || !fi.IsDir() {
t.Errorf("skeleton dir %q not created: %v", d, err)
}
}
if _, err := os.Stat(stray); err != nil {
t.Errorf("a pre-existing customer dir was removed — zero-removals violated: %v", err)
}
}
+78 -4
View File
@@ -2,7 +2,10 @@ package appbackup
import (
"os"
"path"
"path/filepath"
"sort"
"strings"
)
// Customer-facing userdata layout + the shared-storage ownership convention (v0.66.0).
@@ -28,9 +31,41 @@ func UserdataDir(nsRoot string) string {
return filepath.Join(nsRoot, "userdata")
}
// UserdataSkeleton is the standard subtree created on every storage path (relative to UserdataDir).
// ImportDirName is the single import (drop-zone) subtree name under a userdata root.
const ImportDirName = "import"
// ImportDir returns the CANONICAL drop-zone root under a namespace root (R-75).
//
// Unlike every other userdata dir, this one is drive-INDEPENDENT: the caller resolves it against the
// SYSTEM drive's namespace root, never against the app's own HDD_PATH, so a multi-drive box has
// exactly ONE import tree. That is the whole point. Each drop-zone app has exactly one ingest bind,
// so a per-drive import/ would put a folder that LOOKS like a drop-zone on every drive while only
// one of them does anything — and because import paths are `class: excluded`, files stranded in a
// dead one are never backed up either.
//
// It deliberately stays INSIDE the userdata tree, so the 2775/setgid/GID-1000 convention, the
// FileBrowser mount and the ownership rules all apply to it unchanged.
func ImportDir(nsRoot string) string {
return filepath.Join(UserdataDir(nsRoot), ImportDirName)
}
// UserdataSkeletonCarry is the explicit NON-DERIVED carry-list: every entry the v0.171.0 hardcoded
// skeleton created, retained verbatim and forever.
//
// It exists so the catalog-derived skeleton (R-75) can only ever ADD. That makes the zero-removals
// invariant true BY CONSTRUCTION rather than by review, and it is not hypothetical:
//
// - `documents` is implied by NO catalog app (SPIKE P0(a)) yet exists on both demo boxes and is
// customer-visible — it may hold customer files. Derivation alone would drop it.
// - `import/paperless` and `import/calibre` moved to the canonical system-drive root in R-75, so
// derivation no longer implies them under a data drive either. The pre-existing ones stay put;
// nothing in this arc deletes a directory.
//
// It doubles as the fresh-box floor: on a box whose catalog has not synced yet the derived set is
// empty, and the customer still gets the full standard tree instead of a nearly-empty one.
//
// ASCII, no spaces (flows through ${} interpolation, shell, and the rsync merge walk).
func UserdataSkeleton() []string {
func UserdataSkeletonCarry() []string {
return []string{
"media", "media/movies", "media/tv", "media/music", "media/audiobooks",
"media/books", "media/comics", "media/photos",
@@ -41,6 +76,41 @@ func UserdataSkeleton() []string {
}
}
// BuildUserdataSkeleton merges the catalog-derived dirs with the carry-list into the final, SORTED
// set. Each entry is expanded to its ancestor chain ("media/podcasts" implies "media"), deduped, and
// sorted.
//
// SORTING IS A HARD REQUIREMENT, not tidiness. The FileBrowser config is regenerated from this set
// and fbNeedsRecreate force-recreates the container on ANY byte difference. Go randomises map
// iteration, and the spike measured the naive map-order derivation producing 20 DISTINCT outputs from
// 20 identical runs (SPIKE P6) — which across SyncFileBrowserMounts' ~14 call sites is a fleet-wide
// FileBrowser restart loop. TestSkeletonDeterminism pins this.
func BuildUserdataSkeleton(derived []string) []string {
set := make(map[string]bool, len(derived)+16)
addChain := func(rel string) {
rel = path.Clean(strings.TrimPrefix(filepath.ToSlash(rel), "/"))
if rel == "" || rel == "." || rel == ".." || strings.HasPrefix(rel, "../") {
return // never let a traversal or an empty entry become a directory to create
}
parts := strings.Split(rel, "/")
for i := range parts {
set[strings.Join(parts[:i+1], "/")] = true
}
}
for _, d := range UserdataSkeletonCarry() {
addChain(d)
}
for _, d := range derived {
addChain(d)
}
out := make([]string, 0, len(set))
for d := range set { // map order is RANDOM — the sort below is what makes this deterministic
out = append(out, d)
}
sort.Strings(out)
return out
}
// EnsureDirOwned creates path (idempotent) and enforces the convention: mode 2775 via an explicit
// Chmod incl. setgid (MkdirAll cannot) + group = gid. Setting an arbitrary group needs CAP_CHOWN —
// the in-guest controller runs as root, so this succeeds in production. Returns the first hard error.
@@ -60,7 +130,11 @@ func EnsureUserdataDir(path string) error { return EnsureDirOwned(path, SharedCo
// EnsureUserdataSkeleton creates the full userdata tree under a namespace root with the convention.
// It creates ALL dirs even if one errors (so a single chown/chmod hiccup doesn't truncate the tree),
// returning the first error seen for the caller to log.
func EnsureUserdataSkeleton(nsRoot string) error {
//
// dirs is the merged, sorted set from BuildUserdataSkeleton. This function only ever CREATES: there
// is no removal path here or anywhere in R-75, so a directory the current catalog no longer implies
// simply stays where it is (Scenario D).
func EnsureUserdataSkeleton(nsRoot string, dirs []string) error {
base := UserdataDir(nsRoot)
var firstErr error
rec := func(e error) {
@@ -69,7 +143,7 @@ func EnsureUserdataSkeleton(nsRoot string) error {
}
}
rec(EnsureUserdataDir(base))
for _, sub := range UserdataSkeleton() {
for _, sub := range dirs {
rec(EnsureUserdataDir(filepath.Join(base, sub)))
}
return firstErr
@@ -13,10 +13,12 @@ func TestSharedContentGID(t *testing.T) {
}
}
// TestUserdataSkeleton_List asserts the locked skeleton subdir set.
// TestUserdataSkeleton_List asserts the locked skeleton subdir set. R-75 renamed the hardcoded list
// to UserdataSkeletonCarry (it is now the non-derived carry-list); the asserted set is UNCHANGED,
// which is exactly the zero-removals promise.
func TestUserdataSkeleton_List(t *testing.T) {
got := map[string]bool{}
for _, s := range UserdataSkeleton() {
for _, s := range UserdataSkeletonCarry() {
got[s] = true
}
for _, want := range []string{
@@ -41,9 +43,10 @@ func TestUserdataDir(t *testing.T) {
// is ignored — dirs + setgid still land). Runs cross-platform.
func TestEnsureUserdataSkeleton_Structure(t *testing.T) {
ns := t.TempDir()
_ = EnsureUserdataSkeleton(ns) // ignore chown error on a non-root CI host
dirs := BuildUserdataSkeleton(nil) // no catalog derived → the carry-list floor
_ = EnsureUserdataSkeleton(ns, dirs) // ignore chown error on a non-root CI host
base := UserdataDir(ns)
for _, sub := range append([]string{""}, UserdataSkeleton()...) {
for _, sub := range append([]string{""}, dirs...) {
p := filepath.Join(base, sub)
if fi, err := os.Stat(p); err != nil || !fi.IsDir() {
t.Errorf("skeleton dir missing: %s (%v)", p, err)