v0.172.0 — R-75: canonical import root, catalog-derived skeleton, import surfaces
${IMPORT_PATH} = <system namespace root>/userdata/import — ONE drop-zone per box,
on the system drive, injected at BOTH compose-env builders with NO per-drive
fallback (unresolvable leaves it unset so compose fails loudly rather than
quietly building a second, dead drop-zone).
Third BindRoot (RootImport) + Import list in BackupSpec, extended through
ValidateBackupSpec/ClassifyBinds. Load-bearing: a stale `userdata: import/<app>`
entry against the moved bind would be a WHOLE-BLOCK reject, taking the app's
mandatory hdd classification with it.
Exhaustive-root audit: resolveAbs/structuralGuard/ComputeCaptureSet/
ComputeFabBuckets now take importRoot explicitly (an import bind resolved
against hddPath would name a directory on the wrong drive); unresolvable is
refused loudly into Skipped. GetImportRoot added to both provider interfaces.
Catalog-derived skeleton: UserdataSkeleton() -> UserdataSkeletonCarry() +
BuildUserdataSkeleton(), SORTED. The carry-list makes zero-removals true by
construction (`documents` is in no catalog app but on both boxes) and is the
fresh-box floor. The sort is not tidiness: the naive map-order derivation
measured 20 distinct outputs from 20 identical runs, which with fbNeedsRecreate
is a fleet-wide FileBrowser restart loop.
One authoritative compose parser: ParseComposeUserdataMounts now delegates to
ParseComposeClassifiableBinds. Import root excluded from per-app migration.
Surfaces: FileBrowser /srv/beolvasas source; app-page "Hova tegyem a fajlokat?"
with PathEscape deep links (never QueryEscape) and class-driven copy;
data_paths: annotation with the Fork-3 asymmetry; system-owned beolvasas SMB
share refused server-side at handler AND store, button omitted in template.
Caught on the way: the sharing template's row struct was function-local, so
adding {{if .System}} would have 500'd every share row. ShareRow is now
package-level and the render test uses the handler's own type.
Tests 915 -> 949, all green. MinAgent unchanged.
This commit is contained in:
@@ -19,7 +19,11 @@ type StackDataProvider interface {
|
||||
GetStackComposePath(name string) (composePath string, ok bool)
|
||||
ListDeployedStacks() []StackSummary
|
||||
GetStackHDDMounts(name string) []string
|
||||
GetStackHDDPath(name string) string // raw HDD_PATH from app.yaml (empty if no HDD)
|
||||
GetStackHDDPath(name string) string // raw HDD_PATH from app.yaml (empty if no HDD)
|
||||
// GetImportRoot returns the CANONICAL drop-zone root (R-75): <system namespace root>/userdata/import.
|
||||
// It is app-INDEPENDENT and lives on the SYSTEM drive, so ${IMPORT_PATH} binds cannot be resolved
|
||||
// from GetStackHDDPath. Empty when unresolvable — structuralGuard refuses such binds loudly.
|
||||
GetImportRoot() string
|
||||
GetDockerVolumes(name string) []string // full Docker volume names (project-prefixed)
|
||||
StopStack(name string) error
|
||||
StartStack(name string) error
|
||||
|
||||
@@ -59,6 +59,8 @@ const (
|
||||
reasonEscape = "path escapes the drive root"
|
||||
reasonBareRoot = "bare drive-root bind would capture the backups tree"
|
||||
reasonReserved = "path inside the reserved backups zone"
|
||||
// reasonNoImportRoot: a ${IMPORT_PATH} bind with no resolvable system namespace root (R-75).
|
||||
reasonNoImportRoot = "canonical import root unresolvable (system_data_path unconfigured)"
|
||||
)
|
||||
|
||||
// ComputeCaptureSet resolves an app's classified binds into the tier-filtered absolute capture set.
|
||||
@@ -71,16 +73,17 @@ const (
|
||||
// so the engines' no-block branch stays byte-identical to today (the SQ5 cost-regression guard).
|
||||
//
|
||||
// Resolution: RootHDD → path.Join(hddPath, relPath); RootUserdata → path.Join(hddPath, "userdata",
|
||||
// relPath). Guards run AFTER the tier filter, so Skipped means exactly "would have been captured by
|
||||
// this tier, refused for structural safety".
|
||||
func ComputeCaptureSet(binds []ClassifiedBind, hasClassification bool, tier CaptureTier, hddPath string) CaptureSet {
|
||||
// relPath); RootImport → path.Join(importRoot, relPath) — the SYSTEM drive, never hddPath (R-75).
|
||||
// Guards run AFTER the tier filter, so Skipped means exactly "would have been captured by this tier,
|
||||
// refused for structural safety".
|
||||
func ComputeCaptureSet(binds []ClassifiedBind, hasClassification bool, tier CaptureTier, hddPath, importRoot string) CaptureSet {
|
||||
if !hasClassification {
|
||||
return CaptureSet{HasClassification: false}
|
||||
}
|
||||
cs := CaptureSet{HasClassification: true}
|
||||
|
||||
// Stages 1–3: tier filter → structural guards → equal-Abs collapse (shared with ComputeFabBuckets).
|
||||
uniq, skipped := resolveGuardCollapse(binds, hddPath, func(c BindClass) bool { return tierKeeps(tier, c) })
|
||||
uniq, skipped := resolveGuardCollapse(binds, hddPath, importRoot, func(c BindClass) bool { return tierKeeps(tier, c) })
|
||||
cs.Skipped = skipped
|
||||
|
||||
// Stage 4: containment dedup — drop any path whose ancestor is already present (keep the ancestor).
|
||||
@@ -107,18 +110,18 @@ func ComputeCaptureSet(binds []ClassifiedBind, hasClassification bool, tier Capt
|
||||
// collapse (mandatory > optional > excluded; ties by smaller Root/RelPath). It does NOT apply
|
||||
// containment dedup — the caller decides (ComputeCaptureSet does; ComputeFabBuckets must not, so a
|
||||
// mandatory child inside an excluded parent stays independently addressable).
|
||||
func resolveGuardCollapse(binds []ClassifiedBind, hddPath string, keep func(BindClass) bool) (uniq []CapturePath, skipped []SkippedPath) {
|
||||
func resolveGuardCollapse(binds []ClassifiedBind, hddPath, importRoot string, keep func(BindClass) bool) (uniq []CapturePath, skipped []SkippedPath) {
|
||||
var resolved []CapturePath
|
||||
for _, b := range binds {
|
||||
if !keep(b.Class) {
|
||||
continue
|
||||
}
|
||||
if reason, bad := structuralGuard(b.Root, b.RelPath); bad {
|
||||
if reason, bad := structuralGuard(b.Root, b.RelPath, importRoot); bad {
|
||||
skipped = append(skipped, SkippedPath{Root: b.Root, RelPath: b.RelPath, Class: b.Class, Reason: reason})
|
||||
continue
|
||||
}
|
||||
resolved = append(resolved, CapturePath{
|
||||
Abs: resolveAbs(hddPath, b.Root, b.RelPath), Root: b.Root, RelPath: b.RelPath, Class: b.Class,
|
||||
Abs: resolveAbs(hddPath, importRoot, b.Root, b.RelPath), Root: b.Root, RelPath: b.RelPath, Class: b.Class,
|
||||
})
|
||||
}
|
||||
byAbs := make(map[string]CapturePath, len(resolved))
|
||||
@@ -153,12 +156,12 @@ type FabBuckets struct {
|
||||
// selection UI + plan. Same resolution + structural guards + equal-Abs collapse as ComputeCaptureSet
|
||||
// (via resolveGuardCollapse), bucketed by class, no cross-bucket containment dedup. Each bucket is
|
||||
// Abs-sorted (deterministic).
|
||||
func ComputeFabBuckets(binds []ClassifiedBind, hasClassification bool, hddPath string) FabBuckets {
|
||||
func ComputeFabBuckets(binds []ClassifiedBind, hasClassification bool, hddPath, importRoot string) FabBuckets {
|
||||
if !hasClassification {
|
||||
return FabBuckets{HasClassification: false}
|
||||
}
|
||||
fb := FabBuckets{HasClassification: true}
|
||||
uniq, skipped := resolveGuardCollapse(binds, hddPath, func(BindClass) bool { return true })
|
||||
uniq, skipped := resolveGuardCollapse(binds, hddPath, importRoot, func(BindClass) bool { return true })
|
||||
fb.Skipped = skipped
|
||||
for _, cp := range uniq {
|
||||
switch cp.Class {
|
||||
@@ -201,10 +204,19 @@ func tierKeeps(tier CaptureTier, class BindClass) bool {
|
||||
// there (ParseComposeClassifiableBinds path.Cleans; ValidateBackupSpec vets only SPEC entries), so an
|
||||
// unlisted writable "${HDD_PATH}/../x" bind reaches here classed mandatory — this guard is
|
||||
// load-bearing security, not defence-in-depth.
|
||||
func structuralGuard(root BindRoot, relPath string) (reason string, bad bool) {
|
||||
func structuralGuard(root BindRoot, relPath, importRoot string) (reason string, bad bool) {
|
||||
if relPathEscapes(relPath) {
|
||||
return reasonEscape, true
|
||||
}
|
||||
// RootImport (R-75) resolves against the SYSTEM drive, not hddPath. If that root is unresolvable
|
||||
// (system_data_path unconfigured) the bind cannot be placed at all — refuse it LOUDLY into Skipped
|
||||
// rather than let resolveAbs join onto "" and produce a relative, wrong-drive path. The other two
|
||||
// roots cannot hit this: hddPath is checked by their own callers.
|
||||
if root == RootImport && importRoot == "" {
|
||||
return reasonNoImportRoot, true
|
||||
}
|
||||
// A bare ${IMPORT_PATH} bind is allowed: it resolves to <sysNS>/userdata/import, which nests no
|
||||
// backups/ tree (backups live at <sysNS>/backups, a sibling of userdata).
|
||||
if root == RootHDD {
|
||||
if relPath == "" {
|
||||
return reasonBareRoot, true // bare ${HDD_PATH} would nest <hddPath>/backups into the capture
|
||||
@@ -233,11 +245,21 @@ func relPathEscapes(relPath string) bool {
|
||||
}
|
||||
|
||||
// resolveAbs maps a guarded (root, relPath) to its in-container absolute path via slash algebra.
|
||||
func resolveAbs(hddPath string, root BindRoot, relPath string) string {
|
||||
if root == RootUserdata {
|
||||
//
|
||||
// RootImport is the one root that does NOT resolve against hddPath: the canonical drop-zone lives on
|
||||
// the SYSTEM drive (R-75), so importRoot is supplied separately by the caller. Resolving it against
|
||||
// hddPath would silently name a directory on the WRONG DRIVE — a .fab opt-in would then capture (or
|
||||
// on restore, write) somewhere that merely looks plausible. An empty importRoot is the unresolvable
|
||||
// case and is refused upstream by structuralGuard, never silently joined.
|
||||
func resolveAbs(hddPath, importRoot string, root BindRoot, relPath string) string {
|
||||
switch root {
|
||||
case RootUserdata:
|
||||
return path.Join(hddPath, "userdata", relPath)
|
||||
case RootImport:
|
||||
return path.Join(importRoot, relPath)
|
||||
default:
|
||||
return path.Join(hddPath, relPath)
|
||||
}
|
||||
return path.Join(hddPath, relPath)
|
||||
}
|
||||
|
||||
// strongerCapture picks the winner of an equal-Abs collision: mandatory beats optional; on equal
|
||||
|
||||
@@ -38,12 +38,12 @@ func TestComputeCaptureSet_PerTierSplit(t *testing.T) {
|
||||
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media/photos", ReadOnly: true}, Class: ClassOptional, Origin: OriginExplicit},
|
||||
}
|
||||
|
||||
off := ComputeCaptureSet(binds, true, TierOffsite, drv)
|
||||
off := ComputeCaptureSet(binds, true, TierOffsite, drv, "")
|
||||
if got, want := absList(off), []string{hdd("appdata/immich")}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("offsite Paths = %v, want %v (mandatory only — the :ro optional must NOT ship offsite)", got, want)
|
||||
}
|
||||
|
||||
sec := ComputeCaptureSet(binds, true, TierSecondary, drv)
|
||||
sec := ComputeCaptureSet(binds, true, TierSecondary, drv, "")
|
||||
want := []string{hdd("appdata/immich"), udat("media/photos")}
|
||||
if got := absList(sec); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("secondary Paths = %v, want %v (sorted)", got, want)
|
||||
@@ -72,7 +72,7 @@ func TestComputeCaptureSet_LegacyInert(t *testing.T) {
|
||||
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "appdata/sonarr"}, Origin: OriginLegacy},
|
||||
}
|
||||
for _, tier := range []CaptureTier{TierOffsite, TierSecondary} {
|
||||
cs := ComputeCaptureSet(binds, false, tier, drv)
|
||||
cs := ComputeCaptureSet(binds, false, tier, drv, "")
|
||||
if cs.HasClassification {
|
||||
t.Errorf("%s: HasClassification=true for a legacy app", tier)
|
||||
}
|
||||
@@ -94,7 +94,7 @@ func TestComputeCaptureSet_ExcludedInvisible(t *testing.T) {
|
||||
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "import/paperless"}, Class: ClassExcluded, Origin: OriginExplicit},
|
||||
}
|
||||
for _, tier := range []CaptureTier{TierOffsite, TierSecondary} {
|
||||
cs := ComputeCaptureSet(binds, true, tier, drv)
|
||||
cs := ComputeCaptureSet(binds, true, tier, drv, "")
|
||||
if got, want := absList(cs), []string{hdd("appdata/paperless/media")}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("%s Paths = %v, want %v (excluded filtered)", tier, got, want)
|
||||
}
|
||||
@@ -113,7 +113,7 @@ func TestComputeCaptureSet_StructuralGuards(t *testing.T) {
|
||||
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "backups/primary/x"}, Class: ClassMandatory, Origin: OriginDefaultWritable}, // d3 reserved zone
|
||||
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: ""}, Class: ClassMandatory, Origin: OriginDefaultWritable}, // d4 bare userdata — ALLOWED
|
||||
}
|
||||
cs := ComputeCaptureSet(binds, true, TierOffsite, drv)
|
||||
cs := ComputeCaptureSet(binds, true, TierOffsite, drv, "")
|
||||
|
||||
// Paths: ONLY d4's userdata root — no escaped root, no backups/ anywhere.
|
||||
if got, want := absList(cs), []string{udat("")}; !reflect.DeepEqual(got, want) {
|
||||
@@ -156,7 +156,7 @@ func TestComputeCaptureSet_LegitDotDotName(t *testing.T) {
|
||||
binds := []ClassifiedBind{
|
||||
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "appdata/a..b"}, Class: ClassMandatory, Origin: OriginExplicit},
|
||||
}
|
||||
cs := ComputeCaptureSet(binds, true, TierOffsite, drv)
|
||||
cs := ComputeCaptureSet(binds, true, TierOffsite, drv, "")
|
||||
if got, want := absList(cs), []string{hdd("appdata/a..b")}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("Paths = %v, want %v (a..b is a legit name, not traversal)", got, want)
|
||||
}
|
||||
@@ -174,7 +174,7 @@ func TestComputeCaptureSet_ContainmentAndCollision(t *testing.T) {
|
||||
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "userdata/media"}, Class: ClassOptional, Origin: OriginExplicit}, // Abs collides with next
|
||||
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media"}, Class: ClassMandatory, Origin: OriginExplicit}, // same Abs, mandatory
|
||||
}
|
||||
cs := ComputeCaptureSet(binds, true, TierSecondary, drv)
|
||||
cs := ComputeCaptureSet(binds, true, TierSecondary, drv, "")
|
||||
|
||||
want := []string{hdd("appdata/paperless"), udat("media")}
|
||||
if got := absList(cs); !reflect.DeepEqual(got, want) {
|
||||
@@ -186,7 +186,7 @@ func TestComputeCaptureSet_ContainmentAndCollision(t *testing.T) {
|
||||
}
|
||||
|
||||
// determinism: recompute and compare full struct
|
||||
cs2 := ComputeCaptureSet(binds, true, TierSecondary, drv)
|
||||
cs2 := ComputeCaptureSet(binds, true, TierSecondary, drv, "")
|
||||
if !reflect.DeepEqual(cs, cs2) {
|
||||
t.Error("ComputeCaptureSet is non-deterministic across runs")
|
||||
}
|
||||
|
||||
@@ -34,12 +34,23 @@ type BindRoot string
|
||||
const (
|
||||
RootUserdata BindRoot = "userdata" // relative to ${USERDATA_PATH}
|
||||
RootHDD BindRoot = "hdd" // relative to ${HDD_PATH}
|
||||
// RootImport is relative to ${IMPORT_PATH} — the CANONICAL drop-zone root (R-75). Unlike the
|
||||
// other two it does NOT resolve against the app's own drive: it lives on the system drive's
|
||||
// namespace, so every app's ingest folder is in one place. Resolvers therefore need the import
|
||||
// root passed in separately; they cannot derive it from hddPath.
|
||||
RootImport BindRoot = "import"
|
||||
)
|
||||
|
||||
// BackupSpec is the .felhom.yml `backup:` block. Paths are forward-slash, relative, path.Clean'd.
|
||||
type BackupSpec struct {
|
||||
Userdata []BindSpec `yaml:"userdata,omitempty" json:"userdata,omitempty"`
|
||||
HDD []BindSpec `yaml:"hdd,omitempty" json:"hdd,omitempty"`
|
||||
// Import classifies ${IMPORT_PATH}-relative binds (R-75). An app whose ingest bind moved from
|
||||
// ${USERDATA_PATH}/import/<app> to ${IMPORT_PATH}/<app> MUST move its backup entry here in the
|
||||
// same change: ValidateBackupSpec rejects an entry matching no compose bind, and the rejection is
|
||||
// WHOLE-BLOCK, so a stale `userdata: import/<app>` would discard the app's OTHER classifications
|
||||
// (e.g. an hdd appdata path classed mandatory) and silently degrade it to legacy.
|
||||
Import []BindSpec `yaml:"import,omitempty" json:"import,omitempty"`
|
||||
}
|
||||
|
||||
// BindSpec is one classified entry in a BackupSpec.
|
||||
@@ -86,6 +97,42 @@ func validClass(c BindClass) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateRelPath is THE path-safety refusal set for every ${VAR}-relative catalog path — the
|
||||
// `backup:` block and `data_paths:` both run through it, so there is exactly ONE definition of what
|
||||
// a safe relative path is. Refuses: empty, backslash, absolute, non-path.Clean'd, and any leading
|
||||
// ".." escape. It deliberately does NOT check "matches a compose bind" — that rule needs the bind
|
||||
// list and differs per caller (whole-block reject for backup:, per-entry for data_paths:).
|
||||
func ValidateRelPath(root BindRoot, p string) error {
|
||||
where := fmt.Sprintf("%s[%q]", root, p)
|
||||
if p == "" {
|
||||
return fmt.Errorf("%s: empty path", where)
|
||||
}
|
||||
if strings.ContainsRune(p, '\\') {
|
||||
return fmt.Errorf("%s: backslash in path (paths are forward-slash relative)", where)
|
||||
}
|
||||
if path.IsAbs(p) {
|
||||
return fmt.Errorf("%s: absolute path (must be relative to the %s root)", where, root)
|
||||
}
|
||||
if p != path.Clean(p) {
|
||||
return fmt.Errorf("%s: non-clean path (want %q)", where, path.Clean(p))
|
||||
}
|
||||
// path.Clean has run — ".." can only survive as a leading "../" segment.
|
||||
if p == ".." || strings.HasPrefix(p, "../") {
|
||||
return fmt.Errorf("%s: path escapes the root (..)", where)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidRoot reports whether r is one of the three known bind roots.
|
||||
func ValidRoot(r BindRoot) bool {
|
||||
switch r {
|
||||
case RootUserdata, RootHDD, RootImport:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateBackupSpec checks a parsed backup block against the app's actual compose binds and returns
|
||||
// the FIRST defect (whole-block semantics — the caller rejects the ENTIRE block on any error, so the
|
||||
// app degrades to legacy rather than partially classifying). A nil spec is vacuously valid (legacy).
|
||||
@@ -113,21 +160,8 @@ func ValidateBackupSpec(spec *BackupSpec, binds []ComposeBind) error {
|
||||
if !validClass(e.Class) {
|
||||
return fmt.Errorf("%s: invalid class %q (want mandatory|optional|excluded)", where, e.Class)
|
||||
}
|
||||
if e.Path == "" {
|
||||
return fmt.Errorf("%s: empty path", where)
|
||||
}
|
||||
if strings.ContainsRune(e.Path, '\\') {
|
||||
return fmt.Errorf("%s: backslash in path (paths are forward-slash relative)", where)
|
||||
}
|
||||
if path.IsAbs(e.Path) {
|
||||
return fmt.Errorf("%s: absolute path (must be relative to the %s root)", where, root)
|
||||
}
|
||||
if e.Path != path.Clean(e.Path) {
|
||||
return fmt.Errorf("%s: non-clean path (want %q)", where, path.Clean(e.Path))
|
||||
}
|
||||
// path.Clean has run — ".." can only survive as a leading "../" segment.
|
||||
if e.Path == ".." || strings.HasPrefix(e.Path, "../") {
|
||||
return fmt.Errorf("%s: path escapes the root (..)", where)
|
||||
if err := ValidateRelPath(root, e.Path); err != nil {
|
||||
return err
|
||||
}
|
||||
key := string(root) + "\x00" + e.Path
|
||||
if seen[key] {
|
||||
@@ -143,7 +177,10 @@ func ValidateBackupSpec(spec *BackupSpec, binds []ComposeBind) error {
|
||||
if err := check(RootUserdata, spec.Userdata); err != nil {
|
||||
return err
|
||||
}
|
||||
return check(RootHDD, spec.HDD)
|
||||
if err := check(RootHDD, spec.HDD); err != nil {
|
||||
return err
|
||||
}
|
||||
return check(RootImport, spec.Import)
|
||||
}
|
||||
|
||||
// ClassifyBinds resolves every compose bind to a class + origin, applying the two-level default. The
|
||||
@@ -181,6 +218,7 @@ func ClassifyBinds(spec *BackupSpec, binds []ComposeBind) (classified []Classifi
|
||||
}
|
||||
add(RootUserdata, spec.Userdata)
|
||||
add(RootHDD, spec.HDD)
|
||||
add(RootImport, spec.Import)
|
||||
|
||||
for _, b := range binds {
|
||||
cb := ClassifiedBind{ComposeBind: b}
|
||||
|
||||
@@ -21,7 +21,7 @@ func TestComputeFabBuckets_Classified(t *testing.T) {
|
||||
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media/movies"}, Class: ClassExcluded},
|
||||
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "appdata/app"}, Class: ClassMandatory},
|
||||
}
|
||||
fb := ComputeFabBuckets(binds, true, drv)
|
||||
fb := ComputeFabBuckets(binds, true, drv, "")
|
||||
if !fb.HasClassification {
|
||||
t.Fatal("HasClassification must be true")
|
||||
}
|
||||
@@ -39,7 +39,7 @@ func TestComputeFabBuckets_Classified(t *testing.T) {
|
||||
// legacy (no block) → empty buckets (the full-root capture stays out of the classified plan).
|
||||
func TestComputeFabBuckets_LegacyEmpty(t *testing.T) {
|
||||
binds := []ClassifiedBind{{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media/tv"}, Origin: OriginLegacy}}
|
||||
fb := ComputeFabBuckets(binds, false, drv)
|
||||
fb := ComputeFabBuckets(binds, false, drv, "")
|
||||
if fb.HasClassification || fb.Mandatory != nil || fb.Optional != nil || fb.Excluded != nil {
|
||||
t.Errorf("legacy app must yield empty buckets, got %+v", fb)
|
||||
}
|
||||
@@ -52,7 +52,7 @@ func TestComputeFabBuckets_GuardsAllClasses(t *testing.T) {
|
||||
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "../evil"}, Class: ClassExcluded},
|
||||
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "appdata/ok"}, Class: ClassMandatory},
|
||||
}
|
||||
fb := ComputeFabBuckets(binds, true, drv)
|
||||
fb := ComputeFabBuckets(binds, true, drv, "")
|
||||
for _, b := range [][]CapturePath{fb.Mandatory, fb.Optional, fb.Excluded} {
|
||||
for _, p := range b {
|
||||
if p.RelPath == "../evil" {
|
||||
@@ -71,7 +71,7 @@ func TestComputeFabBuckets_NoCrossBucketContainment(t *testing.T) {
|
||||
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media"}, Class: ClassExcluded},
|
||||
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media/books"}, Class: ClassMandatory},
|
||||
}
|
||||
fb := ComputeFabBuckets(binds, true, drv)
|
||||
fb := ComputeFabBuckets(binds, true, drv, "")
|
||||
if got, want := bucketAbs(fb.Mandatory), []string{udat("media/books")}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("mandatory child must survive independently: Mandatory = %v, want %v", got, want)
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func TestComputeFabBuckets_EqualAbsMandatoryWins(t *testing.T) {
|
||||
{ComposeBind: ComposeBind{Root: RootHDD, RelPath: "userdata/media"}, Class: ClassOptional},
|
||||
{ComposeBind: ComposeBind{Root: RootUserdata, RelPath: "media"}, Class: ClassMandatory},
|
||||
}
|
||||
fb := ComputeFabBuckets(binds, true, drv)
|
||||
fb := ComputeFabBuckets(binds, true, drv, "")
|
||||
if got, want := bucketAbs(fb.Mandatory), []string{udat("media")}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("collapsed path must land in Mandatory, got Mandatory=%v", got)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package appbackup
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-75 Scenario C — DETERMINISM. This is the P6 gate.
|
||||
//
|
||||
// The spike measured the naive map-order derivation producing 20 DISTINCT outputs from 20 identical
|
||||
// runs. fbNeedsRecreate force-recreates FileBrowser on ANY byte difference in the generated config,
|
||||
// and SyncFileBrowserMounts has ~14 call sites — so a non-deterministic skeleton is a fleet-wide
|
||||
// FileBrowser restart loop, the v0.151-class bug. 20 identical generations or this fails.
|
||||
func TestScenarioC_SkeletonDeterminism(t *testing.T) {
|
||||
// Deliberately UNSORTED input, with duplicates and a deep path, so the function has real work to
|
||||
// normalise. A sort applied only to the input would not save a map-ordered implementation.
|
||||
derived := []string{
|
||||
"media/podcasts", "roms", "media/books", "downloads", "media",
|
||||
"media/photos", "media/books", "a/b/c/d",
|
||||
}
|
||||
const n = 20
|
||||
first := BuildUserdataSkeleton(derived)
|
||||
for i := 1; i < n; i++ {
|
||||
got := BuildUserdataSkeleton(derived)
|
||||
if !slices.Equal(got, first) {
|
||||
t.Fatalf("generation %d/%d differs — a non-deterministic skeleton force-recreates FileBrowser on every sync pass\n first: %v\n got: %v",
|
||||
i+1, n, first, got)
|
||||
}
|
||||
}
|
||||
if !slices.IsSorted(first) {
|
||||
t.Errorf("skeleton must be sorted, got %v", first)
|
||||
}
|
||||
// Ancestor expansion: a deep derived path implies its whole chain.
|
||||
for _, want := range []string{"a", "a/b", "a/b/c", "a/b/c/d"} {
|
||||
if !slices.Contains(first, want) {
|
||||
t.Errorf("ancestor chain incomplete: %q missing from %v", want, first)
|
||||
}
|
||||
}
|
||||
// Dedup: "media/books" appeared twice in the input and "media" both derived and as an ancestor.
|
||||
for _, d := range []string{"media", "media/books"} {
|
||||
if c := countOf(first, d); c != 1 {
|
||||
t.Errorf("%q appears %d times, want exactly 1", d, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func countOf(xs []string, want string) int {
|
||||
n := 0
|
||||
for _, x := range xs {
|
||||
if x == want {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// R-75 Scenario D — ZERO REMOVALS, proven by construction.
|
||||
//
|
||||
// The derived set drops `documents` (implied by no catalog app) and, after the R-75 move, the two
|
||||
// import/* entries. The carry-list is what keeps them. This asserts the merged set is a strict
|
||||
// SUPERSET of the historical hardcoded skeleton for any derived input — including the empty one, the
|
||||
// fresh-box case where the catalog has not synced yet.
|
||||
func TestScenarioD_SkeletonNeverDropsACarriedDir(t *testing.T) {
|
||||
for _, derived := range [][]string{
|
||||
nil, // fresh box, catalog not yet synced
|
||||
{"media/podcasts"}, // the one genuinely new entry
|
||||
{"roms", "downloads", "media/photos"}, // a partial catalog
|
||||
} {
|
||||
got := BuildUserdataSkeleton(derived)
|
||||
for _, carried := range UserdataSkeletonCarry() {
|
||||
if !slices.Contains(got, carried) {
|
||||
t.Errorf("derived=%v: carried dir %q was DROPPED — zero-removals violated", derived, carried)
|
||||
}
|
||||
}
|
||||
}
|
||||
// And the new entry really is added when the catalog implies it.
|
||||
if !slices.Contains(BuildUserdataSkeleton([]string{"media/podcasts"}), "media/podcasts") {
|
||||
t.Error("media/podcasts must be added when the catalog implies it")
|
||||
}
|
||||
// `documents` is the specific entry the spike flagged: in the carry-list, in no catalog app.
|
||||
if !slices.Contains(BuildUserdataSkeleton([]string{"media/podcasts"}), "documents") {
|
||||
t.Error("`documents` must survive — it exists on both demo boxes and may hold customer files")
|
||||
}
|
||||
}
|
||||
|
||||
// A traversal or absolute entry reaching the skeleton would make EnsureUserdataSkeleton create a
|
||||
// directory outside the userdata root. The derived set comes from a compose parser, so this is a
|
||||
// guard on untrusted-ish catalog input, not defence in depth.
|
||||
func TestSkeletonRefusesEscapes(t *testing.T) {
|
||||
got := BuildUserdataSkeleton([]string{"../escape", "..", "", "/abs/path", "ok/dir"})
|
||||
for _, bad := range []string{"../escape", "..", "", "/abs/path"} {
|
||||
if slices.Contains(got, bad) {
|
||||
t.Errorf("escape entry %q must not reach the skeleton: %v", bad, got)
|
||||
}
|
||||
}
|
||||
for _, d := range got {
|
||||
if filepath.IsAbs(d) || d == ".." || len(d) > 3 && d[:3] == "../" {
|
||||
t.Errorf("unsafe skeleton entry %q", d)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(got, "ok/dir") {
|
||||
t.Error("a legitimate entry alongside bad ones must still be kept")
|
||||
}
|
||||
// "/abs/path" is not dropped outright — it is normalised to a relative path and kept, which is
|
||||
// safe (it lands under the userdata root). Pin that so the behaviour is a decision, not a guess.
|
||||
if !slices.Contains(got, "abs/path") {
|
||||
t.Errorf("an absolute entry should be normalised to relative, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// EnsureUserdataSkeleton creates every dir it is given and NOTHING ELSE, and never removes.
|
||||
func TestEnsureUserdataSkeletonCreatesOnly(t *testing.T) {
|
||||
ns := t.TempDir()
|
||||
// A pre-existing customer dir that no catalog app implies and the carry-list does not contain.
|
||||
stray := filepath.Join(UserdataDir(ns), "sajat-mappa")
|
||||
if err := os.MkdirAll(stray, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dirs := BuildUserdataSkeleton([]string{"media/podcasts"})
|
||||
if err := EnsureUserdataSkeleton(ns, dirs); err != nil {
|
||||
// chown to gid 1000 fails for a non-root test user; the dirs are still created.
|
||||
t.Logf("EnsureUserdataSkeleton returned %v (expected when not running as root)", err)
|
||||
}
|
||||
for _, d := range dirs {
|
||||
if fi, err := os.Stat(filepath.Join(UserdataDir(ns), d)); err != nil || !fi.IsDir() {
|
||||
t.Errorf("skeleton dir %q not created: %v", d, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(stray); err != nil {
|
||||
t.Errorf("a pre-existing customer dir was removed — zero-removals violated: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,10 @@ package appbackup
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Customer-facing userdata layout + the shared-storage ownership convention (v0.66.0).
|
||||
@@ -28,9 +31,41 @@ func UserdataDir(nsRoot string) string {
|
||||
return filepath.Join(nsRoot, "userdata")
|
||||
}
|
||||
|
||||
// UserdataSkeleton is the standard subtree created on every storage path (relative to UserdataDir).
|
||||
// ImportDirName is the single import (drop-zone) subtree name under a userdata root.
|
||||
const ImportDirName = "import"
|
||||
|
||||
// ImportDir returns the CANONICAL drop-zone root under a namespace root (R-75).
|
||||
//
|
||||
// Unlike every other userdata dir, this one is drive-INDEPENDENT: the caller resolves it against the
|
||||
// SYSTEM drive's namespace root, never against the app's own HDD_PATH, so a multi-drive box has
|
||||
// exactly ONE import tree. That is the whole point. Each drop-zone app has exactly one ingest bind,
|
||||
// so a per-drive import/ would put a folder that LOOKS like a drop-zone on every drive while only
|
||||
// one of them does anything — and because import paths are `class: excluded`, files stranded in a
|
||||
// dead one are never backed up either.
|
||||
//
|
||||
// It deliberately stays INSIDE the userdata tree, so the 2775/setgid/GID-1000 convention, the
|
||||
// FileBrowser mount and the ownership rules all apply to it unchanged.
|
||||
func ImportDir(nsRoot string) string {
|
||||
return filepath.Join(UserdataDir(nsRoot), ImportDirName)
|
||||
}
|
||||
|
||||
// UserdataSkeletonCarry is the explicit NON-DERIVED carry-list: every entry the v0.171.0 hardcoded
|
||||
// skeleton created, retained verbatim and forever.
|
||||
//
|
||||
// It exists so the catalog-derived skeleton (R-75) can only ever ADD. That makes the zero-removals
|
||||
// invariant true BY CONSTRUCTION rather than by review, and it is not hypothetical:
|
||||
//
|
||||
// - `documents` is implied by NO catalog app (SPIKE P0(a)) yet exists on both demo boxes and is
|
||||
// customer-visible — it may hold customer files. Derivation alone would drop it.
|
||||
// - `import/paperless` and `import/calibre` moved to the canonical system-drive root in R-75, so
|
||||
// derivation no longer implies them under a data drive either. The pre-existing ones stay put;
|
||||
// nothing in this arc deletes a directory.
|
||||
//
|
||||
// It doubles as the fresh-box floor: on a box whose catalog has not synced yet the derived set is
|
||||
// empty, and the customer still gets the full standard tree instead of a nearly-empty one.
|
||||
//
|
||||
// ASCII, no spaces (flows through ${} interpolation, shell, and the rsync merge walk).
|
||||
func UserdataSkeleton() []string {
|
||||
func UserdataSkeletonCarry() []string {
|
||||
return []string{
|
||||
"media", "media/movies", "media/tv", "media/music", "media/audiobooks",
|
||||
"media/books", "media/comics", "media/photos",
|
||||
@@ -41,6 +76,41 @@ func UserdataSkeleton() []string {
|
||||
}
|
||||
}
|
||||
|
||||
// BuildUserdataSkeleton merges the catalog-derived dirs with the carry-list into the final, SORTED
|
||||
// set. Each entry is expanded to its ancestor chain ("media/podcasts" implies "media"), deduped, and
|
||||
// sorted.
|
||||
//
|
||||
// SORTING IS A HARD REQUIREMENT, not tidiness. The FileBrowser config is regenerated from this set
|
||||
// and fbNeedsRecreate force-recreates the container on ANY byte difference. Go randomises map
|
||||
// iteration, and the spike measured the naive map-order derivation producing 20 DISTINCT outputs from
|
||||
// 20 identical runs (SPIKE P6) — which across SyncFileBrowserMounts' ~14 call sites is a fleet-wide
|
||||
// FileBrowser restart loop. TestSkeletonDeterminism pins this.
|
||||
func BuildUserdataSkeleton(derived []string) []string {
|
||||
set := make(map[string]bool, len(derived)+16)
|
||||
addChain := func(rel string) {
|
||||
rel = path.Clean(strings.TrimPrefix(filepath.ToSlash(rel), "/"))
|
||||
if rel == "" || rel == "." || rel == ".." || strings.HasPrefix(rel, "../") {
|
||||
return // never let a traversal or an empty entry become a directory to create
|
||||
}
|
||||
parts := strings.Split(rel, "/")
|
||||
for i := range parts {
|
||||
set[strings.Join(parts[:i+1], "/")] = true
|
||||
}
|
||||
}
|
||||
for _, d := range UserdataSkeletonCarry() {
|
||||
addChain(d)
|
||||
}
|
||||
for _, d := range derived {
|
||||
addChain(d)
|
||||
}
|
||||
out := make([]string, 0, len(set))
|
||||
for d := range set { // map order is RANDOM — the sort below is what makes this deterministic
|
||||
out = append(out, d)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// EnsureDirOwned creates path (idempotent) and enforces the convention: mode 2775 via an explicit
|
||||
// Chmod incl. setgid (MkdirAll cannot) + group = gid. Setting an arbitrary group needs CAP_CHOWN —
|
||||
// the in-guest controller runs as root, so this succeeds in production. Returns the first hard error.
|
||||
@@ -60,7 +130,11 @@ func EnsureUserdataDir(path string) error { return EnsureDirOwned(path, SharedCo
|
||||
// EnsureUserdataSkeleton creates the full userdata tree under a namespace root with the convention.
|
||||
// It creates ALL dirs even if one errors (so a single chown/chmod hiccup doesn't truncate the tree),
|
||||
// returning the first error seen for the caller to log.
|
||||
func EnsureUserdataSkeleton(nsRoot string) error {
|
||||
//
|
||||
// dirs is the merged, sorted set from BuildUserdataSkeleton. This function only ever CREATES: there
|
||||
// is no removal path here or anywhere in R-75, so a directory the current catalog no longer implies
|
||||
// simply stays where it is (Scenario D).
|
||||
func EnsureUserdataSkeleton(nsRoot string, dirs []string) error {
|
||||
base := UserdataDir(nsRoot)
|
||||
var firstErr error
|
||||
rec := func(e error) {
|
||||
@@ -69,7 +143,7 @@ func EnsureUserdataSkeleton(nsRoot string) error {
|
||||
}
|
||||
}
|
||||
rec(EnsureUserdataDir(base))
|
||||
for _, sub := range UserdataSkeleton() {
|
||||
for _, sub := range dirs {
|
||||
rec(EnsureUserdataDir(filepath.Join(base, sub)))
|
||||
}
|
||||
return firstErr
|
||||
|
||||
@@ -13,10 +13,12 @@ func TestSharedContentGID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestUserdataSkeleton_List asserts the locked skeleton subdir set.
|
||||
// TestUserdataSkeleton_List asserts the locked skeleton subdir set. R-75 renamed the hardcoded list
|
||||
// to UserdataSkeletonCarry (it is now the non-derived carry-list); the asserted set is UNCHANGED,
|
||||
// which is exactly the zero-removals promise.
|
||||
func TestUserdataSkeleton_List(t *testing.T) {
|
||||
got := map[string]bool{}
|
||||
for _, s := range UserdataSkeleton() {
|
||||
for _, s := range UserdataSkeletonCarry() {
|
||||
got[s] = true
|
||||
}
|
||||
for _, want := range []string{
|
||||
@@ -41,9 +43,10 @@ func TestUserdataDir(t *testing.T) {
|
||||
// is ignored — dirs + setgid still land). Runs cross-platform.
|
||||
func TestEnsureUserdataSkeleton_Structure(t *testing.T) {
|
||||
ns := t.TempDir()
|
||||
_ = EnsureUserdataSkeleton(ns) // ignore chown error on a non-root CI host
|
||||
dirs := BuildUserdataSkeleton(nil) // no catalog derived → the carry-list floor
|
||||
_ = EnsureUserdataSkeleton(ns, dirs) // ignore chown error on a non-root CI host
|
||||
base := UserdataDir(ns)
|
||||
for _, sub := range append([]string{""}, UserdataSkeleton()...) {
|
||||
for _, sub := range append([]string{""}, dirs...) {
|
||||
p := filepath.Join(base, sub)
|
||||
if fi, err := os.Stat(p); err != nil || !fi.IsDir() {
|
||||
t.Errorf("skeleton dir missing: %s (%v)", p, err)
|
||||
|
||||
Reference in New Issue
Block a user