Second review of the night's own code: a stray clears when a later removal works (N2), the start-up check never removes this process's own copy (N3), one copy per destination at a time (N4), RunTier2 refuses during a restore for every caller (N5), the import refusal no longer advises removal (N9); red-proved, race-clean; README; REPORT; unreleased
gates / gates (push) Successful in 1m40s

This commit is contained in:
2026-10-22 22:45:42 +02:00
parent 943c517550
commit 9d9e4041c9
9 changed files with 341 additions and 62 deletions
+8
View File
@@ -1399,6 +1399,14 @@ retired in v0.126.0 when the moved blocks were legitimately rewritten onto the s
contain no symlink (`tier2CappedPath`, `filepath.EvalSymlinks(p) == p`: a link could point the mount elsewhere). A
docker start error (exit 125) falls back to plain rsync; any other failure is returned. Measured on 9202 (4 GB, 1.8 GB
held, 6 GB copy): full stall 5.7 s → 0.37 s. Tests `r1214_capped_copy_test.go`.
- **The copy container never outlives the copy (the stray-copy fix, unreleased, rides 0.340.0).** Each copy container has
a fixed name (`felhom-tier2-copy-<app>-<8 hex of the destination>`) and the label `felhom.tier2-copy=1`; a leftover of
the same copy is removed before it starts, and on any error — the timeout included — it is removed by name. At
controller start, BEFORE the scheduler and the web server, every labelled container is removed (logged; retried every
minute until it succeeds). A restore is refused while any second-drive copy runs or a leftover may run
(`backup.Tier2CopyBlocksRestore` → `restoreOpBlocked`). The run uses `--mount` (a path the daemon does not see is
refused, exit 125 → the in-process copy), the controller's image ID with `--pull never`, and `--no-healthcheck`.
Tests `tier2_copy_lifecycle_test.go`, `web/straycopy_restore_refused_test.go`, `cmd/controller/straycopy_wiring_test.go`.
- **A template may leave a named volume out of the app-data backups (unreleased, rides 0.340.0; R-49, operator D21 = B).**
`.felhom.yml` `backup.volumes: [{name: <volume>, class: excluded}]` — every named volume is dumped except
`<project>_<name>` (`appbackup.ExcludedDockerVolumes` → `backup.Manager.dumpableVolumes`), so it leaves Tier 1, Tier 2
+3 -2
View File
@@ -364,11 +364,12 @@ func (e *Exporter) executeImport(req ImportRequest, job *Job) {
// is a corruption shape), reported as success. Only a running app's volumes are removed first. So this import is
// refused here, before anything is written; the household starts the app (or removes it) and imports again. „Not
// running" is IsStackRunning's view: running or degraded only, so an unhealthy or restarting app is refused too —
// the text says so and names removal (review 2026-10-22 night, C3; the import path for those states is R-1187).
// the text says so; it does NOT advise removal, which would lose data newer than the bundle (review 2026-10-23
// night, C3 + N9; the import path for those states is R-1187).
// Pinned by TestR1187c_AStoppedAppIsNotImportedOver.
if isDeployed && !isRunning {
e.logger.Printf("[WARN] Import: refused — %s is installed but not running; importing over it would merge the old data with the bundle's (R-1187 c)", manifest.AppName)
e.failJob(job, step, "Az alkalmazás telepítve van, de nem fut rendben (leállt, vagy hibásan indul), ezért a csomagot nem töltjük vissza: a régi adatai összekeveredhetnének a csomagéval. Az alkalmazás érintetlen. Ha leállt, indítsd el; ha hibásan fut, távolítsd el. Utána próbáld újra.")
e.failJob(job, step, "Az alkalmazás telepítve van, de nem fut rendben (leállt, vagy hibásan indul), ezért a csomagot nem töltjük vissza: a régi adatai összekeveredhetnének a csomagéval. Az alkalmazás érintetlen. Ha leállt, indítsd el, és próbáld újra; ha hibásan fut, kérd az üzemeltető segítségét — ne távolítsd el, mert az adatai a csomagnál újabbak lehetnek.")
return
}
@@ -58,3 +58,25 @@ func TestR1180_TheNightMirrorNeverRunsDuringARestoreOfThatApp(t *testing.T) {
t.Fatalf("after the restore the mirror must run again, got %v", got)
}
}
// Review 2026-10-23 night, N5: RunTier2 itself refuses during a restore of that app — the config-save and debug paths
// call it directly. RED-PROOF: with the restoreRunningFor check at the top of RunTier2 removed, this fails — seen
// failing 2026-10-23 night.
func TestR1180_ADirectRunTier2NeverRunsDuringARestore(t *testing.T) {
m, src, target, prov := newTier2V2(t, "app")
prov.has["app"] = true
prov.binds["app"] = []ClassifiedBind{mHDD("appdata/app/photos")}
m.stackProvider = r1180Provider{prov}
called := 0
m.tier2Mirror = func(s, d string) error { called++; return rsyncMirror(s, d) }
mustWrite(t, filepath.Join(src, "appdata", "app", "photos", "a.jpg"), "a")
m.BeginRestoreOp("tier2-whole-restore", "app")
if err := m.RunTier2("app"); err != nil || called != 0 {
t.Fatalf("a direct run during a restore must copy nothing (err %v, mirror calls %d)", err, called)
}
m.EndRestoreOp(true, "")
if err := m.RunTier2("app"); err != nil || called == 0 {
t.Fatalf("after the restore the direct run must mirror (err %v, calls %d)", err, called)
}
_ = target
}
@@ -23,6 +23,7 @@ func withSeams(t *testing.T, root string, img string, run func(args []string) ([
oldRoot, oldImg, oldRun, oldLog, oldDocker := tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf, tier2Docker
// The copy's lifecycle commands (rm -f by name) never reach Docker from a test (they would act on the host).
tier2Docker = func(context.Context, ...string) ([]byte, error) { return nil, nil }
tier2RetryOff = true
tier2CappedRoot = root
tier2SelfImage = func() string { return img }
tier2RunCapped = func(_ context.Context, args []string) ([]byte, int, error) {
@@ -32,6 +33,7 @@ func withSeams(t *testing.T, root string, img string, run func(args []string) ([
tier2Logf = func(string, ...any) {}
t.Cleanup(func() {
tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf, tier2Docker = oldRoot, oldImg, oldRun, oldLog, oldDocker
tier2RetryOff = false
})
return rec
}
+12
View File
@@ -340,6 +340,12 @@ func (m *Manager) tier2ReconcileRoots(destBase string, roots, legRels []string)
// capture legs (§8). Best-effort and idempotent. Records status for the UI; returns an error only on
// an actual copy failure (no valid target is a recorded status, not an error).
func (m *Manager) RunTier2(stackName string) error {
// R-1180 for EVERY caller (review 2026-10-23 night, N5): the household's „Tier 2 on" save and the debug run call
// this directly, outside RunAllTier2's skip — a mirror (rsync --delete) must never start during a restore of the app.
if m.restoreRunningFor(stackName) {
m.logger.Printf("[WARN] [backup] Tier 2 for %s not started — a restore of it is running; it is mirrored at the next run (R-1180)", stackName)
return nil
}
// Customer turned Tier 2 off for this app (config panel) — skip without touching status.
if m.settings != nil {
if cd := m.settings.GetCrossDriveConfig(stackName); cd != nil && cd.UserDisabled {
@@ -797,6 +803,10 @@ func rsyncMirror(src, dst string) error {
s, d := strings.TrimRight(src, "/"), strings.TrimRight(dst, "/")
rs := []string{"-a", "--delete", s + "/", d + "/"}
name, app := tier2CopyName(d)
// One copy per destination at a time in this process (review N4): a second run waits, it never removes the first.
lk := tier2CopyLock(name)
lk.Lock()
defer lk.Unlock()
tier2CopyBegin(app)
defer tier2CopyEnd(app)
if tier2CappedPath(s) && tier2CappedPath(d) {
@@ -813,7 +823,9 @@ func rsyncMirror(src, dst string) error {
// created empty — an empty source would let `--delete` empty the copy (review 2026-10-22 night, C4).
"--mount", "type=bind,src=" + s + ",dst=" + s + ",readonly", "--mount", "type=bind,src=" + d + ",dst=" + d,
"--entrypoint", "rsync", img}, rs...)
tier2MarkRunning(name, true) // the start-up retry never removes a copy this process runs (review N3)
out, code, err := tier2RunCapped(ctx, args)
tier2MarkRunning(name, false)
if err == nil {
return nil
}
@@ -87,11 +87,39 @@ var tier2Docker = func(ctx context.Context, args ...string) ([]byte, error) {
}
// The controller's own record of copies (package-level: one controller, one backup manager per process).
// - inFlight: app → copies running now (capped or in-process);
// - running: the container names THIS process runs now — the start-up check never removes one of them;
// - strays: names whose removal failed — cleared when a later removal of that name works;
// - startupPending: the start-up check has not succeeded yet.
//
// Restores are refused while any of these is non-empty (review 2026-10-23 night, N2/N3: a stray used to block every
// restore until a restart, and the start-up retry used to remove this process's own running copies).
var tier2Copies = struct {
sync.Mutex
inFlight map[string]int // app → copies running now (capped or in-process)
stray bool // a container this process could not remove may still run, or the start-up removal has not succeeded
}{inFlight: map[string]int{}}
inFlight map[string]int
running map[string]bool
strays map[string]bool
startupPending bool
retrying bool
}{inFlight: map[string]int{}, running: map[string]bool{}, strays: map[string]bool{}}
// tier2CopyLocks serializes copies into the SAME destination within this process (review N4): a second run waits for
// the first instead of removing its container by name (which killed it mid-copy and mailed a false failure).
var tier2CopyLocks = struct {
sync.Mutex
m map[string]*sync.Mutex
}{m: map[string]*sync.Mutex{}}
func tier2CopyLock(name string) *sync.Mutex {
tier2CopyLocks.Lock()
defer tier2CopyLocks.Unlock()
l := tier2CopyLocks.m[name]
if l == nil {
l = &sync.Mutex{}
tier2CopyLocks.m[name] = l
}
return l
}
func tier2CopyBegin(app string) {
tier2Copies.Lock()
@@ -107,55 +135,131 @@ func tier2CopyEnd(app string) {
tier2Copies.Unlock()
}
func tier2SetStray(v bool) {
func tier2MarkRunning(name string, on bool) {
tier2Copies.Lock()
tier2Copies.stray = v
if on {
tier2Copies.running[name] = true
} else {
delete(tier2Copies.running, name)
}
tier2Copies.Unlock()
}
// tier2RemoveCopy removes the copy container `name` (no error when it does not exist). It reports whether the
// container is surely gone; when not, the stray flag is set, so restores stay refused until a removal succeeds.
// container is surely gone; when not, the name is recorded as a stray (restores refused) and retried every minute.
func tier2RemoveCopy(name, why string) bool {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
out, err := tier2Docker(ctx, "rm", "-f", name)
if err != nil && !strings.Contains(strings.ToLower(string(out)), "no such container") {
tier2Logf("[ERROR] [tier2] could not remove the copy container %s (%s): %v %s — restores stay refused until it is removed",
tier2Logf("[ERROR] [tier2] could not remove the copy container %s (%s): %v %s — restores stay refused until it is removed (retried every minute)",
name, why, err, strings.TrimSpace(string(out)))
tier2SetStray(true)
tier2Copies.Lock()
tier2Copies.strays[name] = true
tier2Copies.Unlock()
tier2StartRetry(context.Background())
return false
}
tier2Copies.Lock()
delete(tier2Copies.strays, name)
tier2Copies.Unlock()
return true
}
// ClearLeftoverTier2Copies removes every labelled copy container — a copy a previous controller process started and
// lost (restart, self-update, crash). It runs at controller start, BEFORE the scheduler and the web server; each
// removal is logged. Until it succeeds, restores are refused; on failure it retries every minute in the background
// until it succeeds or ctx ends.
func ClearLeftoverTier2Copies(ctx context.Context) error {
tier2SetStray(true)
err := clearLeftoverTier2CopiesOnce(ctx)
if err == nil {
tier2SetStray(false)
return nil
// tier2RetryEvery is the retry period for strays and a failed start-up check; tier2RetryOff (tests only) keeps the
// background retry from starting at all — a timer that outlives a test would reach the real Docker of the host.
var (
tier2RetryEvery = time.Minute
tier2RetryOff = false
)
// tier2StartRetry starts ONE background retry (if none runs): every tier2RetryEvery it redoes a failed start-up check
// and the removal of every stray, and ends when nothing is left or ctx ends.
func tier2StartRetry(ctx context.Context) {
tier2Copies.Lock()
if tier2Copies.retrying || tier2RetryOff {
tier2Copies.Unlock()
return
}
tier2Logf("[ERROR] [tier2] the start-up check for a leftover second-drive copy failed: %v — restores are refused until it succeeds; retrying every minute", err)
tier2Copies.retrying = true
tier2Copies.Unlock()
every := tier2RetryEvery
go func() {
t := time.NewTicker(time.Minute)
t := time.NewTicker(every)
defer t.Stop()
for {
select {
case <-ctx.Done():
tier2Copies.Lock()
tier2Copies.retrying = false
tier2Copies.Unlock()
return
case <-t.C:
if clearLeftoverTier2CopiesOnce(ctx) == nil {
tier2SetStray(false)
tier2Logf("[INFO] [tier2] the leftover second-drive copy check succeeded — restores allowed again")
return
}
}
if tier2RetryOnce(ctx) {
tier2Logf("[INFO] [tier2] no leftover or stray second-drive copy container is left — restores allowed again")
return
}
}
}()
}
// tier2RetryOnce does one retry round; it reports (and clears `retrying`) when nothing is left to retry.
func tier2RetryOnce(ctx context.Context) bool {
tier2Copies.Lock()
pending := tier2Copies.startupPending
var names []string
for n := range tier2Copies.strays {
names = append(names, n)
}
tier2Copies.Unlock()
if pending && clearLeftoverTier2CopiesOnce(ctx) == nil {
tier2Copies.Lock()
tier2Copies.startupPending = false
tier2Copies.Unlock()
}
for _, n := range names {
tier2RemoveCopyQuiet(ctx, n)
}
tier2Copies.Lock()
defer tier2Copies.Unlock()
if !tier2Copies.startupPending && len(tier2Copies.strays) == 0 {
tier2Copies.retrying = false
return true
}
return false
}
// tier2RemoveCopyQuiet is the retry's removal: a failure keeps the stray without starting another retry.
func tier2RemoveCopyQuiet(ctx context.Context, name string) {
rctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
out, err := tier2Docker(rctx, "rm", "-f", name)
if err != nil && !strings.Contains(strings.ToLower(string(out)), "no such container") {
return
}
tier2Copies.Lock()
delete(tier2Copies.strays, name)
tier2Copies.Unlock()
tier2Logf("[INFO] [tier2] the stray copy container %s is removed", name)
}
// ClearLeftoverTier2Copies removes every labelled copy container a previous controller process started and lost
// (restart, self-update, crash). It runs at controller start, BEFORE the scheduler and the web server; each removal is
// logged. Until it succeeds, restores are refused; on failure it is retried every minute in the background.
func ClearLeftoverTier2Copies(ctx context.Context) error {
tier2Copies.Lock()
tier2Copies.startupPending = true
tier2Copies.Unlock()
err := clearLeftoverTier2CopiesOnce(ctx)
if err == nil {
tier2Copies.Lock()
tier2Copies.startupPending = false
tier2Copies.Unlock()
return nil
}
tier2Logf("[ERROR] [tier2] the start-up check for a leftover second-drive copy failed: %v — restores are refused until it succeeds; retrying every minute", err)
tier2StartRetry(ctx)
return err
}
@@ -166,7 +270,14 @@ func clearLeftoverTier2CopiesOnce(ctx context.Context) error {
if err != nil {
return fmt.Errorf("docker ps: %v %s", err, strings.TrimSpace(string(out)))
}
names := strings.Fields(string(out))
var names []string
tier2Copies.Lock()
for _, n := range strings.Fields(string(out)) {
if !tier2Copies.running[n] { // never a copy THIS process runs (review N3)
names = append(names, n)
}
}
tier2Copies.Unlock()
if len(names) == 0 {
tier2Logf("[INFO] [tier2] start-up: no leftover second-drive copy container")
return nil
@@ -195,7 +306,7 @@ func clearLeftoverTier2CopiesOnce(ctx context.Context) error {
func (m *Manager) Tier2CopyBlocksRestore() (app string, blocked bool) {
tier2Copies.Lock()
defer tier2Copies.Unlock()
if tier2Copies.stray {
if tier2Copies.startupPending || len(tier2Copies.strays) > 0 {
return "", true
}
for a := range tier2Copies.inFlight {
@@ -204,6 +315,14 @@ func (m *Manager) Tier2CopyBlocksRestore() (app string, blocked bool) {
return "", false
}
// tier2ResetForTest empties the record (tests only).
func tier2ResetForTest() {
tier2Copies.Lock()
tier2Copies.inFlight, tier2Copies.running, tier2Copies.strays = map[string]int{}, map[string]bool{}, map[string]bool{}
tier2Copies.startupPending, tier2Copies.retrying = false, false
tier2Copies.Unlock()
}
// MarkTier2CopyForTest records a copy of app as running until the returned func is called — a test-only seam (another
// package's test exercises the restore refusal without a real copy), in the spirit of AcquireRunningForTest.
func MarkTier2CopyForTest(app string) func() {
@@ -33,7 +33,9 @@ func strayCopySeams(t *testing.T, root string, run func(ctx context.Context, arg
fn func(args []string) ([]byte, error)) (*dockerRec, *dockerRec) {
t.Helper()
runs, lifecycle := &dockerRec{}, &dockerRec{}
oldRoot, oldImg, oldRun, oldLog, oldDocker, oldTO := tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf, tier2Docker, tier2CopyTimeout
oldRoot, oldImg, oldRun, oldLog, oldDocker, oldTO, oldRetry := tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf, tier2Docker, tier2CopyTimeout, tier2RetryEvery
tier2RetryEvery = time.Hour
tier2RetryOff = true // the background retry never starts in a test (it would reach the real Docker later)
tier2CappedRoot = root
tier2SelfImage = func() string { return "sha256:abc" }
tier2RunCapped = func(ctx context.Context, args []string) ([]byte, int, error) {
@@ -49,10 +51,11 @@ func strayCopySeams(t *testing.T, root string, run func(ctx context.Context, arg
return fn(args)
}
tier2Logf = func(string, ...any) {}
tier2SetStray(false)
tier2ResetForTest()
t.Cleanup(func() {
tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf, tier2Docker, tier2CopyTimeout = oldRoot, oldImg, oldRun, oldLog, oldDocker, oldTO
tier2SetStray(false)
tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf, tier2Docker, tier2CopyTimeout, tier2RetryEvery = oldRoot, oldImg, oldRun, oldLog, oldDocker, oldTO, oldRetry
tier2RetryOff = false
tier2ResetForTest()
})
return runs, lifecycle
}
@@ -210,3 +213,95 @@ func TestStrayCopy_ARestoreIsRefusedWhileACopyRuns(t *testing.T) {
t.Fatal("after the copy a restore must be allowed")
}
}
// Review 2026-10-23 night. N2: a stray clears when a later removal of that name works (it used to block every restore
// until a restart). N3: the start-up check never removes a copy THIS process runs. N4: a second copy into the same
// destination waits for the first instead of removing its container. RED-PROOF: with the stray delete on a successful
// retry removed → StrayClears fails; with the `running` skip removed → OwnCopyKept fails; with the per-destination lock
// removed → SameDestinationWaits fails — each seen failing 2026-10-23 night.
func TestStrayCopy_AStrayClearsWhenALaterRemovalWorks(t *testing.T) {
root, src, dst := appDirs(t)
fail := true
strayCopySeams(t, root, func(context.Context, []string) ([]byte, int, error) { return nil, 0, nil },
func(args []string) ([]byte, error) {
if fail {
return []byte("Error response from daemon: cannot remove"), errors.New("exit 1")
}
return nil, nil
})
_ = rsyncMirror(src, dst)
if _, blocked := (&Manager{}).Tier2CopyBlocksRestore(); !blocked {
t.Fatal("fixture: an unremovable leftover must refuse restores")
}
fail = false
if !tier2RetryOnce(context.Background()) {
t.Fatal("the retry must report nothing left once the removal works")
}
if _, blocked := (&Manager{}).Tier2CopyBlocksRestore(); blocked {
t.Fatal("after the stray is removed, restores must be allowed again — without a controller restart")
}
}
func TestStrayCopy_TheStartUpCheckKeepsThisProcessesOwnCopy(t *testing.T) {
_, life := strayCopySeams(t, t.TempDir()+"/", nil, func(args []string) ([]byte, error) {
if args[0] == "ps" {
return []byte("felhom-tier2-copy-mine-00000001\nfelhom-tier2-copy-old-00000002\n"), nil
}
return nil, nil
})
tier2MarkRunning("felhom-tier2-copy-mine-00000001", true)
if err := clearLeftoverTier2CopiesOnce(context.Background()); err != nil {
t.Fatal(err)
}
got := life.all()
if strings.Contains(got, "rm -f felhom-tier2-copy-mine-00000001") || !strings.Contains(got, "rm -f felhom-tier2-copy-old-00000002") {
t.Fatalf("the check must remove only the leftover, never this process's running copy:\n%s", got)
}
}
func TestStrayCopy_ASecondCopyOfTheSameDestinationWaits(t *testing.T) {
root, src, dst := appDirs(t)
var mu sync.Mutex
order := []string{}
release := make(chan struct{})
first := true
runs, _ := strayCopySeams(t, root, func(context.Context, []string) ([]byte, int, error) {
mu.Lock()
me := first
first = false
order = append(order, "start")
mu.Unlock()
if me {
<-release
}
mu.Lock()
order = append(order, "end")
mu.Unlock()
return nil, 0, nil
}, func(args []string) ([]byte, error) {
mu.Lock()
order = append(order, strings.Join(args, " "))
mu.Unlock()
return nil, nil
})
d1, d2 := make(chan error), make(chan error)
go func() { d1 <- rsyncMirror(src, dst) }()
for len(strings.Split(runs.all(), "\n")) < 1 || runs.all() == "" {
time.Sleep(5 * time.Millisecond)
}
go func() { d2 <- rsyncMirror(src, dst) }()
time.Sleep(100 * time.Millisecond)
mu.Lock()
during := strings.Join(order, "|")
mu.Unlock()
if strings.Count(during, "rm -f") != 1 {
t.Fatalf("while the first copy runs, the second must not touch its container: %s", during)
}
close(release)
if err := <-d1; err != nil {
t.Fatal(err)
}
if err := <-d2; err != nil {
t.Fatal(err)
}
}