308 lines
11 KiB
Go
308 lines
11 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The stray-copy fix (tier2_copy_lifecycle.go): the capped copy's container never outlives the copy, a leftover from a
|
|
// lost controller process is removed at start, and a restore is refused while a copy runs or may still run.
|
|
// RED-PROOF (2026-10-22 night): each test below was seen failing with its mechanism removed — the rm -f after an
|
|
// error (TimeoutRemoves, ErrorRemoves), the --name/--label args (NamedAndLabelled), the start-up loop (StartUpRemoves),
|
|
// the stray flag (FailedStartUp, Unremovable), the in-flight record (RefusedWhileACopyRuns).
|
|
|
|
type dockerRec struct {
|
|
mu sync.Mutex
|
|
calls []string
|
|
}
|
|
|
|
func (r *dockerRec) all() string {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
return strings.Join(r.calls, "\n")
|
|
}
|
|
|
|
// strayCopySeams stubs every Docker seam; docker answers for `rm`/`ps` come from fn.
|
|
func strayCopySeams(t *testing.T, root string, run func(ctx context.Context, args []string) ([]byte, int, error),
|
|
fn func(args []string) ([]byte, error)) (*dockerRec, *dockerRec) {
|
|
t.Helper()
|
|
runs, lifecycle := &dockerRec{}, &dockerRec{}
|
|
oldRoot, oldImg, oldRun, oldLog, oldDocker, oldTO, oldRetry := tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf, tier2Docker, tier2CopyTimeout, tier2RetryEvery
|
|
tier2RetryEvery = time.Hour
|
|
tier2RetryOff = true // the background retry never starts in a test (it would reach the real Docker later)
|
|
tier2CappedRoot = root
|
|
tier2SelfImage = func() string { return "sha256:abc" }
|
|
tier2RunCapped = func(ctx context.Context, args []string) ([]byte, int, error) {
|
|
runs.mu.Lock()
|
|
runs.calls = append(runs.calls, strings.Join(args, " "))
|
|
runs.mu.Unlock()
|
|
return run(ctx, args)
|
|
}
|
|
tier2Docker = func(_ context.Context, args ...string) ([]byte, error) {
|
|
lifecycle.mu.Lock()
|
|
lifecycle.calls = append(lifecycle.calls, strings.Join(args, " "))
|
|
lifecycle.mu.Unlock()
|
|
return fn(args)
|
|
}
|
|
tier2Logf = func(string, ...any) {}
|
|
tier2ResetForTest()
|
|
t.Cleanup(func() {
|
|
tier2CappedRoot, tier2SelfImage, tier2RunCapped, tier2Logf, tier2Docker, tier2CopyTimeout, tier2RetryEvery = oldRoot, oldImg, oldRun, oldLog, oldDocker, oldTO, oldRetry
|
|
tier2RetryOff = false
|
|
tier2ResetForTest()
|
|
})
|
|
return runs, lifecycle
|
|
}
|
|
|
|
func appDirs(t *testing.T) (root, src, dst string) {
|
|
root = t.TempDir() + "/"
|
|
src = filepath.Join(root, "d1", "backups", "primary", "immich")
|
|
dst = filepath.Join(root, "d2", "backups", "secondary", "immich", "recovery-unit")
|
|
os.MkdirAll(src, 0755)
|
|
return
|
|
}
|
|
|
|
func TestStrayCopy_TheCopyContainerIsNamedAndLabelled(t *testing.T) {
|
|
root, src, dst := appDirs(t)
|
|
runs, life := strayCopySeams(t, root, func(context.Context, []string) ([]byte, int, error) { return nil, 0, nil },
|
|
func([]string) ([]byte, error) { return nil, nil })
|
|
if err := rsyncMirror(src, dst); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
name, app := tier2CopyName(dst)
|
|
if app != "immich" || !strings.HasPrefix(name, "felhom-tier2-copy-immich-") {
|
|
t.Fatalf("name %q app %q", name, app)
|
|
}
|
|
got := runs.all()
|
|
for _, want := range []string{"--name " + name, "--label felhom.tier2-copy=1", "--label felhom.tier2-copy.app=immich"} {
|
|
if !strings.Contains(got, want) {
|
|
t.Fatalf("the copy container lacks %q:\n%s", want, got)
|
|
}
|
|
}
|
|
if life.all() != "rm -f "+name {
|
|
t.Fatalf("a leftover of the same copy must be removed before it starts; lifecycle calls:\n%s", life.all())
|
|
}
|
|
}
|
|
|
|
func TestStrayCopy_ATimeoutRemovesTheContainerByName(t *testing.T) {
|
|
root, src, dst := appDirs(t)
|
|
_, life := strayCopySeams(t, root, func(ctx context.Context, _ []string) ([]byte, int, error) {
|
|
<-ctx.Done() // the docker CLI is killed by the context; the container would keep running
|
|
return nil, -1, errors.New("signal: killed")
|
|
}, func([]string) ([]byte, error) { return nil, nil })
|
|
tier2CopyTimeout = 50 * time.Millisecond
|
|
err := rsyncMirror(src, dst)
|
|
if err == nil || !strings.Contains(err.Error(), "timed out") {
|
|
t.Fatalf("a timed-out copy must be an error that says so: %v", err)
|
|
}
|
|
name, _ := tier2CopyName(dst)
|
|
if c := strings.Count(life.all(), "rm -f "+name); c != 2 {
|
|
t.Fatalf("the container must be removed by name after the timeout (rm calls %d):\n%s", c, life.all())
|
|
}
|
|
}
|
|
|
|
func TestStrayCopy_AnErrorRemovesTheContainerByName(t *testing.T) {
|
|
root, src, dst := appDirs(t)
|
|
_, life := strayCopySeams(t, root, func(context.Context, []string) ([]byte, int, error) {
|
|
return []byte("rsync error"), 23, errors.New("exit 23")
|
|
}, func([]string) ([]byte, error) { return nil, nil })
|
|
if err := rsyncMirror(src, dst); err == nil {
|
|
t.Fatal("an rsync failure must be an error")
|
|
}
|
|
name, _ := tier2CopyName(dst)
|
|
if c := strings.Count(life.all(), "rm -f "+name); c != 2 {
|
|
t.Fatalf("the container must be removed by name after an error (rm calls %d)", c)
|
|
}
|
|
}
|
|
|
|
func TestStrayCopy_AnUnremovableLeftoverStopsTheCopy(t *testing.T) {
|
|
root, src, dst := appDirs(t)
|
|
runs, _ := strayCopySeams(t, root, func(context.Context, []string) ([]byte, int, error) { return nil, 0, nil },
|
|
func([]string) ([]byte, error) {
|
|
return []byte("Error response from daemon: cannot remove"), errors.New("exit 1")
|
|
})
|
|
if err := rsyncMirror(src, dst); err == nil {
|
|
t.Fatal("a copy must not start beside a leftover it could not remove")
|
|
}
|
|
if runs.all() != "" {
|
|
t.Fatalf("the copy started anyway: %s", runs.all())
|
|
}
|
|
if _, blocked := (&Manager{}).Tier2CopyBlocksRestore(); !blocked {
|
|
t.Fatal("a leftover that may still run must refuse restores")
|
|
}
|
|
}
|
|
|
|
func TestStrayCopy_ANotStartedContainerThatIsGoneStillFallsBack(t *testing.T) {
|
|
root, src, dst := appDirs(t)
|
|
os.WriteFile(filepath.Join(src, "f"), []byte("x"), 0644)
|
|
strayCopySeams(t, root, func(context.Context, []string) ([]byte, int, error) {
|
|
return []byte("no such image"), 125, errors.New("exit 125")
|
|
},
|
|
func(args []string) ([]byte, error) {
|
|
return []byte("Error: No such container: x"), errors.New("exit 1")
|
|
})
|
|
if err := rsyncMirror(src, dst); err != nil {
|
|
t.Fatalf("exit 125 and no container left → the in-process copy: %v", err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dst, "f")); err != nil {
|
|
t.Fatal("the fallback copy did not run")
|
|
}
|
|
}
|
|
|
|
func TestStrayCopy_StartUpRemovesEveryLeftover(t *testing.T) {
|
|
_, life := strayCopySeams(t, t.TempDir()+"/", nil, func(args []string) ([]byte, error) {
|
|
if args[0] == "ps" {
|
|
return []byte("felhom-tier2-copy-immich-0a0b0c0d\nfelhom-tier2-copy-nextcloud-01020304\n"), nil
|
|
}
|
|
return nil, nil
|
|
})
|
|
if err := ClearLeftoverTier2Copies(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := life.all()
|
|
for _, want := range []string{"ps -a --filter label=felhom.tier2-copy=1 --format {{.Names}}",
|
|
"rm -f felhom-tier2-copy-immich-0a0b0c0d", "rm -f felhom-tier2-copy-nextcloud-01020304"} {
|
|
if !strings.Contains(got, want) {
|
|
t.Fatalf("start-up lacks %q:\n%s", want, got)
|
|
}
|
|
}
|
|
if _, blocked := (&Manager{}).Tier2CopyBlocksRestore(); blocked {
|
|
t.Fatal("after a clean start-up removal restores must be allowed")
|
|
}
|
|
}
|
|
|
|
func TestStrayCopy_AFailedStartUpCheckRefusesRestores(t *testing.T) {
|
|
strayCopySeams(t, t.TempDir()+"/", nil, func([]string) ([]byte, error) { return nil, errors.New("docker down") })
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel() // ends the retry loop
|
|
if err := ClearLeftoverTier2Copies(ctx); err == nil {
|
|
t.Fatal("a failed check must be reported")
|
|
}
|
|
if _, blocked := (&Manager{}).Tier2CopyBlocksRestore(); !blocked {
|
|
t.Fatal("until the start-up check succeeds, restores must be refused")
|
|
}
|
|
}
|
|
|
|
func TestStrayCopy_ARestoreIsRefusedWhileACopyRuns(t *testing.T) {
|
|
root, src, dst := appDirs(t)
|
|
inside := make(chan struct{})
|
|
release := make(chan struct{})
|
|
strayCopySeams(t, root, func(context.Context, []string) ([]byte, int, error) {
|
|
close(inside)
|
|
<-release
|
|
return nil, 0, nil
|
|
}, func([]string) ([]byte, error) { return nil, nil })
|
|
done := make(chan error)
|
|
go func() { done <- rsyncMirror(src, dst) }()
|
|
<-inside
|
|
m := &Manager{}
|
|
if app, blocked := m.Tier2CopyBlocksRestore(); !blocked || app != "immich" {
|
|
t.Fatalf("during a copy a restore must be refused (app %q, blocked %v)", app, blocked)
|
|
}
|
|
close(release)
|
|
if err := <-done; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, blocked := m.Tier2CopyBlocksRestore(); blocked {
|
|
t.Fatal("after the copy a restore must be allowed")
|
|
}
|
|
}
|
|
|
|
// Review 2026-10-23 night. N2: a stray clears when a later removal of that name works (it used to block every restore
|
|
// until a restart). N3: the start-up check never removes a copy THIS process runs. N4: a second copy into the same
|
|
// destination waits for the first instead of removing its container. RED-PROOF: with the stray delete on a successful
|
|
// retry removed → StrayClears fails; with the `running` skip removed → OwnCopyKept fails; with the per-destination lock
|
|
// removed → SameDestinationWaits fails — each seen failing 2026-10-23 night.
|
|
func TestStrayCopy_AStrayClearsWhenALaterRemovalWorks(t *testing.T) {
|
|
root, src, dst := appDirs(t)
|
|
fail := true
|
|
strayCopySeams(t, root, func(context.Context, []string) ([]byte, int, error) { return nil, 0, nil },
|
|
func(args []string) ([]byte, error) {
|
|
if fail {
|
|
return []byte("Error response from daemon: cannot remove"), errors.New("exit 1")
|
|
}
|
|
return nil, nil
|
|
})
|
|
_ = rsyncMirror(src, dst)
|
|
if _, blocked := (&Manager{}).Tier2CopyBlocksRestore(); !blocked {
|
|
t.Fatal("fixture: an unremovable leftover must refuse restores")
|
|
}
|
|
fail = false
|
|
if !tier2RetryOnce(context.Background()) {
|
|
t.Fatal("the retry must report nothing left once the removal works")
|
|
}
|
|
if _, blocked := (&Manager{}).Tier2CopyBlocksRestore(); blocked {
|
|
t.Fatal("after the stray is removed, restores must be allowed again — without a controller restart")
|
|
}
|
|
}
|
|
|
|
func TestStrayCopy_TheStartUpCheckKeepsThisProcessesOwnCopy(t *testing.T) {
|
|
_, life := strayCopySeams(t, t.TempDir()+"/", nil, func(args []string) ([]byte, error) {
|
|
if args[0] == "ps" {
|
|
return []byte("felhom-tier2-copy-mine-00000001\nfelhom-tier2-copy-old-00000002\n"), nil
|
|
}
|
|
return nil, nil
|
|
})
|
|
tier2MarkRunning("felhom-tier2-copy-mine-00000001", true)
|
|
if err := clearLeftoverTier2CopiesOnce(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := life.all()
|
|
if strings.Contains(got, "rm -f felhom-tier2-copy-mine-00000001") || !strings.Contains(got, "rm -f felhom-tier2-copy-old-00000002") {
|
|
t.Fatalf("the check must remove only the leftover, never this process's running copy:\n%s", got)
|
|
}
|
|
}
|
|
|
|
func TestStrayCopy_ASecondCopyOfTheSameDestinationWaits(t *testing.T) {
|
|
root, src, dst := appDirs(t)
|
|
var mu sync.Mutex
|
|
order := []string{}
|
|
release := make(chan struct{})
|
|
first := true
|
|
runs, _ := strayCopySeams(t, root, func(context.Context, []string) ([]byte, int, error) {
|
|
mu.Lock()
|
|
me := first
|
|
first = false
|
|
order = append(order, "start")
|
|
mu.Unlock()
|
|
if me {
|
|
<-release
|
|
}
|
|
mu.Lock()
|
|
order = append(order, "end")
|
|
mu.Unlock()
|
|
return nil, 0, nil
|
|
}, func(args []string) ([]byte, error) {
|
|
mu.Lock()
|
|
order = append(order, strings.Join(args, " "))
|
|
mu.Unlock()
|
|
return nil, nil
|
|
})
|
|
d1, d2 := make(chan error), make(chan error)
|
|
go func() { d1 <- rsyncMirror(src, dst) }()
|
|
for len(strings.Split(runs.all(), "\n")) < 1 || runs.all() == "" {
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
go func() { d2 <- rsyncMirror(src, dst) }()
|
|
time.Sleep(100 * time.Millisecond)
|
|
mu.Lock()
|
|
during := strings.Join(order, "|")
|
|
mu.Unlock()
|
|
if strings.Count(during, "rm -f") != 1 {
|
|
t.Fatalf("while the first copy runs, the second must not touch its container: %s", during)
|
|
}
|
|
close(release)
|
|
if err := <-d1; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := <-d2; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|