package backup // R-1251 (review round 2, HUNT-1 H1, H2, H7): a pending put-back blocks every start. The first test is the review's // lean-clone proof (audits/review-round-2-2026-10-24/proofs/H1-drive-cut-restore/) turned round: the same cut, the same // unbound drive, and now the app is HELD until the copy is poured back, and a later start pours nothing. // // RED-PROOF (2026-10-24, each seen failing on main 2449223 before appstop_putback_wait.go): // - TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart: „boot 1 must HOLD the app as waiting" — // Recover read the copy as gone and only REFUSED the start, with no hold; // - TestR1251_AFailedPutBackHoldsTheApp: „a failed put-back must leave a restore_mixed hold" — none was written; // - TestR1251_StopPressedDuringTheRestoreStillPutsBack: „the put-back must run" — R-721's skip came first. import ( "errors" "io" "log" "os" "path/filepath" "strings" "testing" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // r1251Rig is one app (immich) whose restore was cut after its copy was recorded, the copy on the app's drive. type r1251Rig struct { marker, drive, undo string sett *settings.Settings puts []string starts []string stopped map[string]bool putErr error } func newR1251Rig(t *testing.T) *r1251Rig { t.Helper() root := t.TempDir() r := &r1251Rig{ marker: filepath.Join(root, "appstop-state.json"), drive: filepath.Join(root, "drive"), stopped: map[string]bool{}, } r.undo = filepath.Join(r.drive, "felhom-data", "backups", "restore-undo", "immich", "20261024T120000Z") mustWrite(t, filepath.Join(r.undo, "undo.json"), `{"stack":"immich","copied":["immich_pgdata"]}`) sett, err := settings.Load(filepath.Join(root, "settings.json"), log.New(io.Discard, "", 0)) if err != nil { t.Fatal(err) } r.sett = sett g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0)) if err := g.Begin("unit-restore:immich", ReasonUnitRestore, []string{"immich"}); err != nil { t.Fatal(err) } if err := g.SetUndo("immich", r.undo); err != nil { t.Fatal(err) } return r // the power cut: no End } func (r *r1251Rig) unbind(t *testing.T) { t.Helper() if err := os.Rename(r.drive, r.drive+".unbound"); err != nil { t.Fatal(err) } } func (r *r1251Rig) bind(t *testing.T) { t.Helper() if err := os.Rename(r.drive+".unbound", r.drive); err != nil { t.Fatal(err) } } func (r *r1251Rig) driveBound() bool { _, err := os.Stat(r.drive); return err == nil } // r1251Starter is the gated starter: it refuses while the drive is absent or a hold is set (as driveStartGate does). type r1251Starter struct{ r *r1251Rig } func (s r1251Starter) StartStack(name string) error { if _, held := s.r.sett.GetRestoreHold(name); held { return ErrStartRefused } if !s.r.driveBound() { return ErrStartRefused } s.r.starts = append(s.r.starts, name) return nil } func (s r1251Starter) WantsStopped(name string) bool { return s.r.stopped[name] } type r1251PutBacker struct{ r *r1251Rig } func (p r1251PutBacker) PutBackInterruptedRestore(stack, dir string) error { if p.r.putErr != nil { return p.r.putErr } p.r.puts = append(p.r.puts, stack) return os.RemoveAll(dir) // the real put-back removes the used copy } // guard is a fresh controller start over the rig's disk, wired as main.go wires it. func (r *r1251Rig) guard() *AppStopGuard { g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0)) g.SetStarter(r1251Starter{r}) g.SetPutBacker(r1251PutBacker{r}) g.SetHolds(r.sett) g.SetUndoReachable(func(string, string) bool { return r.driveBound() }) return g } func (r *r1251Rig) markerActive() bool { b, err := os.ReadFile(r.marker) return err == nil && strings.Contains(string(b), `"active": true`) } func TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart(t *testing.T) { r := newR1251Rig(t) // boot 1: the drive is not bound yet r.unbind(t) g1 := r.guard() rec := g1.Recover() h, held := r.sett.GetRestoreHold("immich") if rec == nil || len(rec.Waiting) != 1 || !held || h.Reason != settings.HoldReasonPutBackPending { t.Fatalf("boot 1 must HOLD the app as waiting for its drive: rec=%+v hold=%+v held=%v", rec, h, held) } if h.SafetyDump != r.undo { t.Fatalf("the hold must name the copy, got %q", h.SafetyDump) } if len(r.starts) != 0 || len(r.puts) != 0 || rec.Alarming() { t.Fatalf("nothing may start or be poured while the drive is absent: starts=%v puts=%v alarming=%v", r.starts, r.puts, rec.Alarming()) } if !r.markerActive() { t.Fatal("the marker must be kept while the put-back waits") } // a resume while the drive is still absent does nothing and keeps the hold if g1.ResumePutBack("immich") { t.Fatal("ResumePutBack must say still-held while the copy cannot be reached") } if _, still := r.sett.GetRestoreHold("immich"); !still { t.Fatal("the hold must stay while the drive is absent") } // the drive binds: the boot recreate (and the drive-return sweep) call ResumePutBack before they start the app r.bind(t) if !g1.ResumePutBack("immich") { t.Fatal("with the drive bound the put-back must run and allow the start") } if strings.Join(r.puts, ",") != "immich" { t.Fatalf("the copy must be poured back exactly once, puts=%v", r.puts) } if _, still := r.sett.GetRestoreHold("immich"); still { t.Fatal("the waiting hold must be lifted after the put-back") } if r.markerActive() { t.Fatal("the marker must let go of the app after its put-back") } if g1.ResumePutBack("immich") != true || len(r.puts) != 1 { t.Fatalf("a second resume must pour nothing, puts=%v", r.puts) } // the household writes on the drive, then any later controller start: nothing is poured back householdWrite := filepath.Join(r.drive, "felhom-data", "new-photo-after-the-put-back") mustWrite(t, householdWrite, "x") if rec2 := r.guard().Recover(); rec2 != nil { t.Fatalf("a later start must find nothing to recover, got %+v", rec2) } if len(r.puts) != 1 { t.Fatalf("a later start poured the copy again over the household's writes: puts=%v", r.puts) } if _, err := os.Stat(householdWrite); err != nil { t.Fatal("the household's newest file must still be there") } } // Every start path is refused while the put-back waits: they all read the ONE hold store. The night chain and the // household's buttons read it through RestoreHoldFor (isHeld, the router's restoreHoldFor, the update guards); the // boot sweep and Recover through driveStartGate's GetRestoreHold; the web paths (boot recreate, drive return, media // restart) through appHeld. The web and main.go wiring is pinned in their own packages. func TestR1251_TheWaitingHoldRefusesEveryStartPath(t *testing.T) { r := newR1251Rig(t) r.unbind(t) r.guard().Recover() m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)} held, why := m.RestoreHoldForLang("immich", "hu") if !held || !strings.Contains(why, "meghajt") || !strings.Contains(why, "immich") { t.Fatalf("the household's sentence must name the app and the drive: held=%v %q", held, why) } if _, en := m.RestoreHoldForLang("immich", "en"); !strings.Contains(en, "drive is back") { t.Fatalf("the English sentence: %q", en) } if !m.isHeld("immich") { t.Fatal("the night chain's isHeld must see the waiting put-back") } // the gated starter (the boot sweep's and Recover's path) refuses too if err := (r1251Starter{r}).StartStack("immich"); !errors.Is(err, ErrStartRefused) { t.Fatalf("the gated start must be refused while held, got %v", err) } // a removal ends the install the copy belongs to: the waiting hold goes with it (a later install must not wait) if cleared, _ := r.sett.ClearUpdateHold("immich"); !cleared { t.Fatal("a removal must clear the waiting put-back hold") } } // H2: a put-back that fails at start leaves the app HELD (the D20 hold, „stopped half-way"), never stopped-and-unheld; // the copy stays and the marker lets go (the hold owns the app). func TestR1251_AFailedPutBackHoldsTheApp(t *testing.T) { r := newR1251Rig(t) r.putErr = errors.New("tar: short read") rec := r.guard().Recover() h, held := r.sett.GetRestoreHold("immich") if !held || h.Reason != settings.HoldReasonRestoreMixed { t.Fatalf("a failed put-back must leave a restore_mixed hold, got %+v held=%v", h, held) } if rec == nil || len(rec.Held) != 1 || !rec.Alarming() || len(r.starts) != 0 { t.Fatalf("held, alarming, not started: rec=%+v starts=%v", rec, r.starts) } if !strings.Contains(h.ReplayError, "short read") || h.SafetyDump != r.undo { t.Fatalf("the hold must carry the cause and the copy: %+v", h) } if _, err := os.Stat(r.undo); err != nil { t.Fatal("the copy must stay on disk for support") } if r.markerActive() { t.Fatal("the hold owns the app now — the marker must let go, so no later start pours an old copy over it") } // the household's Start, the boot recreate and the update are all refused by that hold m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)} if held, why := m.RestoreHoldForLang("immich", "hu"); !held || !strings.Contains(why, "immich") { t.Fatalf("the D20 sentence: held=%v %q", held, why) } } // A failed resume (the drive came back, the put-back broke) holds the app the same way. func TestR1251_AFailedResumeHoldsTheApp(t *testing.T) { r := newR1251Rig(t) r.unbind(t) g := r.guard() g.Recover() r.bind(t) r.putErr = errors.New("disk full") if g.ResumePutBack("immich") { t.Fatal("a failed put-back must not allow the start") } if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed { t.Fatalf("a failed resume must hold for support, got %+v", h) } if r.markerActive() { t.Fatal("the marker must let go after the hold took over") } } // A copy that vanished while its drive is BOUND, during a waiting put-back, cannot be poured: held for support. func TestR1251_ACopyGoneWhileItsDriveIsBoundHoldsTheApp(t *testing.T) { r := newR1251Rig(t) r.unbind(t) g := r.guard() g.Recover() r.bind(t) if err := os.RemoveAll(r.undo); err != nil { t.Fatal(err) } if g.ResumePutBack("immich") { t.Fatal("a vanished copy must not allow the start") } if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed { t.Fatalf("held for support, got %+v", h) } } // H7: Stop pressed during the restore — the copy is still poured back after the cut; only the start is skipped. func TestR1251_StopPressedDuringTheRestoreStillPutsBack(t *testing.T) { r := newR1251Rig(t) r.stopped["immich"] = true rec := r.guard().Recover() if strings.Join(r.puts, ",") != "immich" { t.Fatalf("the put-back must run even though the household pressed Stop, puts=%v rec=%+v", r.puts, rec) } if len(r.starts) != 0 { t.Fatalf("the household stopped it — it must not start, starts=%v", r.starts) } if r.markerActive() { t.Fatal("nothing is owed any more — the marker must clear") } } // The drive returns later (not at boot): the sweep puts back, then starts through the gated starter — unless the // household stopped the app. func TestR1251_ADriveThatReturnsLaterIsPutBackThenStarted(t *testing.T) { for _, stopped := range []bool{false, true} { r := newR1251Rig(t) r.unbind(t) g := r.guard() g.Recover() if got := g.ResumeWaitingPutBacks(); len(got) != 0 || len(r.puts) != 0 { t.Fatalf("drive still absent: nothing may happen, started=%v puts=%v", got, r.puts) } r.bind(t) r.stopped["immich"] = stopped got := g.ResumeWaitingPutBacks() if len(r.puts) != 1 { t.Fatalf("stopped=%v: the put-back must run once, puts=%v", stopped, r.puts) } if want := !stopped; (len(got) == 1) != want || (len(r.starts) == 1) != want { t.Fatalf("stopped=%v: started=%v starts=%v", stopped, got, r.starts) } } } // A later restore that completes drops the waiting put-back: the household chose that restore's data. func TestR1251_ACompletedRestoreDropsTheWaitingPutBack(t *testing.T) { r := newR1251Rig(t) r.unbind(t) r.guard().Recover() m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)} m.clearUpdateHoldAfterRestore("immich") if _, held := r.sett.GetRestoreHold("immich"); held { t.Fatal("a completed restore must drop the waiting put-back hold") } } // MEASURED on Tester 1 (2026-10-24, audits/restore-holes-2026-10-24/P1-tester1/): a power cut 2.1 s after the copy was // recorded left its journal as undo.json.tmp — the marker named a copy the start could not read. The copy's folders // must be synced BEFORE the marker names it. RED-PROOF: with the syncUndoDirs call removed from persistUndoForCrash this // fails on „no folder was synced before the marker named the copy". func TestR1251_TheCopyIsOnDiskBeforeTheMarkerNamesIt(t *testing.T) { root := t.TempDir() marker := filepath.Join(root, "appstop-state.json") copyDir := filepath.Join(root, "drive", "backups", "restore-undo", "app", "20261024T171928Z") if err := os.MkdirAll(copyDir, 0o700); err != nil { t.Fatal(err) } g := NewAppStopGuard(marker, log.New(io.Discard, "", 0)) if err := g.Begin("unit-restore:app", ReasonUnitRestore, []string{"app"}); err != nil { t.Fatal(err) } m := &Manager{logger: log.New(io.Discard, "", 0), appStop: g} var synced []string markerNamedCopyAtSync := false orig := syncUndoDirs t.Cleanup(func() { syncUndoDirs = orig }) syncUndoDirs = func(dir string) error { b, _ := os.ReadFile(marker) markerNamedCopyAtSync = strings.Contains(string(b), dir) synced = append(synced, dir) return orig(dir) } m.persistUndoForCrash("app", &liveVolumeCopy{dir: copyDir, copied: []string{"app_data"}}) if len(synced) != 1 || synced[0] != copyDir { t.Fatalf("no folder was synced before the marker named the copy: %v", synced) } if markerNamedCopyAtSync { t.Fatal("the marker already named the copy when its folders were synced — the order is the fix") } if b, _ := os.ReadFile(marker); !strings.Contains(string(b), copyDir) { t.Fatal("the copy must still be recorded in the marker") } if _, err := os.Stat(filepath.Join(copyDir, "undo.json")); err != nil { t.Fatal("the journal must be in place") } }