321 lines
12 KiB
Go
321 lines
12 KiB
Go
package backup
|
|
|
|
// R-1251 (review round 2, HUNT-1 H1, H2, H7): a pending put-back blocks every start. The first test is the review's
|
|
// lean-clone proof (audits/review-round-2-2026-10-24/proofs/H1-drive-cut-restore/) turned round: the same cut, the same
|
|
// unbound drive, and now the app is HELD until the copy is poured back, and a later start pours nothing.
|
|
//
|
|
// RED-PROOF (2026-10-24, each seen failing on main 2449223 before appstop_putback_wait.go):
|
|
// - TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart: „boot 1 must HOLD the app as waiting" —
|
|
// Recover read the copy as gone and only REFUSED the start, with no hold;
|
|
// - TestR1251_AFailedPutBackHoldsTheApp: „a failed put-back must leave a restore_mixed hold" — none was written;
|
|
// - TestR1251_StopPressedDuringTheRestoreStillPutsBack: „the put-back must run" — R-721's skip came first.
|
|
|
|
import (
|
|
"errors"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
// r1251Rig is one app (immich) whose restore was cut after its copy was recorded, the copy on the app's drive.
|
|
type r1251Rig struct {
|
|
marker, drive, undo string
|
|
sett *settings.Settings
|
|
puts []string
|
|
starts []string
|
|
stopped map[string]bool
|
|
putErr error
|
|
}
|
|
|
|
func newR1251Rig(t *testing.T) *r1251Rig {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
r := &r1251Rig{
|
|
marker: filepath.Join(root, "appstop-state.json"),
|
|
drive: filepath.Join(root, "drive"),
|
|
stopped: map[string]bool{},
|
|
}
|
|
r.undo = filepath.Join(r.drive, "felhom-data", "backups", "restore-undo", "immich", "20261024T120000Z")
|
|
mustWrite(t, filepath.Join(r.undo, "undo.json"), `{"stack":"immich","copied":["immich_pgdata"]}`)
|
|
sett, err := settings.Load(filepath.Join(root, "settings.json"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.sett = sett
|
|
g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0))
|
|
if err := g.Begin("unit-restore:immich", ReasonUnitRestore, []string{"immich"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := g.SetUndo("immich", r.undo); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return r // the power cut: no End
|
|
}
|
|
|
|
func (r *r1251Rig) unbind(t *testing.T) {
|
|
t.Helper()
|
|
if err := os.Rename(r.drive, r.drive+".unbound"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (r *r1251Rig) bind(t *testing.T) {
|
|
t.Helper()
|
|
if err := os.Rename(r.drive+".unbound", r.drive); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (r *r1251Rig) driveBound() bool { _, err := os.Stat(r.drive); return err == nil }
|
|
|
|
// r1251Starter is the gated starter: it refuses while the drive is absent or a hold is set (as driveStartGate does).
|
|
type r1251Starter struct{ r *r1251Rig }
|
|
|
|
func (s r1251Starter) StartStack(name string) error {
|
|
if _, held := s.r.sett.GetRestoreHold(name); held {
|
|
return ErrStartRefused
|
|
}
|
|
if !s.r.driveBound() {
|
|
return ErrStartRefused
|
|
}
|
|
s.r.starts = append(s.r.starts, name)
|
|
return nil
|
|
}
|
|
func (s r1251Starter) WantsStopped(name string) bool { return s.r.stopped[name] }
|
|
|
|
type r1251PutBacker struct{ r *r1251Rig }
|
|
|
|
func (p r1251PutBacker) PutBackInterruptedRestore(stack, dir string) error {
|
|
if p.r.putErr != nil {
|
|
return p.r.putErr
|
|
}
|
|
p.r.puts = append(p.r.puts, stack)
|
|
return os.RemoveAll(dir) // the real put-back removes the used copy
|
|
}
|
|
|
|
// guard is a fresh controller start over the rig's disk, wired as main.go wires it.
|
|
func (r *r1251Rig) guard() *AppStopGuard {
|
|
g := NewAppStopGuard(r.marker, log.New(io.Discard, "", 0))
|
|
g.SetStarter(r1251Starter{r})
|
|
g.SetPutBacker(r1251PutBacker{r})
|
|
g.SetHolds(r.sett)
|
|
g.SetUndoReachable(func(string, string) bool { return r.driveBound() })
|
|
return g
|
|
}
|
|
|
|
func (r *r1251Rig) markerActive() bool {
|
|
b, err := os.ReadFile(r.marker)
|
|
return err == nil && strings.Contains(string(b), `"active": true`)
|
|
}
|
|
|
|
func TestR1251_DriveCutRestoreWaitsForTheDriveThenPutsBackBeforeAnyStart(t *testing.T) {
|
|
r := newR1251Rig(t)
|
|
|
|
// boot 1: the drive is not bound yet
|
|
r.unbind(t)
|
|
g1 := r.guard()
|
|
rec := g1.Recover()
|
|
h, held := r.sett.GetRestoreHold("immich")
|
|
if rec == nil || len(rec.Waiting) != 1 || !held || h.Reason != settings.HoldReasonPutBackPending {
|
|
t.Fatalf("boot 1 must HOLD the app as waiting for its drive: rec=%+v hold=%+v held=%v", rec, h, held)
|
|
}
|
|
if h.SafetyDump != r.undo {
|
|
t.Fatalf("the hold must name the copy, got %q", h.SafetyDump)
|
|
}
|
|
if len(r.starts) != 0 || len(r.puts) != 0 || rec.Alarming() {
|
|
t.Fatalf("nothing may start or be poured while the drive is absent: starts=%v puts=%v alarming=%v", r.starts, r.puts, rec.Alarming())
|
|
}
|
|
if !r.markerActive() {
|
|
t.Fatal("the marker must be kept while the put-back waits")
|
|
}
|
|
// a resume while the drive is still absent does nothing and keeps the hold
|
|
if g1.ResumePutBack("immich") {
|
|
t.Fatal("ResumePutBack must say still-held while the copy cannot be reached")
|
|
}
|
|
if _, still := r.sett.GetRestoreHold("immich"); !still {
|
|
t.Fatal("the hold must stay while the drive is absent")
|
|
}
|
|
|
|
// the drive binds: the boot recreate (and the drive-return sweep) call ResumePutBack before they start the app
|
|
r.bind(t)
|
|
if !g1.ResumePutBack("immich") {
|
|
t.Fatal("with the drive bound the put-back must run and allow the start")
|
|
}
|
|
if strings.Join(r.puts, ",") != "immich" {
|
|
t.Fatalf("the copy must be poured back exactly once, puts=%v", r.puts)
|
|
}
|
|
if _, still := r.sett.GetRestoreHold("immich"); still {
|
|
t.Fatal("the waiting hold must be lifted after the put-back")
|
|
}
|
|
if r.markerActive() {
|
|
t.Fatal("the marker must let go of the app after its put-back")
|
|
}
|
|
if g1.ResumePutBack("immich") != true || len(r.puts) != 1 {
|
|
t.Fatalf("a second resume must pour nothing, puts=%v", r.puts)
|
|
}
|
|
|
|
// the household writes on the drive, then any later controller start: nothing is poured back
|
|
householdWrite := filepath.Join(r.drive, "felhom-data", "new-photo-after-the-put-back")
|
|
mustWrite(t, householdWrite, "x")
|
|
if rec2 := r.guard().Recover(); rec2 != nil {
|
|
t.Fatalf("a later start must find nothing to recover, got %+v", rec2)
|
|
}
|
|
if len(r.puts) != 1 {
|
|
t.Fatalf("a later start poured the copy again over the household's writes: puts=%v", r.puts)
|
|
}
|
|
if _, err := os.Stat(householdWrite); err != nil {
|
|
t.Fatal("the household's newest file must still be there")
|
|
}
|
|
}
|
|
|
|
// Every start path is refused while the put-back waits: they all read the ONE hold store. The night chain and the
|
|
// household's buttons read it through RestoreHoldFor (isHeld, the router's restoreHoldFor, the update guards); the
|
|
// boot sweep and Recover through driveStartGate's GetRestoreHold; the web paths (boot recreate, drive return, media
|
|
// restart) through appHeld. The web and main.go wiring is pinned in their own packages.
|
|
func TestR1251_TheWaitingHoldRefusesEveryStartPath(t *testing.T) {
|
|
r := newR1251Rig(t)
|
|
r.unbind(t)
|
|
r.guard().Recover()
|
|
m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
|
|
held, why := m.RestoreHoldForLang("immich", "hu")
|
|
if !held || !strings.Contains(why, "meghajt") || !strings.Contains(why, "immich") {
|
|
t.Fatalf("the household's sentence must name the app and the drive: held=%v %q", held, why)
|
|
}
|
|
if _, en := m.RestoreHoldForLang("immich", "en"); !strings.Contains(en, "drive is back") {
|
|
t.Fatalf("the English sentence: %q", en)
|
|
}
|
|
if !m.isHeld("immich") {
|
|
t.Fatal("the night chain's isHeld must see the waiting put-back")
|
|
}
|
|
// the gated starter (the boot sweep's and Recover's path) refuses too
|
|
if err := (r1251Starter{r}).StartStack("immich"); !errors.Is(err, ErrStartRefused) {
|
|
t.Fatalf("the gated start must be refused while held, got %v", err)
|
|
}
|
|
// a removal ends the install the copy belongs to: the waiting hold goes with it (a later install must not wait)
|
|
if cleared, _ := r.sett.ClearUpdateHold("immich"); !cleared {
|
|
t.Fatal("a removal must clear the waiting put-back hold")
|
|
}
|
|
}
|
|
|
|
// H2: a put-back that fails at start leaves the app HELD (the D20 hold, „stopped half-way"), never stopped-and-unheld;
|
|
// the copy stays and the marker lets go (the hold owns the app).
|
|
func TestR1251_AFailedPutBackHoldsTheApp(t *testing.T) {
|
|
r := newR1251Rig(t)
|
|
r.putErr = errors.New("tar: short read")
|
|
rec := r.guard().Recover()
|
|
h, held := r.sett.GetRestoreHold("immich")
|
|
if !held || h.Reason != settings.HoldReasonRestoreMixed {
|
|
t.Fatalf("a failed put-back must leave a restore_mixed hold, got %+v held=%v", h, held)
|
|
}
|
|
if rec == nil || len(rec.Held) != 1 || !rec.Alarming() || len(r.starts) != 0 {
|
|
t.Fatalf("held, alarming, not started: rec=%+v starts=%v", rec, r.starts)
|
|
}
|
|
if !strings.Contains(h.ReplayError, "short read") || h.SafetyDump != r.undo {
|
|
t.Fatalf("the hold must carry the cause and the copy: %+v", h)
|
|
}
|
|
if _, err := os.Stat(r.undo); err != nil {
|
|
t.Fatal("the copy must stay on disk for support")
|
|
}
|
|
if r.markerActive() {
|
|
t.Fatal("the hold owns the app now — the marker must let go, so no later start pours an old copy over it")
|
|
}
|
|
// the household's Start, the boot recreate and the update are all refused by that hold
|
|
m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
|
|
if held, why := m.RestoreHoldForLang("immich", "hu"); !held || !strings.Contains(why, "immich") {
|
|
t.Fatalf("the D20 sentence: held=%v %q", held, why)
|
|
}
|
|
}
|
|
|
|
// A failed resume (the drive came back, the put-back broke) holds the app the same way.
|
|
func TestR1251_AFailedResumeHoldsTheApp(t *testing.T) {
|
|
r := newR1251Rig(t)
|
|
r.unbind(t)
|
|
g := r.guard()
|
|
g.Recover()
|
|
r.bind(t)
|
|
r.putErr = errors.New("disk full")
|
|
if g.ResumePutBack("immich") {
|
|
t.Fatal("a failed put-back must not allow the start")
|
|
}
|
|
if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed {
|
|
t.Fatalf("a failed resume must hold for support, got %+v", h)
|
|
}
|
|
if r.markerActive() {
|
|
t.Fatal("the marker must let go after the hold took over")
|
|
}
|
|
}
|
|
|
|
// A copy that vanished while its drive is BOUND, during a waiting put-back, cannot be poured: held for support.
|
|
func TestR1251_ACopyGoneWhileItsDriveIsBoundHoldsTheApp(t *testing.T) {
|
|
r := newR1251Rig(t)
|
|
r.unbind(t)
|
|
g := r.guard()
|
|
g.Recover()
|
|
r.bind(t)
|
|
if err := os.RemoveAll(r.undo); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if g.ResumePutBack("immich") {
|
|
t.Fatal("a vanished copy must not allow the start")
|
|
}
|
|
if h, _ := r.sett.GetRestoreHold("immich"); h.Reason != settings.HoldReasonRestoreMixed {
|
|
t.Fatalf("held for support, got %+v", h)
|
|
}
|
|
}
|
|
|
|
// H7: Stop pressed during the restore — the copy is still poured back after the cut; only the start is skipped.
|
|
func TestR1251_StopPressedDuringTheRestoreStillPutsBack(t *testing.T) {
|
|
r := newR1251Rig(t)
|
|
r.stopped["immich"] = true
|
|
rec := r.guard().Recover()
|
|
if strings.Join(r.puts, ",") != "immich" {
|
|
t.Fatalf("the put-back must run even though the household pressed Stop, puts=%v rec=%+v", r.puts, rec)
|
|
}
|
|
if len(r.starts) != 0 {
|
|
t.Fatalf("the household stopped it — it must not start, starts=%v", r.starts)
|
|
}
|
|
if r.markerActive() {
|
|
t.Fatal("nothing is owed any more — the marker must clear")
|
|
}
|
|
}
|
|
|
|
// The drive returns later (not at boot): the sweep puts back, then starts through the gated starter — unless the
|
|
// household stopped the app.
|
|
func TestR1251_ADriveThatReturnsLaterIsPutBackThenStarted(t *testing.T) {
|
|
for _, stopped := range []bool{false, true} {
|
|
r := newR1251Rig(t)
|
|
r.unbind(t)
|
|
g := r.guard()
|
|
g.Recover()
|
|
if got := g.ResumeWaitingPutBacks(); len(got) != 0 || len(r.puts) != 0 {
|
|
t.Fatalf("drive still absent: nothing may happen, started=%v puts=%v", got, r.puts)
|
|
}
|
|
r.bind(t)
|
|
r.stopped["immich"] = stopped
|
|
got := g.ResumeWaitingPutBacks()
|
|
if len(r.puts) != 1 {
|
|
t.Fatalf("stopped=%v: the put-back must run once, puts=%v", stopped, r.puts)
|
|
}
|
|
if want := !stopped; (len(got) == 1) != want || (len(r.starts) == 1) != want {
|
|
t.Fatalf("stopped=%v: started=%v starts=%v", stopped, got, r.starts)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A later restore that completes drops the waiting put-back: the household chose that restore's data.
|
|
func TestR1251_ACompletedRestoreDropsTheWaitingPutBack(t *testing.T) {
|
|
r := newR1251Rig(t)
|
|
r.unbind(t)
|
|
r.guard().Recover()
|
|
m := &Manager{settings: r.sett, logger: log.New(io.Discard, "", 0)}
|
|
m.clearUpdateHoldAfterRestore("immich")
|
|
if _, held := r.sett.GetRestoreHold("immich"); held {
|
|
t.Fatal("a completed restore must drop the waiting put-back hold")
|
|
}
|
|
}
|