package backup import ( "context" "errors" "io" "log" "os" "strings" "testing" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // R-1340: a restore_mixed hold records WHAT put it in place — which restore, which branch, which put-back step failed, // whether the restore's failure was a database replay, and where the copy is kept — so the operator's mail // (cmd/controller heldStoppedOperatorText, pinned by TestR1340_* there) tells the real story. Measured 2026-10-29 on // Tester 1: Radicale (no database), its volume could not be removed, the put-back's stop failed, and the mail said its // DATABASE restore failed and was rolled back. // // RED-PROOF (2026-10-29): with the R-1340 fields dropped from holdAppAfterMixedRestore's hold literal and from // holdPutBackFailed's (the pre-R-1340 record, which carried no cause), all five tests below fail on „mixed cause """ / // „kind """. func holdOf(t *testing.T, m *Manager, stack string) settings.RestoreHold { t.Helper() if m.settings == nil { t.Fatal("fixture: no settings wired") } h, ok := m.settings.GetRestoreHold(stack) if !ok { t.Fatalf("%s is not held", stack) } return h } // The Radicale shape: a VOLUME restore fails (no database involved), then the put-back's stop fails. func TestR1340_AVolumeFailureWhosePutBackStopFailsRecordsTheStepAndNoDatabase(t *testing.T) { m, vp, rig, _, notified, _ := r893PutBackFixture(t) rig.snapReplayErr = errors.New("volume rm -f immich_pgdata: volume is in use") p := &r1256StopFailProvider{r893Provider: vp, failAt: 2} // 1 = the restore's stop, 2 = the put-back's m.SetStackProvider(p) if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil { t.Fatal("a failed restore must be surfaced") } h := holdOf(t, m, "immich") if h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCausePutBack { t.Fatalf("reason %q, mixed cause %q — want restore_mixed / put_back", h.Reason, h.MixedCause) } if !strings.Contains(h.FailedStep, "stopping the app before the put-back failed") || !strings.Contains(h.FailedStep, "compose down: timeout") { t.Errorf("failed step %q — want the stop and its error", h.FailedStep) } if h.DBReplay { t.Error("a volume failure must not be recorded as a database replay") } if h.RestoreKind != settings.RestoreKindOffsite { t.Errorf("restore kind %q, want offsite", h.RestoreKind) } if h.SafetyDump == "" { t.Error("the kept copy's folder must be recorded") } else if _, err := os.Stat(h.SafetyDump); err != nil { t.Errorf("the recorded copy %s is not on disk: %v", h.SafetyDump, err) } if *notified != 1 { t.Errorf("hold notices %d, want 1", *notified) } } // The database replay fails (option C's shape), then the put-back's stop fails: DBReplay is recorded. func TestR1340_ADatabaseReplayFailureRecordsTheDatabase(t *testing.T) { m, vp, _, _, _, _ := r893PutBackFixture(t) m.SetStackProvider(&r1256StopFailProvider{r893Provider: vp, failAt: 2}) if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil { t.Fatal("a failed restore must be surfaced") } h := holdOf(t, m, "immich") if h.MixedCause != settings.MixedCausePutBack || !h.DBReplay { t.Fatalf("mixed cause %q, db replay %v — want put_back and true", h.MixedCause, h.DBReplay) } } // R-1256 H4: the backup's definition cannot be written and the live one cannot be written back — no data moved. func TestR1340_ADefinitionWriteFailureRecordsItsCause(t *testing.T) { m, _, _ := r1256DefFailFixture(t, 2) if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err == nil { t.Fatal("a failed definition write must be surfaced") } h := holdOf(t, m, "immich") if h.MixedCause != settings.MixedCauseDefinitionWrite || h.DBReplay { t.Fatalf("mixed cause %q, db replay %v — want definition_write and false", h.MixedCause, h.DBReplay) } if !strings.Contains(h.FailedStep, "writing the live app definition back failed") { t.Errorf("failed step %q", h.FailedStep) } } // A LOCAL restore (from the app's own backup) records itself as local, never off-site. func TestR1340_ALocalRestoreRecordsItsKind(t *testing.T) { m, _, rig, _ := r1177Fixture(t) if m.settings == nil { sett, err := settings.Load(t.TempDir()+"/settings.json", log.New(io.Discard, "", 0)) if err != nil { t.Fatal(err) } m.settings = sett } rig.unitErr = errors.New("tar: short read") rig.putBackErr = errors.New("tar xf failed") if _, err := m.RestoreFromRecoveryUnit("app"); err == nil { t.Fatal("a failed put-back must be surfaced") } h := holdOf(t, m, "app") if h.RestoreKind != settings.RestoreKindLocal || h.MixedCause != settings.MixedCausePutBack || h.DBReplay { t.Fatalf("kind %q, cause %q, db %v — want local / put_back / false", h.RestoreKind, h.MixedCause, h.DBReplay) } if !strings.Contains(h.FailedStep, "putting the live volumes back failed") || !strings.Contains(h.FailedStep, "tar xf failed") { t.Errorf("failed step %q", h.FailedStep) } } // A restore cut by a stop whose put-back fails at the next start (R-1251 H2) records its own cause. func TestR1340_AnInterruptedPutBackRecordsItsCause(t *testing.T) { r := newR1251Rig(t) r.putErr = errors.New("compose down: timeout") r.guard().Recover() h, ok := r.sett.GetRestoreHold("immich") if !ok || h.Reason != settings.HoldReasonRestoreMixed || h.MixedCause != settings.MixedCauseInterruptedPutBack { t.Fatalf("hold %+v (held %v) — want restore_mixed / interrupted_put_back", h, ok) } if !strings.Contains(h.FailedStep, "compose down: timeout") { t.Errorf("failed step %q", h.FailedStep) } }