223 lines
10 KiB
Go
223 lines
10 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
|
)
|
|
|
|
// R-893 slice 1 (`09` §3 decision 192, option C of audits/day-2026-10-08/design-R-893.md).
|
|
//
|
|
// An off-site restore of one app writes the snapshot's definition (when the version differs), copies the
|
|
// snapshot's files and replaces the named volumes BEFORE the database replay. When the replay then fails,
|
|
// the rollback puts back only the database rows. Until this slice the app was started on that mix —
|
|
// an older definition, the snapshot's files and volumes, the newer database — and the household was told
|
|
// „your data is back as it was before the restore". That is true only of the database rows.
|
|
//
|
|
// Slice 1 (option C) HELD the app. Since option A (operator D20 = A, 2026-10-20) the box puts the app back
|
|
// EXACTLY as it was and starts it: the live definition is written back and the copy of the live volumes is
|
|
// poured back (r893_putback_test.go pins the volume half). The HOLD remains for the case nothing can undo:
|
|
// volumes replaced with no copy to put back. Pinned here:
|
|
// (a) the definition write is called a second time, with the live definition;
|
|
// (b) put back → the app IS started, no hold, the operator told once (put-back event);
|
|
// (c) no copy → a hold is left (kind restore_mixed), the app is NOT started, the operator notified;
|
|
// (d) the held sentences claim neither that the data is back nor that the app runs (hu and en).
|
|
// The no-version-change, no-volume case keeps today's behaviour: TestR379_ScenarioA.
|
|
|
|
// r893Provider records EVERY definition write (vtReconProvider keeps only the last).
|
|
type r893Provider struct {
|
|
*vtReconProvider
|
|
defs [][]string
|
|
onStop func()
|
|
}
|
|
|
|
func (p *r893Provider) StopStack(name string) error {
|
|
if p.onStop != nil {
|
|
p.onStop()
|
|
}
|
|
return p.vtReconProvider.StopStack(name)
|
|
}
|
|
|
|
func (p *r893Provider) RecreateStackDefinitionFromUnit(name, composeDir string, env map[string]string) error {
|
|
p.defs = append(p.defs, ParseComposeImages(filepath.Join(composeDir, "docker-compose.yml")))
|
|
return p.vtReconProvider.RecreateStackDefinitionFromUnit(name, composeDir, env)
|
|
}
|
|
|
|
func r893Fixture(t *testing.T, versionChange bool, volumes int) (*Manager, *r893Provider, *int, *int) {
|
|
m, vp, rolled, notified, _ := r893FixtureA(t, versionChange, volumes)
|
|
return m, vp, rolled, notified
|
|
}
|
|
|
|
// r893FixtureA is r893Fixture plus the put-back notification counter (option A).
|
|
func r893FixtureA(t *testing.T, versionChange bool, volumes int) (*Manager, *r893Provider, *int, *int, *int) {
|
|
t.Helper()
|
|
m, prov, _ := reconFixture(t, "20260926T021500Z", "2026-09-26T02:15:01Z", pgDump(1))
|
|
m.importDBDump = func(context.Context, DiscoveredDB, string) error { return errors.New("replay blew up") }
|
|
rolled := 0
|
|
m.SetRollbackImportFn(func(context.Context, DiscoveredDB, string) error { rolled++; return nil })
|
|
notified := 0
|
|
m.SetRestoreHoldNotify(func(_ string, replayErr, rollbackErr error) {
|
|
notified++
|
|
if replayErr == nil || rollbackErr != nil {
|
|
t.Errorf("notify got replay=%v rollback=%v — want the replay error and NO rollback error", replayErr, rollbackErr)
|
|
}
|
|
})
|
|
putBack := 0
|
|
m.SetRestorePutBackNotify(func(_ string, replayErr error) {
|
|
putBack++
|
|
if replayErr == nil {
|
|
t.Error("the put-back notification must carry the error that failed the restore")
|
|
}
|
|
})
|
|
m.volumeReplayFrom = func(string, string) (int, error) { return volumes, nil }
|
|
live := "16"
|
|
if versionChange {
|
|
live = "18"
|
|
}
|
|
// The LIVE definition, on disk where the app runs.
|
|
if err := os.WriteFile(prov.composePath, []byte(vtCompose(live)), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
vp := &r893Provider{vtReconProvider: &vtReconProvider{recordingProvider: prov, livePins: ParseComposeImages(prov.composePath)}}
|
|
m.SetStackProvider(vp)
|
|
vtSnapshotUnit(t, m, "16", true)
|
|
return m, vp, &rolled, ¬ified, &putBack
|
|
}
|
|
|
|
// assertR893PutBack is option A's outcome: the app runs again at its live definition, nothing is held.
|
|
func assertR893PutBack(t *testing.T, m *Manager, vp *r893Provider, err error, notified, putBack int) {
|
|
t.Helper()
|
|
if err == nil {
|
|
t.Fatal("a failed restore must still be surfaced as a failure")
|
|
}
|
|
if !vp.fullStarted {
|
|
t.Fatalf("option A: the app must be started again after the put-back — calls %v", vp.calls)
|
|
}
|
|
if held, _ := m.RestoreHoldFor("immich"); held {
|
|
t.Fatal("option A: a put-back app must NOT be held")
|
|
}
|
|
if notified != 0 || putBack != 1 {
|
|
t.Errorf("hold notices %d (want 0), put-back notices %d (want 1)", notified, putBack)
|
|
}
|
|
if len(vp.defs) != 2 || !strings.Contains(strings.Join(vp.defs[1], " "), "postgres:18-alpine") {
|
|
t.Fatalf("definition writes %v — want the snapshot's, then the LIVE one written back", vp.defs)
|
|
}
|
|
low := strings.ToLower(err.Error())
|
|
if !strings.Contains(low, "sikertelen") || !strings.Contains(low, "visszaker") || strings.Contains(low, "kapcsolatot") {
|
|
t.Errorf("the sentence must say the restore failed and the data is back, with no support call: %q", err.Error())
|
|
}
|
|
}
|
|
|
|
func assertR893Held(t *testing.T, m *Manager, vp *r893Provider, err error, rolled, notified int) {
|
|
t.Helper()
|
|
if err == nil {
|
|
t.Fatal("a failed replay must be surfaced as a failure")
|
|
}
|
|
if rolled != 1 {
|
|
t.Fatalf("the database rollback must still run exactly once, ran %d", rolled)
|
|
}
|
|
// (b)
|
|
if vp.fullStarted {
|
|
t.Fatalf("the app was STARTED on a mixed state — calls %v", vp.calls)
|
|
}
|
|
if last := vp.calls[len(vp.calls)-1]; last != "stop" {
|
|
t.Errorf("the database service must be stopped again before the hold; last call %q (%v)", last, vp.calls)
|
|
}
|
|
// (c)
|
|
held, sentence := m.RestoreHoldFor("immich")
|
|
if !held {
|
|
t.Fatal("no hold was left — every start path would start the app on the mixed state")
|
|
}
|
|
if k := m.HoldKind("immich"); k != settings.HoldReasonRestoreMixed {
|
|
t.Errorf("hold kind %q, want %q", k, settings.HoldReasonRestoreMixed)
|
|
}
|
|
if notified != 1 {
|
|
t.Errorf("the operator must be notified once, got %d", notified)
|
|
}
|
|
// (d) — ASCII fragments for the Hungarian (the accented word is matched by its ASCII stem too).
|
|
for _, s := range []string{err.Error(), sentence} {
|
|
low := strings.ToLower(s)
|
|
if strings.Contains(low, "visszaker") || strings.Contains(low, "fut tov") {
|
|
t.Errorf("the sentence still claims the data is back / the app runs: %q", s)
|
|
}
|
|
if !strings.Contains(low, "kapcsolatot") {
|
|
t.Errorf("the sentence must send the household to support: %q", s)
|
|
}
|
|
}
|
|
_, en := m.RestoreHoldForLang("immich", "en")
|
|
if strings.Contains(strings.ToLower(en), "back as it was") || !strings.Contains(en, "help") {
|
|
t.Errorf("English hold sentence: %q", en)
|
|
}
|
|
if enErr := util.Text("en", "err.backup.db_restore_failed_held_mixed", "immich"); strings.Contains(enErr, "back as it was") || !strings.Contains(enErr, "STOPPED") {
|
|
t.Errorf("English error sentence: %q", enErr)
|
|
}
|
|
}
|
|
|
|
// RED-PROOF (option A): restore the slice-1 branch (holdAppAfterMixedRestore + return held_mixed) after the
|
|
// rollback → the app is not started and a hold is left → FAILS.
|
|
func TestR893_AVersionChangeThenAFailedReplayStartsTheAppAtItsLiveDefinition(t *testing.T) {
|
|
m, vp, rolled, notified, putBack := r893FixtureA(t, true, 0)
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if *rolled != 1 {
|
|
t.Fatalf("the database rollback must still run exactly once, ran %d", *rolled)
|
|
}
|
|
assertR893PutBack(t, m, vp, err, *notified, *putBack)
|
|
}
|
|
|
|
// A volume replaced with NO copy to pour back (no snapshot volume tar was measured) cannot be undone: the hold
|
|
// stays (option C). In production the copy exists whenever the snapshot holds a volume tar.
|
|
func TestR893_AReplacedVolumeWithNoCopyThenAFailedReplayHoldsTheApp(t *testing.T) {
|
|
m, vp, rolled, notified := r893Fixture(t, false, 1)
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
assertR893Held(t, m, vp, err, *rolled, *notified)
|
|
if len(vp.defs) != 0 {
|
|
t.Fatalf("no version changed, yet a definition was written: %v", vp.defs)
|
|
}
|
|
}
|
|
|
|
// Security review 2026-10-08 (G1): EVERY failure after the snapshot's definition is written is undone, not only a
|
|
// failed replay. Here the DB-only start fails after a version change: before G1 the app was started at the
|
|
// snapshot's (older) definition on the live database. Since option A the live definition is written back FIRST and
|
|
// the app then starts at it (calls: … recreate, start — the live definition before the start).
|
|
// RED-PROOF: restore `restartStack()` as the only action in the StartStackServices failure branch → only one
|
|
// definition write (the snapshot's) → FAILS.
|
|
func TestR893_AVersionChangeThenAFailedDBStartStartsTheAppAtItsLiveDefinition(t *testing.T) {
|
|
m, vp, rolled, notified, putBack := r893FixtureA(t, true, 0)
|
|
vp.startSvcErr = context.DeadlineExceeded
|
|
_, err := m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if *rolled != 0 {
|
|
t.Errorf("rollback ran %d (want 0: nothing was replayed)", *rolled)
|
|
}
|
|
if err == nil {
|
|
t.Fatal("a failed DB-only start must be surfaced")
|
|
}
|
|
n := len(vp.calls)
|
|
if n < 2 || vp.calls[n-2] != "recreate" || vp.calls[n-1] != "start" {
|
|
t.Fatalf("the live definition must be written back BEFORE the start — calls %v", vp.calls)
|
|
}
|
|
assertR893PutBack(t, m, vp, err, *notified, *putBack)
|
|
}
|
|
|
|
// Security review 2026-10-08 (G3): the hold is persisted BEFORE the app is stopped, so a controller that dies in
|
|
// between cannot restart the app from its app-stop marker with nothing refusing it. The provider's StopStack asserts
|
|
// the hold is already on disk at the moment it is called.
|
|
// RED-PROOF: move SetRestoreHold after StopStack in holdAppAfterMixedRestore → FAILS.
|
|
func TestR893_HoldIsPersistedBeforeTheStop(t *testing.T) {
|
|
m, vp, _, _ := r893Fixture(t, true, 1) // a replaced volume with no copy: the case that still holds
|
|
seenAtStop := []bool{}
|
|
vp.onStop = func() {
|
|
held, _ := m.RestoreHoldFor("immich")
|
|
seenAtStop = append(seenAtStop, held)
|
|
}
|
|
_, _ = m.ReconstituteFromOffsite(context.Background(), "immich", false)
|
|
if len(seenAtStop) == 0 || !seenAtStop[len(seenAtStop)-1] {
|
|
t.Fatalf("the last stop ran before the hold was persisted (held at each stop: %v)", seenAtStop)
|
|
}
|
|
}
|