package backup import ( "errors" "os" "path/filepath" "testing" ) // R-403 Group C — the rehydrate: after a Tier-2 unit restore, the app's own drive gets its package // back, INSIDE the call. // // The cause half. On 2026-08-31 the hollow primary manifest was written TWO SECONDS after a restore // of exactly this shape, by the 5-minute `backup-cache` job. Anything that runs after the call // returns races that job; only doing it before returning cannot lose. // C1 — a hollow primary is refilled from the mirror that was just restored from. func TestR403_HollowPrimaryIsRefilledFromTheMirror(t *testing.T) { f := r102Tier2Fixture(t, []string{"vol_a.tar", "vol_b.tar"}, pgDump(1)) primaryUnit := RecoveryUnitPath(f.liveDrive, "app") // The R-102 scenario: the app's own package is gone. if err := os.RemoveAll(primaryUnit); err != nil { t.Fatal(err) } if unitCarriesData(primaryUnit) { t.Fatal("fixture wrong: the primary still carries data") } var copied [][2]string f.m.unitRehydrate = func(src, dst string) error { copied = append(copied, [2]string{src, dst}) return copyTree(src, dst) } if _, err := f.m.RestoreTier2Unit("app"); err != nil { t.Fatalf("restore: %v", err) } if len(copied) != 1 { t.Fatalf("rehydrate ran %d time(s), want 1: %v", len(copied), copied) } mirrorUnit := tier2UnitDir(f.destBase) if copied[0][0] != mirrorUnit || copied[0][1] != primaryUnit { t.Errorf("rehydrate copied %v → %v, want %v → %v", copied[0][0], copied[0][1], mirrorUnit, primaryUnit) } // THE CONSEQUENCE: the primary is a real package again, so the next capture has nothing hollow to // describe and the next Tier-2 run has nothing poorer to mirror. if !unitCarriesData(primaryUnit) { t.Error("the primary unit is still hollow after the restore — R-403's cause is not closed") } for _, rel := range []string{"volume-dumps/vol_a.tar", "volume-dumps/vol_b.tar", "db-dumps/app-postgres.sql"} { if _, err := os.Stat(filepath.Join(primaryUnit, rel)); err != nil { t.Errorf("%s did not come back to the app's own drive: %v", rel, err) } } } // C2 — TestR403_CompletePrimaryIsLeftByteIdentical. Scenario F, and it is R-403 pointed the other // way: a primary that already carries data may be NEWER than the mirror, and overwriting it with an // older copy is the same defect this task exists to remove. // // Red-proof (recorded in REPORT.md): drop the `unitCarriesData(primaryUnit)` condition and this // fails on the fingerprint. func TestR403_CompletePrimaryIsLeftByteIdentical(t *testing.T) { f := r102Tier2Fixture(t, []string{"vol_a.tar"}, pgDump(1)) primaryUnit := RecoveryUnitPath(f.liveDrive, "app") if !unitCarriesData(primaryUnit) { t.Fatal("fixture wrong: the primary must start complete") } // Make the primary distinguishable from the mirror, so an overwrite would show. mustWrite(t, filepath.Join(primaryUnit, "volume-dumps", "only-on-the-primary.tar"), "NEWER") before := fingerprintTree(t, primaryUnit) var ran int f.m.unitRehydrate = func(src, dst string) error { ran++; return copyTree(src, dst) } if _, err := f.m.RestoreTier2Unit("app"); err != nil { t.Fatalf("restore: %v", err) } if ran != 0 { t.Errorf("the rehydrate ran %d time(s) over a COMPLETE primary — it would overwrite newer material", ran) } if after := fingerprintTree(t, primaryUnit); after != before { t.Error("the complete primary unit was modified; it must be byte-identical") } if _, err := os.Stat(filepath.Join(primaryUnit, "volume-dumps", "only-on-the-primary.tar")); err != nil { t.Error("the primary-only tar was destroyed by the rehydrate") } } // C3 — a FAILED restore writes no package. A package written from a run that did not succeed would // look like a backup and describe data that never landed. func TestR403_FailedRestoreDoesNotWriteAPackage(t *testing.T) { f := r102Tier2Fixture(t, []string{"vol_a.tar"}, pgDump(1)) primaryUnit := RecoveryUnitPath(f.liveDrive, "app") if err := os.RemoveAll(primaryUnit); err != nil { t.Fatal(err) } // Make the restore itself fail, after it has started. f.fake.startSvcErr = errors.New("injected: the database service would not start") f.m.volumeReplayFrom = func(string, string) (int, error) { return 0, errors.New("injected: the volume replay failed") } var ran int f.m.unitRehydrate = func(src, dst string) error { ran++; return copyTree(src, dst) } if _, err := f.m.RestoreTier2Unit("app"); err == nil { t.Fatal("the fixture did not make the restore fail") } if ran != 0 { t.Errorf("the rehydrate ran %d time(s) after a FAILED restore", ran) } if unitCarriesData(primaryUnit) { t.Error("a package was written on the app's drive by a restore that failed") } } // C4 — TestR403_RehydrateHappensBeforeTheCallReturns. Asserted as ORDERING, never as a timer. // // The whole requirement is that the 5-minute capture job cannot observe the hollow state. A test // that slept and then looked would pass on a racing implementation on a fast machine. func TestR403_RehydrateHappensBeforeTheCallReturns(t *testing.T) { f := r102Tier2Fixture(t, []string{"vol_a.tar"}, pgDump(1)) primaryUnit := RecoveryUnitPath(f.liveDrive, "app") if err := os.RemoveAll(primaryUnit); err != nil { t.Fatal(err) } var rehydrated bool f.m.unitRehydrate = func(src, dst string) error { rehydrated = true; return copyTree(src, dst) } // The observation is taken on the line AFTER the call returns, with nothing waited for. If the // implementation deferred the work to a goroutine or a scheduled job, this reads false. _, err := f.m.RestoreTier2Unit("app") if err != nil { t.Fatalf("restore: %v", err) } if !rehydrated { t.Fatal("the rehydrate had NOT run when the call returned — a follow-up job races the capture (R-403)") } if !unitCarriesData(primaryUnit) { t.Error("the primary was still hollow at the instant the call returned") } } // A rehydrate failure must not turn a successful restore into a reported failure — the app is back. func TestR403_RehydrateFailureDoesNotFailTheRestore(t *testing.T) { f := r102Tier2Fixture(t, []string{"vol_a.tar"}, pgDump(1)) if err := os.RemoveAll(RecoveryUnitPath(f.liveDrive, "app")); err != nil { t.Fatal(err) } f.m.unitRehydrate = func(string, string) error { return errors.New("injected: disk full") } res, err := f.m.RestoreTier2Unit("app") if err != nil { t.Fatalf("a failed rehydrate must not fail the restore: %v", err) } if res.VolumesReplayed != 1 { t.Errorf("the restore's own result was disturbed: %+v", res) } }