fd50a73e65
Both are the system reporting healthy while the customer is not, and both live in the same status-derivation code. Neither is fixed by making the system quieter. C9-F1 (HIGH) — Tier-2 writes recovery-unit/ on EVERY run and RestoreTier2Files has never read it (tier2_restore.go:101-104 reads hdd/ + userdata/ only). Phase 0 enumerated all 53 catalog templates against both demo boxes: 43 apps have NO readable subtree, so the button stopped the app, restored 0 files, restarted it and said "Nincs hiányzó fájl — minden fájl megvan a helyén." — at the moment the customer pressed it because files were missing, with 156 MB of BookStack's data unread in the same copy. 9 apps have file legs but never their DB or volumes, so the same sentence was also a clean bill of health over data never opened (immich: 1.3 GB Postgres unit). Honesty half shipped: a pre-flight coverage check refuses UP FRONT without stopping the app and NAMES the action that works; a run that proceeds claims only what it EXAMINED and discloses that the database and volumes are not covered. Completeness is filed as C9-F1b — routing to the Tier-1 unit restore puts a destructive operation behind a non-destructive button, so its confirm copy has to carry that difference. C9-F4 filed: nothing reads the Tier-2 recovery-unit/ mirror, so the second local copy that exists for drive loss is unreachable by any customer action. C9-F2 (HIGH) — a crash loop was counted as working. StateRestarting is deliberately NOT added to IsDownState (that alarms on every deploy fleet-wide, the over-correction F-A1 nearly cost us); a sustained run becomes down after crashLoopAfter = 5m, set above the 120s deploy timeout, Mealie's 60s start_period and R-97b's 180s grace. The dashboard counter uses the same predicate, so it no longer contradicts the alarm on the same screen. README's claim that faults "still surface as restarting" was a wish with no test — corrected in place; it is the seventh such instance. Six red-proofs observed, including the one that matters most: adding StateRestarting to IsDownState fails the brief-restart test with "every deploy and update would page the operator". go test ./... rc=0, 27 packages, run and read separately from this commit.
144 lines
5.6 KiB
Go
144 lines
5.6 KiB
Go
package backup
|
|
|
|
import (
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
// C9-F1 — the Tier-2 restore reads `hdd/` and `userdata/` only. `recovery-unit/` — the app's DB dumps
|
|
// and named-volume tarballs — is written by EVERY Tier-2 run and read by NOTHING on this path.
|
|
//
|
|
// For 43 of the 53 catalog apps (BookStack, Docmost, Vaultwarden, Gitea, …) that is the app's ENTIRE
|
|
// dataset, so the restore was a guaranteed no-op that still took an outage and reported
|
|
// „Nincs hiányzó fájl — minden fájl megvan a helyén."
|
|
//
|
|
// These tests pin the asymmetry itself, so a future change that alters what the restore reads must
|
|
// either keep the coverage answer honest or fail here.
|
|
|
|
// unitOnlyCopy rewrites the fixture's copy into the BookStack shape: a recovery unit and nothing the
|
|
// restore can read. Mirrors the live demo-felhom layout observed in Campaign 9
|
|
// (`legs=[NONE] unit=156M`).
|
|
func unitOnlyCopy(t *testing.T, destDrive string) string {
|
|
t.Helper()
|
|
destBase := filepath.Join(destDrive, "backups", "secondary", "app")
|
|
if err := os.RemoveAll(filepath.Join(destBase, "hdd")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
mustWrite(t, filepath.Join(destBase, "recovery-unit", "volume-dumps", "app_db_data.tar"), "TARBYTES")
|
|
mustWrite(t, filepath.Join(destBase, "recovery-unit", "db-dumps", "app.sql"), "SQLDUMP")
|
|
return destBase
|
|
}
|
|
|
|
// SCENARIO D — a restore that cannot cover an app refuses BEFORE the outage.
|
|
//
|
|
// RED-PROOF (observed): remove the `!cov.CanRestore()` guard from RestoreTier2Files →
|
|
//
|
|
// tier2_coverage_test.go:63: RestoreTier2Files returned <nil> — a copy with nothing restorable was treated as success
|
|
// tier2_coverage_test.go:69: THE APP WAS STOPPED for a restore that could not restore anything: [app]
|
|
func TestRestoreTier2Files_NoRestorableSubtree_RefusesBeforeStopping(t *testing.T) {
|
|
m, fake, _, destDrive := newT2RManager(t)
|
|
unitOnlyCopy(t, destDrive)
|
|
m.restoreFilesCopier = func(string, string) (int, error) {
|
|
t.Fatal("the copier ran for an app with no restorable subtree")
|
|
return 0, nil
|
|
}
|
|
|
|
n, err := m.RestoreTier2Files("app")
|
|
|
|
if !errors.Is(err, ErrTier2NoRestorableData) {
|
|
t.Errorf("RestoreTier2Files returned %v — a copy with nothing restorable was treated as success", err)
|
|
}
|
|
if n != 0 {
|
|
t.Errorf("filesRestored = %d, want 0", n)
|
|
}
|
|
// The whole point: no outage was taken.
|
|
if len(fake.stopped) != 0 {
|
|
t.Errorf("THE APP WAS STOPPED for a restore that could not restore anything: %v", fake.stopped)
|
|
}
|
|
if len(fake.started) != 0 {
|
|
t.Errorf("the app was restarted, so it must have been stopped: %v", fake.started)
|
|
}
|
|
}
|
|
|
|
// The coverage query itself — what the handler pre-flights on, so it can refuse without starting an
|
|
// operation at all.
|
|
func TestTier2RestoreCoverage_ReportsTheAsymmetry(t *testing.T) {
|
|
m, _, _, destDrive := newT2RManager(t)
|
|
|
|
// Class A (paperless/immich shape): an hdd leg the restore reads, plus a unit it does not.
|
|
mustWrite(t, filepath.Join(destDrive, "backups", "secondary", "app", "recovery-unit", "manifest.json"), "{}")
|
|
cov, err := m.Tier2RestoreCoverage("app")
|
|
if err != nil {
|
|
t.Fatalf("coverage: %v", err)
|
|
}
|
|
if !cov.CanRestore() {
|
|
t.Error("an app WITH an hdd leg reported as not restorable — this would refuse the one path that works")
|
|
}
|
|
if !cov.HasUnit {
|
|
t.Error("the recovery unit was not detected — the disclosure would be omitted")
|
|
}
|
|
|
|
// Class B (bookstack/docmost shape): unit only.
|
|
unitOnlyCopy(t, destDrive)
|
|
cov, err = m.Tier2RestoreCoverage("app")
|
|
if err != nil {
|
|
t.Fatalf("coverage: %v", err)
|
|
}
|
|
if cov.CanRestore() {
|
|
t.Error("a unit-only copy reported as restorable — this is exactly C9-F1")
|
|
}
|
|
if !cov.HasUnit {
|
|
t.Error("the unit that holds the app's whole dataset was not detected")
|
|
}
|
|
if len(cov.Legs) != 0 {
|
|
t.Errorf("legs = %v, want none", cov.Legs)
|
|
}
|
|
}
|
|
|
|
// SCENARIO E — an app the restore CAN cover is completely unchanged. This is the regression guard on
|
|
// Campaign 9's headline result (A1/A3, paperless-ngx): byte-identical restore, stop→copy→start, and
|
|
// the additive-only promises intact. Breaking this to fix BookStack would be a straight regression on
|
|
// the only restore path proven to work on live hardware.
|
|
//
|
|
// RED-PROOF (observed): make the coverage guard unconditional (`if true`) →
|
|
//
|
|
// tier2_coverage_test.go:118: a COVERED app was refused: ennek az alkalmazásnak az adatai nem ebből a másolatból állíthatók vissza
|
|
func TestRestoreTier2Files_CoveredAppIsUnchanged(t *testing.T) {
|
|
m, fake, liveDrive, destDrive := newT2RManager(t)
|
|
// A unit is present too — a covered app has one as well; it must not change the outcome.
|
|
mustWrite(t, filepath.Join(destDrive, "backups", "secondary", "app", "recovery-unit", "manifest.json"), "{}")
|
|
|
|
var copied [][2]string
|
|
m.restoreFilesCopier = func(src, dst string) (int, error) {
|
|
copied = append(copied, [2]string{src, dst})
|
|
fake.order = append(fake.order, "copy")
|
|
return 3, nil
|
|
}
|
|
|
|
n, err := m.RestoreTier2Files("app")
|
|
if err != nil {
|
|
t.Fatalf("a COVERED app was refused: %v", err)
|
|
}
|
|
if n != 3 {
|
|
t.Errorf("filesRestored = %d, want 3", n)
|
|
}
|
|
if len(fake.stopped) != 1 || len(fake.started) != 1 {
|
|
t.Errorf("stop/start did not happen exactly once: %v / %v", fake.stopped, fake.started)
|
|
}
|
|
if got := fake.order; len(got) < 3 || got[0] != "stop" || got[len(got)-1] != "start" {
|
|
t.Errorf("order = %v, want stop → copy → start", got)
|
|
}
|
|
if len(copied) == 0 {
|
|
t.Fatal("nothing was copied for a covered app")
|
|
}
|
|
wantSrc := filepath.Join(destDrive, "backups", "secondary", "app", "hdd")
|
|
if copied[0][0] != wantSrc {
|
|
t.Errorf("src = %q, want %q", copied[0][0], wantSrc)
|
|
}
|
|
if copied[0][1] != liveDrive {
|
|
t.Errorf("dst = %q, want the live namespace root %q", copied[0][1], liveDrive)
|
|
}
|
|
}
|