Files
felhom-controller/controller/internal/backup/tier2_coverage_test.go
T
admin fd50a73e65 C9-F1 + C9-F2: a restore that restored nothing, and a crash loop nobody saw (v0.183.0)
Both are the system reporting healthy while the customer is not, and both live in the same
status-derivation code. Neither is fixed by making the system quieter.

C9-F1 (HIGH) — Tier-2 writes recovery-unit/ on EVERY run and RestoreTier2Files has never read
it (tier2_restore.go:101-104 reads hdd/ + userdata/ only). Phase 0 enumerated all 53 catalog
templates against both demo boxes: 43 apps have NO readable subtree, so the button stopped the
app, restored 0 files, restarted it and said "Nincs hiányzó fájl — minden fájl megvan a helyén."
— at the moment the customer pressed it because files were missing, with 156 MB of BookStack's
data unread in the same copy. 9 apps have file legs but never their DB or volumes, so the same
sentence was also a clean bill of health over data never opened (immich: 1.3 GB Postgres unit).

Honesty half shipped: a pre-flight coverage check refuses UP FRONT without stopping the app and
NAMES the action that works; a run that proceeds claims only what it EXAMINED and discloses that
the database and volumes are not covered. Completeness is filed as C9-F1b — routing to the
Tier-1 unit restore puts a destructive operation behind a non-destructive button, so its confirm
copy has to carry that difference. C9-F4 filed: nothing reads the Tier-2 recovery-unit/ mirror,
so the second local copy that exists for drive loss is unreachable by any customer action.

C9-F2 (HIGH) — a crash loop was counted as working. StateRestarting is deliberately NOT added to
IsDownState (that alarms on every deploy fleet-wide, the over-correction F-A1 nearly cost us); a
sustained run becomes down after crashLoopAfter = 5m, set above the 120s deploy timeout, Mealie's
60s start_period and R-97b's 180s grace. The dashboard counter uses the same predicate, so it no
longer contradicts the alarm on the same screen. README's claim that faults "still surface as
restarting" was a wish with no test — corrected in place; it is the seventh such instance.

Six red-proofs observed, including the one that matters most: adding StateRestarting to
IsDownState fails the brief-restart test with "every deploy and update would page the operator".
go test ./... rc=0, 27 packages, run and read separately from this commit.
2026-07-28 18:53:56 +02:00

144 lines
5.6 KiB
Go

package backup
import (
"errors"
"os"
"path/filepath"
"testing"
)
// C9-F1 — the Tier-2 restore reads `hdd/` and `userdata/` only. `recovery-unit/` — the app's DB dumps
// and named-volume tarballs — is written by EVERY Tier-2 run and read by NOTHING on this path.
//
// For 43 of the 53 catalog apps (BookStack, Docmost, Vaultwarden, Gitea, …) that is the app's ENTIRE
// dataset, so the restore was a guaranteed no-op that still took an outage and reported
// „Nincs hiányzó fájl — minden fájl megvan a helyén."
//
// These tests pin the asymmetry itself, so a future change that alters what the restore reads must
// either keep the coverage answer honest or fail here.
// unitOnlyCopy rewrites the fixture's copy into the BookStack shape: a recovery unit and nothing the
// restore can read. Mirrors the live demo-felhom layout observed in Campaign 9
// (`legs=[NONE] unit=156M`).
func unitOnlyCopy(t *testing.T, destDrive string) string {
t.Helper()
destBase := filepath.Join(destDrive, "backups", "secondary", "app")
if err := os.RemoveAll(filepath.Join(destBase, "hdd")); err != nil {
t.Fatal(err)
}
mustWrite(t, filepath.Join(destBase, "recovery-unit", "volume-dumps", "app_db_data.tar"), "TARBYTES")
mustWrite(t, filepath.Join(destBase, "recovery-unit", "db-dumps", "app.sql"), "SQLDUMP")
return destBase
}
// SCENARIO D — a restore that cannot cover an app refuses BEFORE the outage.
//
// RED-PROOF (observed): remove the `!cov.CanRestore()` guard from RestoreTier2Files →
//
// tier2_coverage_test.go:63: RestoreTier2Files returned <nil> — a copy with nothing restorable was treated as success
// tier2_coverage_test.go:69: THE APP WAS STOPPED for a restore that could not restore anything: [app]
func TestRestoreTier2Files_NoRestorableSubtree_RefusesBeforeStopping(t *testing.T) {
m, fake, _, destDrive := newT2RManager(t)
unitOnlyCopy(t, destDrive)
m.restoreFilesCopier = func(string, string) (int, error) {
t.Fatal("the copier ran for an app with no restorable subtree")
return 0, nil
}
n, err := m.RestoreTier2Files("app")
if !errors.Is(err, ErrTier2NoRestorableData) {
t.Errorf("RestoreTier2Files returned %v — a copy with nothing restorable was treated as success", err)
}
if n != 0 {
t.Errorf("filesRestored = %d, want 0", n)
}
// The whole point: no outage was taken.
if len(fake.stopped) != 0 {
t.Errorf("THE APP WAS STOPPED for a restore that could not restore anything: %v", fake.stopped)
}
if len(fake.started) != 0 {
t.Errorf("the app was restarted, so it must have been stopped: %v", fake.started)
}
}
// The coverage query itself — what the handler pre-flights on, so it can refuse without starting an
// operation at all.
func TestTier2RestoreCoverage_ReportsTheAsymmetry(t *testing.T) {
m, _, _, destDrive := newT2RManager(t)
// Class A (paperless/immich shape): an hdd leg the restore reads, plus a unit it does not.
mustWrite(t, filepath.Join(destDrive, "backups", "secondary", "app", "recovery-unit", "manifest.json"), "{}")
cov, err := m.Tier2RestoreCoverage("app")
if err != nil {
t.Fatalf("coverage: %v", err)
}
if !cov.CanRestore() {
t.Error("an app WITH an hdd leg reported as not restorable — this would refuse the one path that works")
}
if !cov.HasUnit {
t.Error("the recovery unit was not detected — the disclosure would be omitted")
}
// Class B (bookstack/docmost shape): unit only.
unitOnlyCopy(t, destDrive)
cov, err = m.Tier2RestoreCoverage("app")
if err != nil {
t.Fatalf("coverage: %v", err)
}
if cov.CanRestore() {
t.Error("a unit-only copy reported as restorable — this is exactly C9-F1")
}
if !cov.HasUnit {
t.Error("the unit that holds the app's whole dataset was not detected")
}
if len(cov.Legs) != 0 {
t.Errorf("legs = %v, want none", cov.Legs)
}
}
// SCENARIO E — an app the restore CAN cover is completely unchanged. This is the regression guard on
// Campaign 9's headline result (A1/A3, paperless-ngx): byte-identical restore, stop→copy→start, and
// the additive-only promises intact. Breaking this to fix BookStack would be a straight regression on
// the only restore path proven to work on live hardware.
//
// RED-PROOF (observed): make the coverage guard unconditional (`if true`) →
//
// tier2_coverage_test.go:118: a COVERED app was refused: ennek az alkalmazásnak az adatai nem ebből a másolatból állíthatók vissza
func TestRestoreTier2Files_CoveredAppIsUnchanged(t *testing.T) {
m, fake, liveDrive, destDrive := newT2RManager(t)
// A unit is present too — a covered app has one as well; it must not change the outcome.
mustWrite(t, filepath.Join(destDrive, "backups", "secondary", "app", "recovery-unit", "manifest.json"), "{}")
var copied [][2]string
m.restoreFilesCopier = func(src, dst string) (int, error) {
copied = append(copied, [2]string{src, dst})
fake.order = append(fake.order, "copy")
return 3, nil
}
n, err := m.RestoreTier2Files("app")
if err != nil {
t.Fatalf("a COVERED app was refused: %v", err)
}
if n != 3 {
t.Errorf("filesRestored = %d, want 3", n)
}
if len(fake.stopped) != 1 || len(fake.started) != 1 {
t.Errorf("stop/start did not happen exactly once: %v / %v", fake.stopped, fake.started)
}
if got := fake.order; len(got) < 3 || got[0] != "stop" || got[len(got)-1] != "start" {
t.Errorf("order = %v, want stop → copy → start", got)
}
if len(copied) == 0 {
t.Fatal("nothing was copied for a covered app")
}
wantSrc := filepath.Join(destDrive, "backups", "secondary", "app", "hdd")
if copied[0][0] != wantSrc {
t.Errorf("src = %q, want %q", copied[0][0], wantSrc)
}
if copied[0][1] != liveDrive {
t.Errorf("dst = %q, want the live namespace root %q", copied[0][1], liveDrive)
}
}