package backup import ( "context" "encoding/json" "os" "path/filepath" "strings" "sync" "testing" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // R-87 Group B — the JOB. // // Everything here drives the real `ProveOffboxUnit` through the two seams REUSE.md names: // `SetOffboxRunner` (the restic exec seam — deliberately NOT a `resticStepFn`, so the // `unlock --remove-all` escalation stays visible to the argv assertions) and // `SetOffboxLatestSnapshotFn`. No restic, no ssh, no docker. // proofHarness is a Manager wired for the proof job, with a recorder for every restic argv. type proofHarness struct { m *Manager sett *settings.Settings prov *offbox3aProvider // drive is the registered storage path everything is resolved under. drive string mu sync.Mutex argv [][]string // unitFor decides what the fake restore materialises for a stack; nil = a healthy unit. unitFor map[string]unitFixture // snapFor overrides the snapshot id per stack; "" entries mean "no snapshot". snapFor map[string]string // failRestore makes the fake restic return an error. failRestore bool } // unitPathFor is the absolute path a snapshot records for a stack's recovery unit — the shape // `offboxUnitPathOf` matches on. func unitPathFor(stack string) string { return "/mnt/sys_drive/felhom-data/backups/primary/" + stack } func newProofHarness(t *testing.T, stacks ...string) *proofHarness { t.Helper() m, sett := newOffboxManager(t) drive := t.TempDir() if err := sett.AddStoragePath(settings.StoragePath{Path: drive, Label: "data", Schedulable: true}); err != nil { t.Fatal(err) } deployed := map[string]bool{} for _, s := range stacks { deployed[s] = true } prov := &offbox3aProvider{ hdd: map[string]string{}, binds: map[string][]ClassifiedBind{}, has: map[string]bool{}, deployed: deployed, } m.SetStackProvider(prov) h := &proofHarness{m: m, sett: sett, prov: prov, drive: drive, unitFor: map[string]unitFixture{}, snapFor: map[string]string{}} m.SetOffboxLatestSnapshotFn(func(_ context.Context, stack string) (string, []string, error) { id, ok := h.snapFor[stack] if !ok { id = "snap-" + stack } if id == "" { return "", nil, os.ErrNotExist } return id, []string{unitPathFor(stack)}, nil }) // The fake restic: records the argv, and for a `restore` MATERIALISES the unit the test asked // for, at the path inside --target that a real restic would write it to. m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) { h.mu.Lock() h.argv = append(h.argv, append([]string{}, args...)) h.mu.Unlock() if len(args) == 0 || !containsArg(args, "restore") { return nil, nil } if h.failRestore { return []byte("simulated restic failure"), os.ErrPermission } target, include := argValue(args, "--target"), argValue(args, "--include") if target == "" || include == "" { return nil, nil } stack := filepath.Base(include) dest := filepath.Join(target, strings.TrimPrefix(include, string(filepath.Separator))) materialiseUnit(t, dest, h.unitFor[stack]) return nil, nil }) // Plenty of headroom unless a test says otherwise. m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 }) return h } // containsArg lives in r358_scratch_marker_test.go — the same question, one implementation. func argValue(args []string, flag string) string { for i, a := range args { if a == flag && i+1 < len(args) { return args[i+1] } } return "" } // materialiseUnit writes a recovery unit at dest, mirroring what a restore would leave behind. // The zero fixture is a HEALTHY database app. func materialiseUnit(t *testing.T, dest string, f unitFixture) { t.Helper() if f.compose == "" && len(f.dbDumps) == 0 && len(f.volumeDumps) == 0 && !f.noManifest && f.rawManifest == "" { f = unitFixture{ compose: composeWithDB, dbDumps: []string{"app-mariadb.sql"}, volumeDumps: []string{"a.tar", "b.tar"}, } } for _, sub := range []string{"compose", "db-dumps", "volume-dumps"} { if err := os.MkdirAll(filepath.Join(dest, sub), 0o755); err != nil { t.Fatalf("mkdir: %v", err) } } if f.compose != "" { if err := os.WriteFile(filepath.Join(dest, "compose", "docker-compose.yml"), []byte(f.compose), 0o644); err != nil { t.Fatalf("compose: %v", err) } } if !f.declareOnly { for sub, names := range map[string][]string{"db-dumps": f.dbDumps, "volume-dumps": f.volumeDumps} { for _, n := range names { if err := os.WriteFile(filepath.Join(dest, sub, n), []byte("x"), 0o644); err != nil { t.Fatalf("write: %v", err) } } } } if f.noManifest { return } b, _ := json.Marshal(RecoveryManifest{DBDumps: f.dbDumps, VolumeDumps: f.volumeDumps}) if f.rawManifest != "" { b = []byte(f.rawManifest) } if err := os.WriteFile(filepath.Join(dest, "manifest.json"), b, 0o644); err != nil { t.Fatalf("manifest: %v", err) } } func (h *proofHarness) allArgs() [][]string { h.mu.Lock() defer h.mu.Unlock() return h.argv } func (h *proofHarness) proofScratch(t *testing.T, stack string) string { t.Helper() s, _, err := h.m.offboxProofScratchDir(stack) if err != nil { t.Fatalf("proof scratch dir: %v", err) } return s } // ── B1 ─────────────────────────────────────────────────────────────────────────────────────────── // TestR87_SkipsWhenRunningFlagHeld asserts a NON-EFFECT, the way TestR359_SkipsWhenRunningFlagHeld // does: restic was never invoked at all, and due-ness did not move. func TestR87_SkipsWhenRunningFlagHeld(t *testing.T) { h := newProofHarness(t, "kimai") if err := h.m.AcquireRunningForTest(); err != nil { t.Fatalf("could not take the flag: %v", err) } res := h.m.ProveOffboxUnit(context.Background()) if !res.Skipped { t.Fatalf("must SKIP while the single-writer flag is held; got %+v", res) } if got := len(h.allArgs()); got != 0 { t.Fatalf("a skip must invoke restic ZERO times; got %d invocations: %v", got, h.allArgs()) } if res.Verdict() != "" { t.Fatalf("a skip reaches no verdict; got %q", res.Verdict()) } h.m.RecordProofVerdict(res) if tt := h.sett.GetOffboxTarget(); len(tt.ProvedSnapshots) != 0 || tt.LastProofResult != "" { t.Fatalf("a skip must NOT advance due-ness; got proved=%v result=%q", tt.ProvedSnapshots, tt.LastProofResult) } } // ── B2 ─────────────────────────────────────────────────────────────────────────────────────────── // TestR87_ScratchIsDeletedOnEveryPath — pass, fail, cannot-judge and a restore error. // // RED-PROOF (run 2026-08-31): removing the `defer m.removeProofScratch(...)` makes the pass and fail // rows fail with the scratch directory still present. func TestR87_ScratchIsDeletedOnEveryPath(t *testing.T) { cases := map[string]struct { fixture unitFixture failRestore bool }{ "pass": {fixture: unitFixture{}}, "fail empty": {fixture: unitFixture{compose: composeWithDB}}, "cannot judge": {fixture: unitFixture{noManifest: false, compose: "", dbDumps: []string{"d.sql"}}}, "restore error": {failRestore: true}, } for name, c := range cases { t.Run(name, func(t *testing.T) { h := newProofHarness(t, "kimai") h.unitFor["kimai"] = c.fixture h.failRestore = c.failRestore scratch := h.proofScratch(t, "kimai") h.m.ProveOffboxUnit(context.Background()) if _, err := os.Stat(scratch); !os.IsNotExist(err) { t.Fatalf("the proof copy must be gone on EVERY path; %s still exists (stat err=%v)", scratch, err) } }) } } // TestR87_ProofScratchIsNotTheCustomerVerificationCopy — the delete above must never be able to reach // a copy the CUSTOMER made. Different roots is how that is guaranteed rather than hoped. func TestR87_ProofScratchIsNotTheCustomerVerificationCopy(t *testing.T) { h := newProofHarness(t, "kimai") proof := h.proofScratch(t, "kimai") customer, _, err := h.m.offboxRestoreScratchDir("kimai") if err != nil { t.Fatal(err) } if proof == customer { t.Fatal("the proof copy and the customer's verification copy MUST NOT share a path — the nightly delete would destroy the customer's copy") } if !strings.Contains(proof, "offsite-proof") || !strings.Contains(customer, "offsite-restore") { t.Fatalf("roots are not the expected pair: proof=%q customer=%q", proof, customer) } // And the customer's copy survives a full proof run that deletes its own. if err := os.MkdirAll(customer, 0o755); err != nil { t.Fatal(err) } sentinel := filepath.Join(customer, "customer-copy-marker") if err := os.WriteFile(sentinel, []byte("keep me"), 0o644); err != nil { t.Fatal(err) } h.m.ProveOffboxUnit(context.Background()) if _, err := os.Stat(sentinel); err != nil { t.Fatalf("the nightly proof deleted the CUSTOMER's verification copy: %v", err) } } // ── B3 ─────────────────────────────────────────────────────────────────────────────────────────── // TestR87_NeverWritesToTheRepository is Scenario E, asserted as a NON-EFFECT on the argv rather than // as the absence of an error. // // RED-PROOF (run 2026-08-31): routing the restore through `resticStep` with the `unlockStale` // pre-flight — i.e. what `RestoreOffboxScratch` does — makes this fail on `unlock` appearing in the // argv and on `--no-lock` being absent. func TestR87_NeverWritesToTheRepository(t *testing.T) { h := newProofHarness(t, "kimai") h.m.ProveOffboxUnit(context.Background()) all := h.allArgs() if len(all) == 0 { t.Fatal("the proof must have invoked restic — a zero-invocation run proves nothing about the argv") } // Every write verb restic has that this codebase ever issues. for _, args := range all { for _, verb := range []string{"unlock", "forget", "prune", "backup", "init", "--remove-all"} { if containsArg(args, verb) { t.Fatalf("the proof issued a WRITE verb %q against the repository: %v", verb, args) } } } var sawRestore bool for _, args := range all { if containsArg(args, "restore") { sawRestore = true if !containsArg(args, "--no-lock") { t.Fatalf("the proof restore must carry --no-lock so no lock file can be created: %v", args) } } } if !sawRestore { t.Fatal("no restore was issued — the --no-lock assertion above never ran") } } // ── B4, B5, B6 — due-ness ──────────────────────────────────────────────────────────────────────── func TestR87_OneAppPerRun(t *testing.T) { h := newProofHarness(t, "bookstack", "docmost", "kimai") res := h.m.ProveOffboxUnit(context.Background()) if res.Stack == "" { t.Fatal("a run must pick an app") } var restores int for _, args := range h.allArgs() { if containsArg(args, "restore") { restores++ } } if restores != 1 { t.Fatalf("one app per run: want exactly 1 restore, got %d", restores) } } // TestR87_ProvedSnapshotIsRecordedNotATimestamp. // // RED-PROOF (run 2026-08-31): recording `time.Now()` in `ProvedSnapshots[stack]` instead of the // snapshot ID makes this fail on the stored value, and makes B6 fail too because the app never // becomes due again. func TestR87_ProvedSnapshotIsRecordedNotATimestamp(t *testing.T) { h := newProofHarness(t, "kimai") h.snapFor["kimai"] = "a07c36a1" res := h.m.ProveOffboxUnit(context.Background()) if res.Verdict() != string(UnitProofPass) { t.Fatalf("fixture should pass; got %q (%q)", res.Verdict(), res.Judgement.Reason) } h.m.RecordProofVerdict(res) tt := h.sett.GetOffboxTarget() if got := tt.ProvedSnapshots["kimai"]; got != "a07c36a1" { t.Fatalf("the SNAPSHOT ID must be recorded; got %q", got) } if tt.LastProofSnapshot != "a07c36a1" || tt.LastProofStack != "kimai" || tt.LastProofResult != "pass" { t.Fatalf("the verdict record is wrong: %+v", tt) } // The record must not be a time: a timestamp would parse as one and an ID must not. if strings.Contains(tt.ProvedSnapshots["kimai"], ":") || strings.Contains(tt.ProvedSnapshots["kimai"], "T") { t.Fatalf("ProvedSnapshots looks like a timestamp, not a snapshot ID: %q", tt.ProvedSnapshots["kimai"]) } } func TestR87_AlreadyProvedSnapshotIsNotReProved(t *testing.T) { h := newProofHarness(t, "kimai") h.snapFor["kimai"] = "same-id" h.m.RecordProofVerdict(ProofResult{Stack: "kimai", SnapshotID: "same-id", Judgement: UnitProofResult{Verdict: UnitProofPass}}) res := h.m.ProveOffboxUnit(context.Background()) if !res.NoSnapshot { t.Fatalf("an already-proved newest snapshot must leave nothing due; got stack=%q", res.Stack) } for _, args := range h.allArgs() { if containsArg(args, "restore") { t.Fatalf("nothing due must download nothing; got %v", args) } } } // TestR87_NewSnapshotMakesAProvedAppDueAgain — the half a timestamp cannot do. func TestR87_NewSnapshotMakesAProvedAppDueAgain(t *testing.T) { h := newProofHarness(t, "kimai") h.snapFor["kimai"] = "old-id" h.m.RecordProofVerdict(ProofResult{Stack: "kimai", SnapshotID: "old-id", Judgement: UnitProofResult{Verdict: UnitProofPass}}) h.snapFor["kimai"] = "new-id" // the nightly backup landed res := h.m.ProveOffboxUnit(context.Background()) if res.Stack != "kimai" || res.SnapshotID != "new-id" { t.Fatalf("a NEW snapshot must make the app due again; got stack=%q snapshot=%q noSnapshot=%v", res.Stack, res.SnapshotID, res.NoSnapshot) } } // TestR87_RotationCoversEveryAppInAsManyNights — Scenario D end to end. func TestR87_RotationCoversEveryAppInAsManyNights(t *testing.T) { stacks := []string{"bookstack", "docmost", "kimai", "opengist"} h := newProofHarness(t, stacks...) seen := map[string]bool{} for i := 0; i < len(stacks); i++ { res := h.m.ProveOffboxUnit(context.Background()) if res.Stack == "" { t.Fatalf("night %d picked nothing; %d apps covered so far", i+1, len(seen)) } if seen[res.Stack] { t.Fatalf("night %d re-picked %s — the rotation must move on", i+1, res.Stack) } seen[res.Stack] = true h.m.RecordProofVerdict(res) } if len(seen) != len(stacks) { t.Fatalf("four nights must cover four apps; covered %v", seen) } // A fifth night has nothing due. if res := h.m.ProveOffboxUnit(context.Background()); !res.NoSnapshot { t.Fatalf("a fifth night must find nothing due; got %q", res.Stack) } } // ── B7, B8 ─────────────────────────────────────────────────────────────────────────────────────── // TestR87_HeadroomRefusalHappensBeforeTheDownload — a gate after the download is not a gate. func TestR87_HeadroomRefusalHappensBeforeTheDownload(t *testing.T) { h := newProofHarness(t, "kimai") h.m.SetOffboxFreeFn(func(string) int64 { return 1 << 20 }) // 1 MiB, well under the floor res := h.m.ProveOffboxUnit(context.Background()) if res.Err == nil { t.Fatal("an unwritable-headroom box must refuse") } for _, args := range h.allArgs() { if containsArg(args, "restore") { t.Fatalf("the refusal must happen BEFORE any download; a restore was issued: %v", args) } } // The customer-grade wording is shared with the restore path (REUSE.md's offsiteNoSpaceMsgFmt // rule). ASCII-only fragment, because an accented grep has returned 0 for strings that were there. if !strings.Contains(res.Err.Error(), "szabad hely") { t.Fatalf("the refusal must use the shared headroom wording; got %q", res.Err.Error()) } if res.Verdict() != "" { t.Fatal("a refusal reaches no verdict about the backup") } } func TestR87_NoSnapshotYetIsNotAFailure(t *testing.T) { h := newProofHarness(t, "newapp") h.snapFor["newapp"] = "" // never backed up res := h.m.ProveOffboxUnit(context.Background()) if !res.NoSnapshot { t.Fatalf("an app with no off-site snapshot is not a failure of this test; got %+v", res) } if res.Err != nil || res.Verdict() != "" { t.Fatalf("it must not be an error and must reach no verdict; err=%v verdict=%q", res.Err, res.Verdict()) } } // TestR87_RestoreFailureIsNotAVerdictAboutTheBackup — a download that did not finish has seen // nothing, so it must never become the "intact but empty" alarm. func TestR87_RestoreFailureIsNotAVerdictAboutTheBackup(t *testing.T) { h := newProofHarness(t, "kimai") h.failRestore = true res := h.m.ProveOffboxUnit(context.Background()) if res.Err == nil { t.Fatal("a failed restore must surface as an error") } if res.Verdict() != "" { t.Fatalf("a failed restore must reach NO verdict; got %q", res.Verdict()) } h.m.RecordProofVerdict(res) if tt := h.sett.GetOffboxTarget(); tt.LastProofResult != "" || len(tt.ProvedSnapshots) != 0 { t.Fatalf("a failed restore must not advance due-ness; got %+v", tt) } } // TestR87_HollowUnitReachesAFailVerdictThroughTheWholeJob — Scenario B through the job, not just the // predicate. The judgement is right in isolation and could still be wired to nothing. func TestR87_HollowUnitReachesAFailVerdictThroughTheWholeJob(t *testing.T) { h := newProofHarness(t, "kimai") h.unitFor["kimai"] = unitFixture{compose: composeWithDB} // the R-403 shape res := h.m.ProveOffboxUnit(context.Background()) if res.Verdict() != string(UnitProofFail) || res.Judgement.Reason != ProofReasonNoDatabaseDump { t.Fatalf("the job must reach a FAIL verdict on a hollow unit; got %q/%q", res.Verdict(), res.Judgement.Reason) } h.m.RecordProofVerdict(res) if tt := h.sett.GetOffboxTarget(); tt.LastProofResult != "fail" || tt.LastProofReason != string(ProofReasonNoDatabaseDump) { t.Fatalf("the failing verdict must be persisted with its reason; got result=%q reason=%q", tt.LastProofResult, tt.LastProofReason) } } // TestR87_CannotJudgeIsRecordedAndNotAPass. func TestR87_CannotJudgeIsRecordedAndNotAPass(t *testing.T) { h := newProofHarness(t, "kimai") h.unitFor["kimai"] = unitFixture{dbDumps: []string{"d.sql"}} // no compose materialised res := h.m.ProveOffboxUnit(context.Background()) if res.Verdict() != string(UnitProofCannotJudge) { t.Fatalf("a unit with no compose must be CANNOT JUDGE; got %q/%q", res.Verdict(), res.Judgement.Reason) } h.m.RecordProofVerdict(res) if tt := h.sett.GetOffboxTarget(); tt.LastProofResult != "cannot_judge" { t.Fatalf("cannot-judge must be recorded as itself; got %q", tt.LastProofResult) } } // TestR87_VerdictIsPublishedOnTheReportStatus — Part 2.4, including the absence rule. func TestR87_VerdictIsPublishedOnTheReportStatus(t *testing.T) { h := newProofHarness(t, "kimai") // Before any proof, the field must be ABSENT — not "fail", not "pass". if st := h.m.OffboxReportStatus(); st == nil || st.LastProofResult != "" { t.Fatalf("a box that has never proved must report NOT RECORDED; got %+v", st) } res := h.m.ProveOffboxUnit(context.Background()) h.m.RecordProofVerdict(res) st := h.m.OffboxReportStatus() if st.LastProofResult != "pass" || st.LastProofStack != "kimai" || st.LastProofSnapshot == "" || st.LastProofRun == "" { t.Fatalf("the verdict must reach the report status; got %+v", st) } // omitempty must keep the absent case off the wire entirely, or a reader cannot tell it apart. b, err := json.Marshal(&OffboxReportStatus{}) if err != nil { t.Fatal(err) } if strings.Contains(string(b), "last_proof_result") { t.Fatalf("an empty status must not emit last_proof_result at all; got %s", b) } }