fef07c3923
gates / gates (push) Successful in 9s
B2's capture floor (v0.192.0) was consulted in exactly ONE place — captureAllRecoveryUnits, which writes a few KB. The two legs that write the BULK into the same backups/primary/<app> tree, the DB dump and the volume dump, ran FIRST and unguarded. Measured live on demo-hp 2026-08-03 06:40:03: opengist's volume dump wrote 2.0 GB with no check, free fell to 1.0 GB, and the floor then refused the cheap write it had already lost the argument to. Its refusal message claimed "the previous unit is untouched" — measured false: that app's tar had gone 182,272 B -> 2,147,666,432 B under a stale manifest. Sixth entry in CLAUDE.md's table of shipped guarantees the code did not provide. Fix: ONE admission verdict per app per run (internal/backup/admission.go), taken before that app's FIRST write and covering all three legs — they write under one per-app root, which is why one verdict can honestly cover them. - Lazy, at the app's first write, NOT once at run start: app A's dump can put app B under the reserve, so a run-start verdict reads a disk that no longer exists. - Remembered for the run, never re-decided between an app's own legs — that is the split this closes. Reset per run. - Placed ahead of DumpAppVolumesSafe, which stops the stack as its first act, so a refused app is never bounced. After the volume-less check, which has no write. - Exactly one operator alert per refused app per run. - Leg order unchanged: volume dumps still precede the capture. The floor is now SIZE-AWARE: it asks whether THIS app's write would cross the reserve, not only whether the filesystem is already below it — which is how an app was admitted at 96% and then allowed to write 2 GB. Estimate = the app's previous .sql + .tar on disk. No history -> headroom-only, deliberately, and the alert says so. A container-based du per volume was MEASURED and rejected: 66 timed runs on demo-hp guest 9201, median ~355 ms/volume (341-404) on volumes holding tens of KB — container start-up, not the walk. Decisive on top: docker run needs the writable layer, so it can fail under exactly the pressure the reserve handles. The message was NOT weakened; the behaviour was moved so the wording became true. It now also names which term bound. Every claim is checked against a sha256 fingerprint of the tree it describes, never against the log line. Still refuses and never deletes: nothing here is generational. 11 new tests through the production functions. The DB leg cannot run without Docker, so its gate is pinned by an AST walk of backup.go asserting admitApp precedes DumpOne (strings.Contains is insufficient — a commented-out call still contains the string). 4 red-proofs demonstrated failing then restored.
700 lines
28 KiB
Go
700 lines
28 KiB
Go
package backup
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"go/ast"
|
|
"go/parser"
|
|
"go/token"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// R-181 — the reserve guards the write that fills the disk, and its promise is true.
|
|
//
|
|
// WHAT THESE ASSERT, AND WHY IT IS THE TREE AND NOT THE LOG. The defect being closed is precisely a
|
|
// log line that claimed something the filesystem contradicted: B2 printed *"the previous unit is
|
|
// untouched"* while the volume leg had already rewritten that unit's tar 182,272 B → 2,147,666,432 B.
|
|
// So a test that reads the message and believes it would have passed against the broken code. Every
|
|
// refusal test here checksums the whole `backups/primary` tree before and after and compares.
|
|
|
|
// ── Harness ──────────────────────────────────────────────────────────────────────────────────────
|
|
|
|
// admissionProvider records the two acts a refused app must never suffer: its recovery info being
|
|
// read (a capture that was ATTEMPTED) and its stack being stopped (which DumpAppVolumesSafe does as
|
|
// its first act, before any check of its own).
|
|
type admissionProvider struct {
|
|
stacks []string
|
|
volumes map[string][]string
|
|
dir string
|
|
infoHits []string
|
|
stopped []string
|
|
}
|
|
|
|
func (p *admissionProvider) GetStackComposePath(string) (string, bool) { return "", false }
|
|
func (p *admissionProvider) ListDeployedStacks() []StackSummary {
|
|
out := make([]StackSummary, 0, len(p.stacks))
|
|
for _, s := range p.stacks {
|
|
out = append(out, StackSummary{Name: s})
|
|
}
|
|
return out
|
|
}
|
|
func (p *admissionProvider) GetStackHDDMounts(string) []string { return nil }
|
|
func (p *admissionProvider) GetStackHDDPath(string) string { return "" }
|
|
func (p *admissionProvider) GetImportRoot() string { return "" }
|
|
func (p *admissionProvider) GetDockerVolumes(name string) []string {
|
|
if p.volumes == nil {
|
|
return []string{name + "_data"} // every app is volume-bearing unless told otherwise
|
|
}
|
|
return p.volumes[name]
|
|
}
|
|
func (p *admissionProvider) StopStack(name string) error {
|
|
p.stopped = append(p.stopped, name)
|
|
return nil
|
|
}
|
|
func (p *admissionProvider) StartStack(string) error { return nil }
|
|
func (p *admissionProvider) RefreshAndIsRunning(string) bool { return true }
|
|
func (p *admissionProvider) GetStackRecoveryInfo(name string) (RecoveryInfo, bool) {
|
|
p.infoHits = append(p.infoHits, name)
|
|
return RecoveryInfo{StackDir: filepath.Join(p.dir, "stacks", name)}, true
|
|
}
|
|
func (p *admissionProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
|
|
func (p *admissionProvider) RecreateStackDefinitionFromUnit(string, string, map[string]string) error {
|
|
return nil
|
|
}
|
|
func (p *admissionProvider) StartStackServices(string, []string) error { return nil }
|
|
func (p *admissionProvider) GetStackClassifiedBinds(string) ([]ClassifiedBind, bool) {
|
|
return nil, false
|
|
}
|
|
|
|
type admissionHarness struct {
|
|
m *Manager
|
|
prov *admissionProvider
|
|
events []unitEvent
|
|
usage map[string]*UnitSpace
|
|
dir string
|
|
logs *bytes.Buffer
|
|
volDumped []string
|
|
}
|
|
|
|
func newAdmissionHarness(t *testing.T, stacks ...string) *admissionHarness {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
h := &admissionHarness{
|
|
prov: &admissionProvider{stacks: stacks, dir: dir},
|
|
usage: map[string]*UnitSpace{},
|
|
dir: dir,
|
|
logs: &bytes.Buffer{},
|
|
}
|
|
h.m = &Manager{
|
|
logger: log.New(h.logs, "", 0),
|
|
systemDataPath: dir,
|
|
stackProvider: h.prov,
|
|
unitSpaceFn: func(name string) *UnitSpace { return h.usage[name] },
|
|
}
|
|
// The volume-dump seam records the leg that writes the BULK — the one B2 never gated. A refused
|
|
// app must not reach it.
|
|
h.m.dumpVolumesSafe = func(name string) error {
|
|
h.volDumped = append(h.volDumped, name)
|
|
// Write what the real leg writes, so an ungated call is visible in the tree checksum too.
|
|
dumpDir := AppVolumeDumpPath(h.nsRoot(), name)
|
|
if err := os.MkdirAll(dumpDir, 0o755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(filepath.Join(dumpDir, name+"_data.tar"), []byte("FRESH TAR FROM THIS RUN"), 0o644)
|
|
}
|
|
h.m.SetUnitNotify(func(name string, err error, u *UnitSpace) {
|
|
h.events = append(h.events, unitEvent{app: name, err: err.Error(), usage: u})
|
|
})
|
|
return h
|
|
}
|
|
|
|
func (h *admissionHarness) nsRoot() string { return filepath.Join(h.dir, "felhom-data") }
|
|
|
|
// setSpace states the filesystem's occupancy as a test INPUT — the whole point of the unitSpaceFn
|
|
// seam, so no test has to manufacture disk pressure on a real disk.
|
|
func (h *admissionHarness) setSpace(app string, usedPct, availGB, totalGB float64) {
|
|
h.usage[app] = &UnitSpace{
|
|
Path: h.dir, UsedPercent: usedPct, AvailGB: availGB,
|
|
TotalGB: totalGB, UsedGB: totalGB * usedPct / 100,
|
|
}
|
|
}
|
|
|
|
// seedUnit writes a previous recovery unit for an app: a manifest, a captured app.yaml, a DB dump and
|
|
// a volume tar of the given size. The tar is SPARSE (Truncate), so a 2 GiB "previous backup" costs no
|
|
// disk — the estimator reads st_size, which is what the next write will actually cost.
|
|
func (h *admissionHarness) seedUnit(t *testing.T, app string, tarBytes int64) {
|
|
t.Helper()
|
|
ns := h.nsRoot()
|
|
for _, d := range []string{
|
|
RecoveryUnitComposePath(ns, app),
|
|
AppDBDumpPath(ns, app),
|
|
AppVolumeDumpPath(ns, app),
|
|
} {
|
|
if err := os.MkdirAll(d, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
write := func(p string, b []byte, mode os.FileMode) {
|
|
if err := os.WriteFile(p, b, mode); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
write(RecoveryUnitManifestPath(ns, app), []byte(`{"app_name":"`+app+`","created_at":"2026-08-02T00:00:00Z"}`), 0o644)
|
|
write(filepath.Join(RecoveryUnitComposePath(ns, app), "app.yaml"), []byte("deployed: true\nenv:\n A: previous-good-value\n"), 0o600)
|
|
write(filepath.Join(AppDBDumpPath(ns, app), app+"-postgres.sql"), []byte("-- previous good dump\n"), 0o644)
|
|
|
|
tar := filepath.Join(AppVolumeDumpPath(ns, app), app+"_data.tar")
|
|
f, err := os.Create(tar)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.WriteString("PREVIOUS GOOD TAR"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if tarBytes > 0 {
|
|
if err := f.Truncate(tarBytes); err != nil { // sparse — st_size is the estimate, blocks are not spent
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// runOneBackupRun performs exactly the sequence runDBDumpsInternal performs for the two legs that can
|
|
// be driven without Docker: the admission scope is opened, the volume leg runs, then the capture leg.
|
|
// The DB leg's wiring is pinned structurally by TestAdmission_IsWiredIntoEveryProductionWriteLeg,
|
|
// because DiscoverDatabases shells out to `docker` and cannot honestly run here.
|
|
func (h *admissionHarness) runOneBackupRun() {
|
|
done := h.m.beginAdmissionRun()
|
|
defer done()
|
|
h.m.runVolumeDumps()
|
|
h.m.captureAllRecoveryUnits()
|
|
}
|
|
|
|
// ── The instrument: a checksum of the whole backup tree ──────────────────────────────────────────
|
|
|
|
// treeFingerprint walks every file under backups/primary and returns "relpath mode sha256" lines,
|
|
// sorted. It is the ONLY honest way to check the refusal's claim: it detects a rewritten payload, an
|
|
// added file and a deleted one alike, which a log line and an exit code both fail to do.
|
|
func treeFingerprint(t *testing.T, root string) string {
|
|
t.Helper()
|
|
var lines []string
|
|
err := filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if fi.IsDir() {
|
|
return nil
|
|
}
|
|
f, err := os.Open(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
sum := sha256.New()
|
|
if _, err := io.Copy(sum, f); err != nil {
|
|
return err
|
|
}
|
|
rel, _ := filepath.Rel(root, p)
|
|
lines = append(lines, fmt.Sprintf("%s %o %d %s", rel, fi.Mode().Perm(), fi.Size(), hex.EncodeToString(sum.Sum(nil))))
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("fingerprinting %s: %v", root, err)
|
|
}
|
|
sort.Strings(lines)
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
// treeStatFingerprint is the instrument for trees holding a multi-GiB fixture, where hashing every
|
|
// byte costs more than it proves: name + mode + SIZE. It still catches the act being tested — the
|
|
// volume leg replacing a 2 GiB tar with a freshly written one — because that changes the size, and it
|
|
// catches an added or deleted file by name. Content-identical-but-different-bytes is the one thing it
|
|
// cannot see, which is why the small-tree tests use treeFingerprint instead.
|
|
func treeStatFingerprint(t *testing.T, root string) string {
|
|
t.Helper()
|
|
var lines []string
|
|
_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
|
|
if err != nil || fi.IsDir() {
|
|
return nil
|
|
}
|
|
rel, _ := filepath.Rel(root, p)
|
|
lines = append(lines, fmt.Sprintf("%s %o %d", rel, fi.Mode().Perm(), fi.Size()))
|
|
return nil
|
|
})
|
|
sort.Strings(lines)
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
// treeFileList is the weaker instrument used for Scenario F: names only, so the assertion is
|
|
// specifically about DELETION and cannot be satisfied or broken by a content change.
|
|
func treeFileList(t *testing.T, root string) []string {
|
|
t.Helper()
|
|
var names []string
|
|
_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
|
|
if err != nil || fi.IsDir() {
|
|
return nil
|
|
}
|
|
rel, _ := filepath.Rel(root, p)
|
|
names = append(names, rel)
|
|
return nil
|
|
})
|
|
sort.Strings(names)
|
|
return names
|
|
}
|
|
|
|
func (h *admissionHarness) primaryRoot() string {
|
|
return PrimaryBackupPath(h.nsRoot())
|
|
}
|
|
|
|
// ── Scenario A — one decision, taken before the first byte ───────────────────────────────────────
|
|
|
|
func TestAdmission_RefusedAppWritesNothingAndIsNotStopped(t *testing.T) {
|
|
h := newAdmissionHarness(t, "privatebin", "opengist", "homebox")
|
|
h.setSpace("privatebin", 40, 60, 100)
|
|
h.setSpace("opengist", 98, 0.4, 70) // below the reserve on BOTH terms
|
|
h.setSpace("homebox", 40, 60, 100)
|
|
h.seedUnit(t, "opengist", 0)
|
|
|
|
// Scoped to the REFUSED app's own unit: its two siblings are admitted and legitimately write
|
|
// theirs, so a whole-tree fingerprint would change for the right reason and prove nothing here.
|
|
// Scenario F below takes the whole-tree view, where every app is refused.
|
|
refusedUnit := RecoveryUnitPath(h.nsRoot(), "opengist")
|
|
before := treeFingerprint(t, refusedUnit)
|
|
if before == "" {
|
|
t.Fatal("the fixture seeded no previous unit, so 'byte-identical' would be vacuously true")
|
|
}
|
|
h.runOneBackupRun()
|
|
after := treeFingerprint(t, refusedUnit)
|
|
|
|
// 1. NOT ONE of the three legs ran for the refused app.
|
|
for _, got := range h.volDumped {
|
|
if got == "opengist" {
|
|
t.Fatal("the VOLUME leg ran for a refused app — this is the R-181 defect exactly: the leg " +
|
|
"that writes the bulk was never gated, so the reserve it protects was consumed by the " +
|
|
"very step it exists to bound")
|
|
}
|
|
}
|
|
for _, got := range h.prov.infoHits {
|
|
if got == "opengist" {
|
|
t.Fatal("the CAPTURE leg was attempted for a refused app — the verdict must be taken before " +
|
|
"any write is prepared, not partway through one")
|
|
}
|
|
}
|
|
|
|
// 2. The tree is byte-identical. This is the assertion the broken code could not pass.
|
|
if after != before {
|
|
t.Fatalf("the backup tree CHANGED across a refusal.\n--- before ---\n%s\n--- after ---\n%s\n"+
|
|
"A refusal that has already rewritten the payload is the defect, not the fix", before, after)
|
|
}
|
|
|
|
// 3. The app was never stopped. DumpAppVolumesSafe stops the stack as its FIRST act, so a gate
|
|
// placed inside it would bounce the app it is refusing to back up.
|
|
for _, got := range h.prov.stopped {
|
|
if got == "opengist" {
|
|
t.Fatal("the refused app was STOPPED — the reserve check has drifted behind the stop")
|
|
}
|
|
}
|
|
|
|
// 4. Exactly ONE alert, for that app, carrying the space figures. Three legs must not mean three
|
|
// emails about one disk.
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d alerts, want exactly 1 (one app refused, three legs): %+v", len(h.events), h.events)
|
|
}
|
|
if h.events[0].app != "opengist" {
|
|
t.Fatalf("alert names %q, want opengist", h.events[0].app)
|
|
}
|
|
if h.events[0].usage == nil || h.events[0].usage.AvailGB != 0.4 {
|
|
t.Fatalf("the alert carries no/incorrect space figures: %+v", h.events[0].usage)
|
|
}
|
|
}
|
|
|
|
// ── Scenario B — the other apps are unaffected ───────────────────────────────────────────────────
|
|
|
|
func TestAdmission_SiblingAppsProceedAndOnlyTheRefusedOneAlerts(t *testing.T) {
|
|
h := newAdmissionHarness(t, "privatebin", "opengist", "homebox")
|
|
h.setSpace("privatebin", 40, 60, 100)
|
|
h.setSpace("opengist", 99, 0.2, 70)
|
|
h.setSpace("homebox", 40, 60, 100)
|
|
|
|
h.runOneBackupRun()
|
|
|
|
for _, app := range []string{"privatebin", "homebox"} {
|
|
if !hasStr(h.volDumped, app) {
|
|
t.Errorf("%s was not volume-dumped (dumped=%v) — one app's refusal silenced its siblings", app, h.volDumped)
|
|
}
|
|
if !hasStr(h.prov.infoHits, app) {
|
|
t.Errorf("%s was not captured (attempted=%v) — the loop did not continue past the refusal", app, h.prov.infoHits)
|
|
}
|
|
if _, err := os.Stat(RecoveryUnitManifestPath(h.nsRoot(), app)); err != nil {
|
|
t.Errorf("%s has no manifest after the run: %v — an admitted app must be backed up normally", app, err)
|
|
}
|
|
}
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d alerts, want exactly 1: %+v", len(h.events), h.events)
|
|
}
|
|
}
|
|
|
|
// ── Scenario C — the promise is true ─────────────────────────────────────────────────────────────
|
|
|
|
// Every claim the shipped message makes is checked against the tree it describes. The wording is NOT
|
|
// weakened to fit the behaviour; the behaviour was moved so the wording became true (§8.3).
|
|
func TestAdmission_EveryClaimInTheRefusalMessageHoldsAgainstTheTree(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
h.setSpace("opengist", 98, 0.5, 70)
|
|
h.seedUnit(t, "opengist", 0)
|
|
|
|
beforeFP := treeFingerprint(t, h.primaryRoot())
|
|
beforeList := treeFileList(t, h.primaryRoot())
|
|
h.runOneBackupRun()
|
|
msg := h.logs.String()
|
|
|
|
if !strings.Contains(msg, "REFUSED for opengist") {
|
|
t.Fatalf("no refusal was logged for opengist; log was:\n%s", msg)
|
|
}
|
|
|
|
// Claim 1: "NO database dump, NO volume dump and NO recovery-unit capture was written for it".
|
|
for _, claim := range []string{"NO database dump", "NO volume dump", "NO recovery-unit capture"} {
|
|
if !strings.Contains(msg, claim) {
|
|
t.Fatalf("the message no longer claims %q — if a leg cannot be brought under the verdict the "+
|
|
"wording must be narrowed deliberately and the gap named, not dropped silently.\n%s", claim, msg)
|
|
}
|
|
}
|
|
if len(h.volDumped) != 0 || len(h.prov.infoHits) != 0 {
|
|
t.Fatalf("the message claims no leg ran, but volume=%v capture=%v", h.volDumped, h.prov.infoHits)
|
|
}
|
|
|
|
// Claim 2: "the previous unit is untouched" — the claim that was MEASURED FALSE in R-181.
|
|
if !strings.Contains(msg, "the previous unit is untouched") {
|
|
t.Fatalf("the message dropped the untouched claim: %s", msg)
|
|
}
|
|
if got := treeFingerprint(t, h.primaryRoot()); got != beforeFP {
|
|
t.Fatalf("the message says the previous unit is untouched; the tree says otherwise.\n"+
|
|
"--- before ---\n%s\n--- after ---\n%s", beforeFP, got)
|
|
}
|
|
|
|
// Claim 3: "NOTHING was deleted".
|
|
if !strings.Contains(msg, "NOTHING was deleted") {
|
|
t.Fatalf("the message dropped the no-deletion claim: %s", msg)
|
|
}
|
|
if got := treeFileList(t, h.primaryRoot()); !equalStrs(got, beforeList) {
|
|
t.Fatalf("files disappeared across a refusal: before=%v after=%v", beforeList, got)
|
|
}
|
|
|
|
// Claim 4: the reason is named, so the operator can tell which term bound.
|
|
if !strings.Contains(msg, "headroom") {
|
|
t.Fatalf("the message does not name WHICH term bound — an operator cannot tell 'the disk is "+
|
|
"full' from 'this app's backup is too big for what is left':\n%s", msg)
|
|
}
|
|
}
|
|
|
|
// ── Scenario D — size-aware, not just headroom-aware ─────────────────────────────────────────────
|
|
|
|
// The live R-181 sequence, reproduced as a unit: the filesystem is ABOVE the reserve on both terms
|
|
// when the run reaches the app, and the app's own write is what crosses it. Under B2 this app was
|
|
// admitted at 96% and then allowed to write 2 GB.
|
|
func TestAdmission_SizeTermRefusesAnAppWhoseOwnWriteWouldCrossTheReserve(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
// 96% used of 70 GiB, 3.0 GiB free — BOTH reserve terms deliberately still clear (97% / 1.0 GiB),
|
|
// exactly as on demo-hp at 06:40:03, so a headroom-only rule starts the run.
|
|
h.setSpace("opengist", 96, 3.0, 70)
|
|
if _, r := h.m.floorVerdict(h.usage["opengist"], 0); r != floorAdmit {
|
|
t.Fatalf("fixture is wrong: the headroom term already refuses (%v), so this test would pass "+
|
|
"without a size term and prove nothing", r)
|
|
}
|
|
h.seedUnit(t, "opengist", 2<<30) // its last backup was 2 GiB — the figure measured live
|
|
|
|
before := treeStatFingerprint(t, h.primaryRoot())
|
|
h.runOneBackupRun()
|
|
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d alerts, want 1 — the app was admitted at 96%% and would have been allowed to "+
|
|
"write 2 GiB, which is the R-181 sequence: %+v", len(h.events), h.events)
|
|
}
|
|
if !strings.Contains(h.logs.String(), "(size)") {
|
|
t.Fatalf("the refusal was not attributed to the SIZE term:\n%s", h.logs.String())
|
|
}
|
|
if !strings.Contains(h.events[0].err, "last backup was 2.00 GiB") {
|
|
t.Fatalf("the alert does not carry the estimate that produced the refusal: %q", h.events[0].err)
|
|
}
|
|
if len(h.volDumped) != 0 {
|
|
t.Fatalf("the volume leg ran anyway: %v", h.volDumped)
|
|
}
|
|
if got := treeStatFingerprint(t, h.primaryRoot()); got != before {
|
|
t.Fatalf("the tree changed despite the size-term refusal.\nbefore=%s\nafter =%s", before, got)
|
|
}
|
|
}
|
|
|
|
// ── Scenario E — a first-ever backup is not blocked by having no history ─────────────────────────
|
|
|
|
func TestAdmission_FirstEverBackupIsAdmitted(t *testing.T) {
|
|
h := newAdmissionHarness(t, "brandnew")
|
|
h.setSpace("brandnew", 40, 600, 1000) // ample room, and NO previous unit on disk
|
|
|
|
if est, ok := h.m.estimatedWriteGiB("brandnew"); ok || est != 0 {
|
|
t.Fatalf("estimatedWriteGiB = (%v, %v) for an app with no history, want (0, false)", est, ok)
|
|
}
|
|
h.runOneBackupRun()
|
|
|
|
if len(h.events) != 0 {
|
|
t.Fatalf("a brand-new app was refused: %+v — refusing every app that has no size to estimate "+
|
|
"from would make the FIRST backup the one that can never happen", h.events)
|
|
}
|
|
if !hasStr(h.volDumped, "brandnew") || !hasStr(h.prov.infoHits, "brandnew") {
|
|
t.Fatalf("the app was not backed up (volume=%v capture=%v)", h.volDumped, h.prov.infoHits)
|
|
}
|
|
}
|
|
|
|
// ── Scenario F — the reserve still never deletes ─────────────────────────────────────────────────
|
|
|
|
func TestAdmission_NothingUnderBackupsIsEverRemoved(t *testing.T) {
|
|
h := newAdmissionHarness(t, "privatebin", "opengist", "homebox")
|
|
for _, app := range []string{"privatebin", "opengist", "homebox"} {
|
|
h.setSpace(app, 99, 0.1, 70) // every app refused — maximum pressure to "make room"
|
|
h.seedUnit(t, app, 0)
|
|
}
|
|
|
|
before := treeFileList(t, h.primaryRoot())
|
|
h.runOneBackupRun()
|
|
after := treeFileList(t, h.primaryRoot())
|
|
|
|
if !equalStrs(before, after) {
|
|
t.Fatalf("the file list changed under the reserve.\nbefore=%v\nafter =%v\n"+
|
|
"Nothing here is generational — a unit is ONE fixed path per app — so 'prune the oldest' "+
|
|
"could only mean destroying a DIFFERENT app's only local recovery unit", before, after)
|
|
}
|
|
if len(before) == 0 {
|
|
t.Fatal("the fixture seeded no files, so this test would pass against code that deleted everything")
|
|
}
|
|
}
|
|
|
|
// ── §8.1 — one verdict per app per run, and it resets between runs ───────────────────────────────
|
|
|
|
// The verdict must not be re-taken between an app's own legs. Re-deciding is how the split this fixes
|
|
// came about: DB leg admitted, volume leg admitted, capture refused — with the bulk already written.
|
|
func TestAdmission_VerdictIsTakenOncePerAppPerRunAndNotRedecidedBetweenLegs(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
reads := 0
|
|
h.m.unitSpaceFn = func(string) *UnitSpace {
|
|
reads++
|
|
if reads == 1 {
|
|
return &UnitSpace{Path: h.dir, UsedPercent: 99, AvailGB: 0.1, TotalGB: 70, UsedGB: 69.3}
|
|
}
|
|
// The disk "recovers" mid-run. A re-decided verdict would admit the capture leg here — which
|
|
// is precisely the split R-181 closes, arriving from the other direction.
|
|
return &UnitSpace{Path: h.dir, UsedPercent: 10, AvailGB: 60, TotalGB: 70, UsedGB: 7}
|
|
}
|
|
|
|
h.runOneBackupRun()
|
|
|
|
if reads != 1 {
|
|
t.Fatalf("the filesystem was read %d times for ONE app in ONE run — the verdict is being "+
|
|
"re-decided between legs, which reintroduces the split (bulk written, capture refused)", reads)
|
|
}
|
|
if len(h.prov.infoHits) != 0 {
|
|
t.Fatal("the capture leg ran after the app was refused earlier in the same run")
|
|
}
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d alerts, want exactly 1 per app per run: %+v", len(h.events), h.events)
|
|
}
|
|
}
|
|
|
|
// A set carried between runs is a wrong answer with a confident face: tonight's question answered
|
|
// with last night's disk.
|
|
func TestAdmission_TheRememberedSetResetsBetweenRuns(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
h.setSpace("opengist", 99, 0.1, 70)
|
|
h.runOneBackupRun()
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("run 1: want 1 alert, got %+v", h.events)
|
|
}
|
|
|
|
h.setSpace("opengist", 20, 55, 70) // space freed between runs
|
|
h.runOneBackupRun()
|
|
|
|
if !hasStr(h.volDumped, "opengist") {
|
|
t.Fatal("the second run still refused the app — the previous run's verdict was carried over, " +
|
|
"so freeing space could never take effect")
|
|
}
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("the second (admitted) run alerted again: %+v", h.events)
|
|
}
|
|
}
|
|
|
|
// ── §8.4 — a nil reading neither refuses nor warns, across ALL THREE legs ────────────────────────
|
|
|
|
// Unchanged behaviour, re-pinned because the decision now governs three legs instead of one: an
|
|
// unreadable filesystem must not silently stop an app being backed up at all.
|
|
func TestAdmission_UnreadableFilesystemAdmitsEveryLegAndDoesNotWarn(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
// No usage entry → the reader returns nil, which is what system.GetDiskUsage does on error.
|
|
|
|
h.runOneBackupRun()
|
|
|
|
if len(h.events) != 0 {
|
|
t.Fatalf("an unreadable filesystem produced %d alert(s): %+v — that is the drive gate's "+
|
|
"business and has its own alert", len(h.events), h.events)
|
|
}
|
|
if !hasStr(h.volDumped, "opengist") {
|
|
t.Fatal("the VOLUME leg was refused on an unreadable read — a drive that merely blipped would " +
|
|
"now stop the bulk of the backup, not just the capture")
|
|
}
|
|
if !hasStr(h.prov.infoHits, "opengist") {
|
|
t.Fatal("the CAPTURE leg was refused on an unreadable read")
|
|
}
|
|
}
|
|
|
|
// ── The estimator, through the production path (no seam) ─────────────────────────────────────────
|
|
|
|
func TestEstimatedWriteGiB_SumsTheAppsPreviousDumpsFromRealFiles(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
h.seedUnit(t, "opengist", 3<<30) // 3 GiB sparse tar + a small .sql
|
|
|
|
est, ok := h.m.estimatedWriteGiB("opengist")
|
|
if !ok {
|
|
t.Fatal("history on disk was not recognised as history")
|
|
}
|
|
if est < 3.0 || est > 3.001 {
|
|
t.Fatalf("estimate = %.4f GiB, want ~3.0 (the .tar plus the small .sql)", est)
|
|
}
|
|
|
|
// An app whose unit exists but holds no dumps yet is history-LESS, not a zero-byte estimate.
|
|
other := AppVolumeDumpPath(h.nsRoot(), "empty")
|
|
if err := os.MkdirAll(other, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if est, ok := h.m.estimatedWriteGiB("empty"); ok || est != 0 {
|
|
t.Fatalf("an empty unit reported history (%v, %v) — an absent dump is not a 0-byte one", est, ok)
|
|
}
|
|
}
|
|
|
|
// ── The seam is WIRED — walked as an AST, not grepped ────────────────────────────────────────────
|
|
|
|
// FOUR mechanisms in this project have been built and left disconnected (REUSE.md's seam register).
|
|
// The behavioural tests above drive the two legs that can run without Docker; the DB leg cannot, so
|
|
// its gate is pinned HERE, structurally. `strings.Contains` is deliberately not used: a commented-out
|
|
// call still contains the string, and so does a call inside dead code.
|
|
func TestAdmission_IsWiredIntoEveryProductionWriteLeg(t *testing.T) {
|
|
fset := token.NewFileSet()
|
|
file, err := parser.ParseFile(fset, "backup.go", nil, 0) // comments dropped — only real calls survive
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
calls := map[string][]string{} // enclosing func → called names, in source order
|
|
var current string
|
|
ast.Inspect(file, func(n ast.Node) bool {
|
|
switch v := n.(type) {
|
|
case *ast.FuncDecl:
|
|
current = v.Name.Name
|
|
case *ast.CallExpr:
|
|
name := ""
|
|
switch fn := v.Fun.(type) {
|
|
case *ast.Ident:
|
|
name = fn.Name
|
|
case *ast.SelectorExpr:
|
|
name = fn.Sel.Name
|
|
}
|
|
if name != "" && current != "" {
|
|
calls[current] = append(calls[current], name)
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
|
|
// 1. The run scope is opened by the orchestrator of all three legs.
|
|
if !hasStr(calls["runDBDumpsInternal"], "beginAdmissionRun") {
|
|
t.Fatal("runDBDumpsInternal does not open the admission scope — without it every leg decides " +
|
|
"independently and the per-run memo never exists, which is the pre-R-181 behaviour")
|
|
}
|
|
|
|
// 2. The DB leg consults it BEFORE the dump. Order is the whole point: a gate after the write is
|
|
// the defect, relocated.
|
|
assertGateBefore(t, calls["runDBDumpsInternal"], "admitApp", "DumpOne",
|
|
"the DATABASE leg dumps before consulting the reserve")
|
|
|
|
// 3. The volume leg consults it BEFORE the dump seam — which stops the stack as its first act.
|
|
assertGateBefore(t, calls["runVolumeDumps"], "admitApp", "dump",
|
|
"the VOLUME leg — the one that writes the bulk, and the one B2 never gated — dumps before "+
|
|
"consulting the reserve")
|
|
|
|
// 4. The capture leg, in its own file.
|
|
rfset := token.NewFileSet()
|
|
rfile, err := parser.ParseFile(rfset, "recovery_unit.go", nil, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
capCalls := map[string][]string{}
|
|
current = ""
|
|
ast.Inspect(rfile, func(n ast.Node) bool {
|
|
switch v := n.(type) {
|
|
case *ast.FuncDecl:
|
|
current = v.Name.Name
|
|
case *ast.CallExpr:
|
|
if sel, ok := v.Fun.(*ast.SelectorExpr); ok && current != "" {
|
|
capCalls[current] = append(capCalls[current], sel.Sel.Name)
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
assertGateBefore(t, capCalls["captureAllRecoveryUnits"], "admitApp", "CaptureRecoveryUnit",
|
|
"the CAPTURE leg captures before consulting the reserve")
|
|
}
|
|
|
|
// assertGateBefore checks that `gate` appears in the call list before `act`.
|
|
func assertGateBefore(t *testing.T, calls []string, gate, act, why string) {
|
|
t.Helper()
|
|
gi, ai := -1, -1
|
|
for i, c := range calls {
|
|
if c == gate && gi < 0 {
|
|
gi = i
|
|
}
|
|
if c == act && ai < 0 {
|
|
ai = i
|
|
}
|
|
}
|
|
if gi < 0 {
|
|
t.Fatalf("%s: %q is never called there at all (calls=%v)", why, gate, calls)
|
|
}
|
|
if ai < 0 {
|
|
t.Fatalf("fixture drift: %q is no longer called in that function (calls=%v) — this test can no "+
|
|
"longer see the act it is ordering the gate against", act, calls)
|
|
}
|
|
if gi > ai {
|
|
t.Fatalf("%s: %q first appears at %d, after %q at %d", why, gate, gi, act, ai)
|
|
}
|
|
}
|
|
|
|
func hasStr(hay []string, needle string) bool {
|
|
for _, s := range hay {
|
|
if s == needle {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func equalStrs(a, b []string) bool {
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
for i := range a {
|
|
if a[i] != b[i] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|