88897a224e
gates / gates (push) Successful in 8s
MEASURED, not supposed. On 2026-08-03 nine per-app recovery_unit_capture_failed events reached the hub and TWO operator emails went out. The hub's operator cooldown key is customerID:eventType(+tier) and that event carries `app` but no `tier`, so the key held no app identifier: the first refused app took the hour's slot and every other app's failure was discarded BEFORE anything was written down, leaving no row on any channel. The obvious fix — put `app` in the key — was ruled against: on a full disk it produces one email per app, the volume problem wearing the correctness problem's clothes. internal/backup/runsummary.go: a per-run collector with exactly admissionSet's lifetime, fed by all three write legs, emitting backup_run_failures ONCE at the end and only when something failed. A clean run emits nothing. The per-app event stays and becomes the RECORD — the hub routes it record-only, stored and logged every time, never competing for an email slot. The record and the notification are now different things. Deliberate skips (disconnected, decommissioned) are excluded: they have their own alert, and a nightly email about an unplugged drive is one the operator learns to ignore. A manual run always reports: the digest carries a unique run_id the cooldown cannot collapse. Someone pressing the button is actively trying to get a backup. THE PERIODIC SWEEP GETS A DIGEST TOO. With the per-app event now record-only, a capture failure found between runs would be recorded and never notified — a new silence introduced while closing one. That path emits a digest with NO run_id, so the ordinary 1-hour cooldown caps it exactly as before while the mail now lists every failing app instead of whichever was first. A refusal is recorded ONCE, where the verdict is taken, not at the three legs that consult it — R-181's contract is one verdict per app per run. Noting it per leg listed one refused app three times and produced "2 of 1 apps failed". Found by the digest's own test, not in review. Silence is safe because the hub's deadline check raises expected_backup_missed from report freshness, independently of any mail this box sends (monitor/deadline.go:396,417). Confirmed, not assumed. 7 new tests, 4 red-proofs. The main.go seam walk did NOT fail on its first attempt — the AST test walked the backup package and not main.go; the test was fixed and the mutation re-run rather than the pass recorded.
742 lines
29 KiB
Go
742 lines
29 KiB
Go
package backup
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"go/ast"
|
|
"go/parser"
|
|
"go/token"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
// R-181 — the reserve guards the write that fills the disk, and its promise is true.
|
|
//
|
|
// WHAT THESE ASSERT, AND WHY IT IS THE TREE AND NOT THE LOG. The defect being closed is precisely a
|
|
// log line that claimed something the filesystem contradicted: B2 printed *"the previous unit is
|
|
// untouched"* while the volume leg had already rewritten that unit's tar 182,272 B → 2,147,666,432 B.
|
|
// So a test that reads the message and believes it would have passed against the broken code. Every
|
|
// refusal test here checksums the whole `backups/primary` tree before and after and compares.
|
|
|
|
// ── Harness ──────────────────────────────────────────────────────────────────────────────────────
|
|
|
|
// admissionProvider records the two acts a refused app must never suffer: its recovery info being
|
|
// read (a capture that was ATTEMPTED) and its stack being stopped (which DumpAppVolumesSafe does as
|
|
// its first act, before any check of its own).
|
|
type admissionProvider struct {
|
|
stacks []string
|
|
volumes map[string][]string
|
|
hdd map[string]string // per-app drive path, for the drive-state skip tests
|
|
dir string
|
|
infoHits []string
|
|
stopped []string
|
|
}
|
|
|
|
func (p *admissionProvider) GetStackComposePath(string) (string, bool) { return "", false }
|
|
func (p *admissionProvider) ListDeployedStacks() []StackSummary {
|
|
out := make([]StackSummary, 0, len(p.stacks))
|
|
for _, s := range p.stacks {
|
|
out = append(out, StackSummary{Name: s})
|
|
}
|
|
return out
|
|
}
|
|
func (p *admissionProvider) GetStackHDDMounts(string) []string { return nil }
|
|
func (p *admissionProvider) GetStackHDDPath(n string) string { return p.hdd[n] }
|
|
func (p *admissionProvider) GetImportRoot() string { return "" }
|
|
func (p *admissionProvider) GetDockerVolumes(name string) []string {
|
|
if p.volumes == nil {
|
|
return []string{name + "_data"} // every app is volume-bearing unless told otherwise
|
|
}
|
|
return p.volumes[name]
|
|
}
|
|
func (p *admissionProvider) StopStack(name string) error {
|
|
p.stopped = append(p.stopped, name)
|
|
return nil
|
|
}
|
|
func (p *admissionProvider) StartStack(string) error { return nil }
|
|
func (p *admissionProvider) RefreshAndIsRunning(string) bool { return true }
|
|
func (p *admissionProvider) GetStackRecoveryInfo(name string) (RecoveryInfo, bool) {
|
|
p.infoHits = append(p.infoHits, name)
|
|
return RecoveryInfo{StackDir: filepath.Join(p.dir, "stacks", name)}, true
|
|
}
|
|
func (p *admissionProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
|
|
func (p *admissionProvider) RecreateStackDefinitionFromUnit(string, string, map[string]string) error {
|
|
return nil
|
|
}
|
|
func (p *admissionProvider) StartStackServices(string, []string) error { return nil }
|
|
func (p *admissionProvider) GetStackClassifiedBinds(string) ([]ClassifiedBind, bool) {
|
|
return nil, false
|
|
}
|
|
|
|
type admissionHarness struct {
|
|
m *Manager
|
|
prov *admissionProvider
|
|
events []unitEvent
|
|
usage map[string]*UnitSpace
|
|
dir string
|
|
logs *bytes.Buffer
|
|
volDumped []string
|
|
}
|
|
|
|
func newAdmissionHarness(t *testing.T, stacks ...string) *admissionHarness {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
h := &admissionHarness{
|
|
prov: &admissionProvider{stacks: stacks, dir: dir, hdd: map[string]string{}},
|
|
usage: map[string]*UnitSpace{},
|
|
dir: dir,
|
|
logs: &bytes.Buffer{},
|
|
}
|
|
h.m = &Manager{
|
|
logger: log.New(h.logs, "", 0),
|
|
systemDataPath: dir,
|
|
stackProvider: h.prov,
|
|
unitSpaceFn: func(name string) *UnitSpace { return h.usage[name] },
|
|
}
|
|
// The volume-dump seam records the leg that writes the BULK — the one B2 never gated. A refused
|
|
// app must not reach it.
|
|
h.m.dumpVolumesSafe = func(name string) error {
|
|
h.volDumped = append(h.volDumped, name)
|
|
// Write what the real leg writes, so an ungated call is visible in the tree checksum too.
|
|
dumpDir := AppVolumeDumpPath(h.nsRoot(), name)
|
|
if err := os.MkdirAll(dumpDir, 0o755); err != nil {
|
|
return err
|
|
}
|
|
return os.WriteFile(filepath.Join(dumpDir, name+"_data.tar"), []byte("FRESH TAR FROM THIS RUN"), 0o644)
|
|
}
|
|
h.m.SetUnitNotify(func(name string, err error, u *UnitSpace) {
|
|
h.events = append(h.events, unitEvent{app: name, err: err.Error(), usage: u})
|
|
})
|
|
return h
|
|
}
|
|
|
|
// markDisconnected / markDecommissioned put a real settings row behind the drive-state skips, so
|
|
// Scenario F exercises the production guards rather than a stub of them.
|
|
func (h *admissionHarness) markDisconnected(app string) {
|
|
h.driveState(app, true, false)
|
|
}
|
|
|
|
func (h *admissionHarness) markDecommissioned(app string) {
|
|
h.driveState(app, false, true)
|
|
}
|
|
|
|
func (h *admissionHarness) driveState(app string, disconnected, decommissioned bool) {
|
|
if h.m.settings == nil {
|
|
sett, err := settings.Load(filepath.Join(h.dir, "settings.json"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
h.m.settings = sett
|
|
}
|
|
// Each such app gets its OWN drive path, or marking one would skip them all.
|
|
p := filepath.Join(h.dir, "drives", app)
|
|
if err := os.MkdirAll(p, 0o755); err != nil {
|
|
panic(err)
|
|
}
|
|
h.prov.hdd[app] = p
|
|
if err := h.m.settings.AddStoragePath(settings.StoragePath{Path: p, Label: app}); err != nil {
|
|
panic(err)
|
|
}
|
|
if disconnected {
|
|
if err := h.m.settings.SetDisconnected(p, true, nil); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
if decommissioned {
|
|
if err := h.m.settings.SetDecommissioned(p, ""); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *admissionHarness) nsRoot() string { return filepath.Join(h.dir, "felhom-data") }
|
|
|
|
// setSpace states the filesystem's occupancy as a test INPUT — the whole point of the unitSpaceFn
|
|
// seam, so no test has to manufacture disk pressure on a real disk.
|
|
func (h *admissionHarness) setSpace(app string, usedPct, availGB, totalGB float64) {
|
|
h.usage[app] = &UnitSpace{
|
|
Path: h.dir, UsedPercent: usedPct, AvailGB: availGB,
|
|
TotalGB: totalGB, UsedGB: totalGB * usedPct / 100,
|
|
}
|
|
}
|
|
|
|
// seedUnit writes a previous recovery unit for an app: a manifest, a captured app.yaml, a DB dump and
|
|
// a volume tar of the given size. The tar is SPARSE (Truncate), so a 2 GiB "previous backup" costs no
|
|
// disk — the estimator reads st_size, which is what the next write will actually cost.
|
|
func (h *admissionHarness) seedUnit(t *testing.T, app string, tarBytes int64) {
|
|
t.Helper()
|
|
ns := h.nsRoot()
|
|
for _, d := range []string{
|
|
RecoveryUnitComposePath(ns, app),
|
|
AppDBDumpPath(ns, app),
|
|
AppVolumeDumpPath(ns, app),
|
|
} {
|
|
if err := os.MkdirAll(d, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
write := func(p string, b []byte, mode os.FileMode) {
|
|
if err := os.WriteFile(p, b, mode); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
write(RecoveryUnitManifestPath(ns, app), []byte(`{"app_name":"`+app+`","created_at":"2026-08-02T00:00:00Z"}`), 0o644)
|
|
write(filepath.Join(RecoveryUnitComposePath(ns, app), "app.yaml"), []byte("deployed: true\nenv:\n A: previous-good-value\n"), 0o600)
|
|
write(filepath.Join(AppDBDumpPath(ns, app), app+"-postgres.sql"), []byte("-- previous good dump\n"), 0o644)
|
|
|
|
tar := filepath.Join(AppVolumeDumpPath(ns, app), app+"_data.tar")
|
|
f, err := os.Create(tar)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.WriteString("PREVIOUS GOOD TAR"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if tarBytes > 0 {
|
|
if err := f.Truncate(tarBytes); err != nil { // sparse — st_size is the estimate, blocks are not spent
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// runOneBackupRun performs exactly the sequence runDBDumpsInternal performs for the two legs that can
|
|
// be driven without Docker: the admission scope is opened, the volume leg runs, then the capture leg.
|
|
// The DB leg's wiring is pinned structurally by TestAdmission_IsWiredIntoEveryProductionWriteLeg,
|
|
// because DiscoverDatabases shells out to `docker` and cannot honestly run here.
|
|
func (h *admissionHarness) runOneBackupRun() {
|
|
done := h.m.beginAdmissionRun()
|
|
defer done()
|
|
h.m.runVolumeDumps()
|
|
h.m.captureAllRecoveryUnits()
|
|
}
|
|
|
|
// ── The instrument: a checksum of the whole backup tree ──────────────────────────────────────────
|
|
|
|
// treeFingerprint walks every file under backups/primary and returns "relpath mode sha256" lines,
|
|
// sorted. It is the ONLY honest way to check the refusal's claim: it detects a rewritten payload, an
|
|
// added file and a deleted one alike, which a log line and an exit code both fail to do.
|
|
func treeFingerprint(t *testing.T, root string) string {
|
|
t.Helper()
|
|
var lines []string
|
|
err := filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if fi.IsDir() {
|
|
return nil
|
|
}
|
|
f, err := os.Open(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
sum := sha256.New()
|
|
if _, err := io.Copy(sum, f); err != nil {
|
|
return err
|
|
}
|
|
rel, _ := filepath.Rel(root, p)
|
|
lines = append(lines, fmt.Sprintf("%s %o %d %s", rel, fi.Mode().Perm(), fi.Size(), hex.EncodeToString(sum.Sum(nil))))
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("fingerprinting %s: %v", root, err)
|
|
}
|
|
sort.Strings(lines)
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
// treeStatFingerprint is the instrument for trees holding a multi-GiB fixture, where hashing every
|
|
// byte costs more than it proves: name + mode + SIZE. It still catches the act being tested — the
|
|
// volume leg replacing a 2 GiB tar with a freshly written one — because that changes the size, and it
|
|
// catches an added or deleted file by name. Content-identical-but-different-bytes is the one thing it
|
|
// cannot see, which is why the small-tree tests use treeFingerprint instead.
|
|
func treeStatFingerprint(t *testing.T, root string) string {
|
|
t.Helper()
|
|
var lines []string
|
|
_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
|
|
if err != nil || fi.IsDir() {
|
|
return nil
|
|
}
|
|
rel, _ := filepath.Rel(root, p)
|
|
lines = append(lines, fmt.Sprintf("%s %o %d", rel, fi.Mode().Perm(), fi.Size()))
|
|
return nil
|
|
})
|
|
sort.Strings(lines)
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
// treeFileList is the weaker instrument used for Scenario F: names only, so the assertion is
|
|
// specifically about DELETION and cannot be satisfied or broken by a content change.
|
|
func treeFileList(t *testing.T, root string) []string {
|
|
t.Helper()
|
|
var names []string
|
|
_ = filepath.Walk(root, func(p string, fi os.FileInfo, err error) error {
|
|
if err != nil || fi.IsDir() {
|
|
return nil
|
|
}
|
|
rel, _ := filepath.Rel(root, p)
|
|
names = append(names, rel)
|
|
return nil
|
|
})
|
|
sort.Strings(names)
|
|
return names
|
|
}
|
|
|
|
func (h *admissionHarness) primaryRoot() string {
|
|
return PrimaryBackupPath(h.nsRoot())
|
|
}
|
|
|
|
// ── Scenario A — one decision, taken before the first byte ───────────────────────────────────────
|
|
|
|
func TestAdmission_RefusedAppWritesNothingAndIsNotStopped(t *testing.T) {
|
|
h := newAdmissionHarness(t, "privatebin", "opengist", "homebox")
|
|
h.setSpace("privatebin", 40, 60, 100)
|
|
h.setSpace("opengist", 98, 0.4, 70) // below the reserve on BOTH terms
|
|
h.setSpace("homebox", 40, 60, 100)
|
|
h.seedUnit(t, "opengist", 0)
|
|
|
|
// Scoped to the REFUSED app's own unit: its two siblings are admitted and legitimately write
|
|
// theirs, so a whole-tree fingerprint would change for the right reason and prove nothing here.
|
|
// Scenario F below takes the whole-tree view, where every app is refused.
|
|
refusedUnit := RecoveryUnitPath(h.nsRoot(), "opengist")
|
|
before := treeFingerprint(t, refusedUnit)
|
|
if before == "" {
|
|
t.Fatal("the fixture seeded no previous unit, so 'byte-identical' would be vacuously true")
|
|
}
|
|
h.runOneBackupRun()
|
|
after := treeFingerprint(t, refusedUnit)
|
|
|
|
// 1. NOT ONE of the three legs ran for the refused app.
|
|
for _, got := range h.volDumped {
|
|
if got == "opengist" {
|
|
t.Fatal("the VOLUME leg ran for a refused app — this is the R-181 defect exactly: the leg " +
|
|
"that writes the bulk was never gated, so the reserve it protects was consumed by the " +
|
|
"very step it exists to bound")
|
|
}
|
|
}
|
|
for _, got := range h.prov.infoHits {
|
|
if got == "opengist" {
|
|
t.Fatal("the CAPTURE leg was attempted for a refused app — the verdict must be taken before " +
|
|
"any write is prepared, not partway through one")
|
|
}
|
|
}
|
|
|
|
// 2. The tree is byte-identical. This is the assertion the broken code could not pass.
|
|
if after != before {
|
|
t.Fatalf("the backup tree CHANGED across a refusal.\n--- before ---\n%s\n--- after ---\n%s\n"+
|
|
"A refusal that has already rewritten the payload is the defect, not the fix", before, after)
|
|
}
|
|
|
|
// 3. The app was never stopped. DumpAppVolumesSafe stops the stack as its FIRST act, so a gate
|
|
// placed inside it would bounce the app it is refusing to back up.
|
|
for _, got := range h.prov.stopped {
|
|
if got == "opengist" {
|
|
t.Fatal("the refused app was STOPPED — the reserve check has drifted behind the stop")
|
|
}
|
|
}
|
|
|
|
// 4. Exactly ONE alert, for that app, carrying the space figures. Three legs must not mean three
|
|
// emails about one disk.
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d alerts, want exactly 1 (one app refused, three legs): %+v", len(h.events), h.events)
|
|
}
|
|
if h.events[0].app != "opengist" {
|
|
t.Fatalf("alert names %q, want opengist", h.events[0].app)
|
|
}
|
|
if h.events[0].usage == nil || h.events[0].usage.AvailGB != 0.4 {
|
|
t.Fatalf("the alert carries no/incorrect space figures: %+v", h.events[0].usage)
|
|
}
|
|
}
|
|
|
|
// ── Scenario B — the other apps are unaffected ───────────────────────────────────────────────────
|
|
|
|
func TestAdmission_SiblingAppsProceedAndOnlyTheRefusedOneAlerts(t *testing.T) {
|
|
h := newAdmissionHarness(t, "privatebin", "opengist", "homebox")
|
|
h.setSpace("privatebin", 40, 60, 100)
|
|
h.setSpace("opengist", 99, 0.2, 70)
|
|
h.setSpace("homebox", 40, 60, 100)
|
|
|
|
h.runOneBackupRun()
|
|
|
|
for _, app := range []string{"privatebin", "homebox"} {
|
|
if !hasStr(h.volDumped, app) {
|
|
t.Errorf("%s was not volume-dumped (dumped=%v) — one app's refusal silenced its siblings", app, h.volDumped)
|
|
}
|
|
if !hasStr(h.prov.infoHits, app) {
|
|
t.Errorf("%s was not captured (attempted=%v) — the loop did not continue past the refusal", app, h.prov.infoHits)
|
|
}
|
|
if _, err := os.Stat(RecoveryUnitManifestPath(h.nsRoot(), app)); err != nil {
|
|
t.Errorf("%s has no manifest after the run: %v — an admitted app must be backed up normally", app, err)
|
|
}
|
|
}
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d alerts, want exactly 1: %+v", len(h.events), h.events)
|
|
}
|
|
}
|
|
|
|
// ── Scenario C — the promise is true ─────────────────────────────────────────────────────────────
|
|
|
|
// Every claim the shipped message makes is checked against the tree it describes. The wording is NOT
|
|
// weakened to fit the behaviour; the behaviour was moved so the wording became true (§8.3).
|
|
func TestAdmission_EveryClaimInTheRefusalMessageHoldsAgainstTheTree(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
h.setSpace("opengist", 98, 0.5, 70)
|
|
h.seedUnit(t, "opengist", 0)
|
|
|
|
beforeFP := treeFingerprint(t, h.primaryRoot())
|
|
beforeList := treeFileList(t, h.primaryRoot())
|
|
h.runOneBackupRun()
|
|
msg := h.logs.String()
|
|
|
|
if !strings.Contains(msg, "REFUSED for opengist") {
|
|
t.Fatalf("no refusal was logged for opengist; log was:\n%s", msg)
|
|
}
|
|
|
|
// Claim 1: "NO database dump, NO volume dump and NO recovery-unit capture was written for it".
|
|
for _, claim := range []string{"NO database dump", "NO volume dump", "NO recovery-unit capture"} {
|
|
if !strings.Contains(msg, claim) {
|
|
t.Fatalf("the message no longer claims %q — if a leg cannot be brought under the verdict the "+
|
|
"wording must be narrowed deliberately and the gap named, not dropped silently.\n%s", claim, msg)
|
|
}
|
|
}
|
|
if len(h.volDumped) != 0 || len(h.prov.infoHits) != 0 {
|
|
t.Fatalf("the message claims no leg ran, but volume=%v capture=%v", h.volDumped, h.prov.infoHits)
|
|
}
|
|
|
|
// Claim 2: "the previous unit is untouched" — the claim that was MEASURED FALSE in R-181.
|
|
if !strings.Contains(msg, "the previous unit is untouched") {
|
|
t.Fatalf("the message dropped the untouched claim: %s", msg)
|
|
}
|
|
if got := treeFingerprint(t, h.primaryRoot()); got != beforeFP {
|
|
t.Fatalf("the message says the previous unit is untouched; the tree says otherwise.\n"+
|
|
"--- before ---\n%s\n--- after ---\n%s", beforeFP, got)
|
|
}
|
|
|
|
// Claim 3: "NOTHING was deleted".
|
|
if !strings.Contains(msg, "NOTHING was deleted") {
|
|
t.Fatalf("the message dropped the no-deletion claim: %s", msg)
|
|
}
|
|
if got := treeFileList(t, h.primaryRoot()); !equalStrs(got, beforeList) {
|
|
t.Fatalf("files disappeared across a refusal: before=%v after=%v", beforeList, got)
|
|
}
|
|
|
|
// Claim 4: the reason is named, so the operator can tell which term bound.
|
|
if !strings.Contains(msg, "headroom") {
|
|
t.Fatalf("the message does not name WHICH term bound — an operator cannot tell 'the disk is "+
|
|
"full' from 'this app's backup is too big for what is left':\n%s", msg)
|
|
}
|
|
}
|
|
|
|
// ── Scenario D — size-aware, not just headroom-aware ─────────────────────────────────────────────
|
|
|
|
// The live R-181 sequence, reproduced as a unit: the filesystem is ABOVE the reserve on both terms
|
|
// when the run reaches the app, and the app's own write is what crosses it. Under B2 this app was
|
|
// admitted at 96% and then allowed to write 2 GB.
|
|
func TestAdmission_SizeTermRefusesAnAppWhoseOwnWriteWouldCrossTheReserve(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
// 96% used of 70 GiB, 3.0 GiB free — BOTH reserve terms deliberately still clear (97% / 1.0 GiB),
|
|
// exactly as on demo-hp at 06:40:03, so a headroom-only rule starts the run.
|
|
h.setSpace("opengist", 96, 3.0, 70)
|
|
if _, r := h.m.floorVerdict(h.usage["opengist"], 0); r != floorAdmit {
|
|
t.Fatalf("fixture is wrong: the headroom term already refuses (%v), so this test would pass "+
|
|
"without a size term and prove nothing", r)
|
|
}
|
|
h.seedUnit(t, "opengist", 2<<30) // its last backup was 2 GiB — the figure measured live
|
|
|
|
before := treeStatFingerprint(t, h.primaryRoot())
|
|
h.runOneBackupRun()
|
|
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d alerts, want 1 — the app was admitted at 96%% and would have been allowed to "+
|
|
"write 2 GiB, which is the R-181 sequence: %+v", len(h.events), h.events)
|
|
}
|
|
if !strings.Contains(h.logs.String(), "(size)") {
|
|
t.Fatalf("the refusal was not attributed to the SIZE term:\n%s", h.logs.String())
|
|
}
|
|
if !strings.Contains(h.events[0].err, "last backup was 2.0 GB") {
|
|
t.Fatalf("the alert does not carry the estimate that produced the refusal: %q", h.events[0].err)
|
|
}
|
|
if len(h.volDumped) != 0 {
|
|
t.Fatalf("the volume leg ran anyway: %v", h.volDumped)
|
|
}
|
|
if got := treeStatFingerprint(t, h.primaryRoot()); got != before {
|
|
t.Fatalf("the tree changed despite the size-term refusal.\nbefore=%s\nafter =%s", before, got)
|
|
}
|
|
}
|
|
|
|
// ── Scenario E — a first-ever backup is not blocked by having no history ─────────────────────────
|
|
|
|
func TestAdmission_FirstEverBackupIsAdmitted(t *testing.T) {
|
|
h := newAdmissionHarness(t, "brandnew")
|
|
h.setSpace("brandnew", 40, 600, 1000) // ample room, and NO previous unit on disk
|
|
|
|
if est, ok := h.m.estimatedWriteBytes("brandnew"); ok || est != 0 {
|
|
t.Fatalf("estimatedWriteBytes = (%v, %v) for an app with no history, want (0, false)", est, ok)
|
|
}
|
|
h.runOneBackupRun()
|
|
|
|
if len(h.events) != 0 {
|
|
t.Fatalf("a brand-new app was refused: %+v — refusing every app that has no size to estimate "+
|
|
"from would make the FIRST backup the one that can never happen", h.events)
|
|
}
|
|
if !hasStr(h.volDumped, "brandnew") || !hasStr(h.prov.infoHits, "brandnew") {
|
|
t.Fatalf("the app was not backed up (volume=%v capture=%v)", h.volDumped, h.prov.infoHits)
|
|
}
|
|
}
|
|
|
|
// ── Scenario F — the reserve still never deletes ─────────────────────────────────────────────────
|
|
|
|
func TestAdmission_NothingUnderBackupsIsEverRemoved(t *testing.T) {
|
|
h := newAdmissionHarness(t, "privatebin", "opengist", "homebox")
|
|
for _, app := range []string{"privatebin", "opengist", "homebox"} {
|
|
h.setSpace(app, 99, 0.1, 70) // every app refused — maximum pressure to "make room"
|
|
h.seedUnit(t, app, 0)
|
|
}
|
|
|
|
before := treeFileList(t, h.primaryRoot())
|
|
h.runOneBackupRun()
|
|
after := treeFileList(t, h.primaryRoot())
|
|
|
|
if !equalStrs(before, after) {
|
|
t.Fatalf("the file list changed under the reserve.\nbefore=%v\nafter =%v\n"+
|
|
"Nothing here is generational — a unit is ONE fixed path per app — so 'prune the oldest' "+
|
|
"could only mean destroying a DIFFERENT app's only local recovery unit", before, after)
|
|
}
|
|
if len(before) == 0 {
|
|
t.Fatal("the fixture seeded no files, so this test would pass against code that deleted everything")
|
|
}
|
|
}
|
|
|
|
// ── §8.1 — one verdict per app per run, and it resets between runs ───────────────────────────────
|
|
|
|
// The verdict must not be re-taken between an app's own legs. Re-deciding is how the split this fixes
|
|
// came about: DB leg admitted, volume leg admitted, capture refused — with the bulk already written.
|
|
func TestAdmission_VerdictIsTakenOncePerAppPerRunAndNotRedecidedBetweenLegs(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
reads := 0
|
|
h.m.unitSpaceFn = func(string) *UnitSpace {
|
|
reads++
|
|
if reads == 1 {
|
|
return &UnitSpace{Path: h.dir, UsedPercent: 99, AvailGB: 0.1, TotalGB: 70, UsedGB: 69.3}
|
|
}
|
|
// The disk "recovers" mid-run. A re-decided verdict would admit the capture leg here — which
|
|
// is precisely the split R-181 closes, arriving from the other direction.
|
|
return &UnitSpace{Path: h.dir, UsedPercent: 10, AvailGB: 60, TotalGB: 70, UsedGB: 7}
|
|
}
|
|
|
|
h.runOneBackupRun()
|
|
|
|
if reads != 1 {
|
|
t.Fatalf("the filesystem was read %d times for ONE app in ONE run — the verdict is being "+
|
|
"re-decided between legs, which reintroduces the split (bulk written, capture refused)", reads)
|
|
}
|
|
if len(h.prov.infoHits) != 0 {
|
|
t.Fatal("the capture leg ran after the app was refused earlier in the same run")
|
|
}
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("got %d alerts, want exactly 1 per app per run: %+v", len(h.events), h.events)
|
|
}
|
|
}
|
|
|
|
// A set carried between runs is a wrong answer with a confident face: tonight's question answered
|
|
// with last night's disk.
|
|
func TestAdmission_TheRememberedSetResetsBetweenRuns(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
h.setSpace("opengist", 99, 0.1, 70)
|
|
h.runOneBackupRun()
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("run 1: want 1 alert, got %+v", h.events)
|
|
}
|
|
|
|
h.setSpace("opengist", 20, 55, 70) // space freed between runs
|
|
h.runOneBackupRun()
|
|
|
|
if !hasStr(h.volDumped, "opengist") {
|
|
t.Fatal("the second run still refused the app — the previous run's verdict was carried over, " +
|
|
"so freeing space could never take effect")
|
|
}
|
|
if len(h.events) != 1 {
|
|
t.Fatalf("the second (admitted) run alerted again: %+v", h.events)
|
|
}
|
|
}
|
|
|
|
// ── §8.4 — a nil reading neither refuses nor warns, across ALL THREE legs ────────────────────────
|
|
|
|
// Unchanged behaviour, re-pinned because the decision now governs three legs instead of one: an
|
|
// unreadable filesystem must not silently stop an app being backed up at all.
|
|
func TestAdmission_UnreadableFilesystemAdmitsEveryLegAndDoesNotWarn(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
// No usage entry → the reader returns nil, which is what system.GetDiskUsage does on error.
|
|
|
|
h.runOneBackupRun()
|
|
|
|
if len(h.events) != 0 {
|
|
t.Fatalf("an unreadable filesystem produced %d alert(s): %+v — that is the drive gate's "+
|
|
"business and has its own alert", len(h.events), h.events)
|
|
}
|
|
if !hasStr(h.volDumped, "opengist") {
|
|
t.Fatal("the VOLUME leg was refused on an unreadable read — a drive that merely blipped would " +
|
|
"now stop the bulk of the backup, not just the capture")
|
|
}
|
|
if !hasStr(h.prov.infoHits, "opengist") {
|
|
t.Fatal("the CAPTURE leg was refused on an unreadable read")
|
|
}
|
|
}
|
|
|
|
// ── The estimator, through the production path (no seam) ─────────────────────────────────────────
|
|
|
|
func TestEstimatedWriteBytes_SumsTheAppsPreviousDumpsFromRealFiles(t *testing.T) {
|
|
h := newAdmissionHarness(t, "opengist")
|
|
h.seedUnit(t, "opengist", 3<<30) // 3 GiB sparse tar + a small .sql
|
|
|
|
est, ok := h.m.estimatedWriteBytes("opengist")
|
|
if !ok {
|
|
t.Fatal("history on disk was not recognised as history")
|
|
}
|
|
if est < 3<<30 || est > (3<<30)+4096 {
|
|
t.Fatalf("estimate = %d B, want ~%d (the .tar plus the small .sql)", est, int64(3)<<30)
|
|
}
|
|
|
|
// An app whose unit exists but holds no dumps yet is history-LESS, not a zero-byte estimate.
|
|
other := AppVolumeDumpPath(h.nsRoot(), "empty")
|
|
if err := os.MkdirAll(other, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if est, ok := h.m.estimatedWriteBytes("empty"); ok || est != 0 {
|
|
t.Fatalf("an empty unit reported history (%v, %v) — an absent dump is not a 0-byte one", est, ok)
|
|
}
|
|
}
|
|
|
|
// ── The seam is WIRED — walked as an AST, not grepped ────────────────────────────────────────────
|
|
|
|
// FOUR mechanisms in this project have been built and left disconnected (REUSE.md's seam register).
|
|
// The behavioural tests above drive the two legs that can run without Docker; the DB leg cannot, so
|
|
// its gate is pinned HERE, structurally. `strings.Contains` is deliberately not used: a commented-out
|
|
// call still contains the string, and so does a call inside dead code.
|
|
func TestAdmission_IsWiredIntoEveryProductionWriteLeg(t *testing.T) {
|
|
fset := token.NewFileSet()
|
|
file, err := parser.ParseFile(fset, "backup.go", nil, 0) // comments dropped — only real calls survive
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
calls := map[string][]string{} // enclosing func → called names, in source order
|
|
var current string
|
|
ast.Inspect(file, func(n ast.Node) bool {
|
|
switch v := n.(type) {
|
|
case *ast.FuncDecl:
|
|
current = v.Name.Name
|
|
case *ast.CallExpr:
|
|
name := ""
|
|
switch fn := v.Fun.(type) {
|
|
case *ast.Ident:
|
|
name = fn.Name
|
|
case *ast.SelectorExpr:
|
|
name = fn.Sel.Name
|
|
}
|
|
if name != "" && current != "" {
|
|
calls[current] = append(calls[current], name)
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
|
|
// 1. The run scope is opened by the orchestrator of all three legs.
|
|
if !hasStr(calls["runDBDumpsInternal"], "beginAdmissionRun") {
|
|
t.Fatal("runDBDumpsInternal does not open the admission scope — without it every leg decides " +
|
|
"independently and the per-run memo never exists, which is the pre-R-181 behaviour")
|
|
}
|
|
|
|
// 2. The DB leg consults it BEFORE the dump. Order is the whole point: a gate after the write is
|
|
// the defect, relocated.
|
|
assertGateBefore(t, calls["runDBDumpsInternal"], "admitApp", "DumpOne",
|
|
"the DATABASE leg dumps before consulting the reserve")
|
|
|
|
// 3. The volume leg consults it BEFORE the dump seam — which stops the stack as its first act.
|
|
assertGateBefore(t, calls["runVolumeDumps"], "admitApp", "dump",
|
|
"the VOLUME leg — the one that writes the bulk, and the one B2 never gated — dumps before "+
|
|
"consulting the reserve")
|
|
|
|
// 4. The capture leg, in its own file.
|
|
rfset := token.NewFileSet()
|
|
rfile, err := parser.ParseFile(rfset, "recovery_unit.go", nil, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
capCalls := map[string][]string{}
|
|
current = ""
|
|
ast.Inspect(rfile, func(n ast.Node) bool {
|
|
switch v := n.(type) {
|
|
case *ast.FuncDecl:
|
|
current = v.Name.Name
|
|
case *ast.CallExpr:
|
|
if sel, ok := v.Fun.(*ast.SelectorExpr); ok && current != "" {
|
|
capCalls[current] = append(capCalls[current], sel.Sel.Name)
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
assertGateBefore(t, capCalls["captureAllRecoveryUnits"], "admitApp", "CaptureRecoveryUnit",
|
|
"the CAPTURE leg captures before consulting the reserve")
|
|
}
|
|
|
|
// assertGateBefore checks that `gate` appears in the call list before `act`.
|
|
func assertGateBefore(t *testing.T, calls []string, gate, act, why string) {
|
|
t.Helper()
|
|
gi, ai := -1, -1
|
|
for i, c := range calls {
|
|
if c == gate && gi < 0 {
|
|
gi = i
|
|
}
|
|
if c == act && ai < 0 {
|
|
ai = i
|
|
}
|
|
}
|
|
if gi < 0 {
|
|
t.Fatalf("%s: %q is never called there at all (calls=%v)", why, gate, calls)
|
|
}
|
|
if ai < 0 {
|
|
t.Fatalf("fixture drift: %q is no longer called in that function (calls=%v) — this test can no "+
|
|
"longer see the act it is ordering the gate against", act, calls)
|
|
}
|
|
if gi > ai {
|
|
t.Fatalf("%s: %q first appears at %d, after %q at %d", why, gate, gi, act, ai)
|
|
}
|
|
}
|
|
|
|
func hasStr(hay []string, needle string) bool {
|
|
for _, s := range hay {
|
|
if s == needle {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func equalStrs(a, b []string) bool {
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
for i := range a {
|
|
if a[i] != b[i] {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|