package backup import ( "crypto/sha256" "encoding/hex" "errors" "io" "log" "os" "path/filepath" "strings" "testing" "gitea.dooplex.hu/admin/felhom-controller/internal/fillwatch" ) // R-165 / decision B2 — the capture floor that replaces the `mp1` bulkhead. // // Before the merge, the 20 G backup partition kept a runaway capture from reaching // `/var/lib/docker`, because it was a different filesystem. After the merge it is the same one, and a // full Docker data-root is a stopped box. These pin the replacement. // floorProvider lists stacks and always resolves recovery info — the floor must refuse BEFORE any of // that is consulted, so a capture that gets as far as GetStackRecoveryInfo has already lost. type floorProvider struct { stacks []string dir string infoHits []string // records every app whose recovery info was read = a capture that was ATTEMPTED } func (p *floorProvider) GetStackComposePath(string) (string, bool) { return "", false } func (p *floorProvider) ListDeployedStacks() []StackSummary { out := make([]StackSummary, 0, len(p.stacks)) for _, s := range p.stacks { out = append(out, StackSummary{Name: s}) } return out } func (p *floorProvider) GetStackHDDMounts(string) []string { return nil } func (p *floorProvider) GetStackHDDPath(string) string { return "" } func (p *floorProvider) GetImportRoot() string { return "" } func (p *floorProvider) GetDockerVolumes(string) []string { return nil } func (p *floorProvider) StopStack(string) error { return nil } func (p *floorProvider) StartStack(string) error { return nil } func (p *floorProvider) RefreshAndIsRunning(string) bool { return true } func (p *floorProvider) GetStackRecoveryInfo(name string) (RecoveryInfo, bool) { p.infoHits = append(p.infoHits, name) return RecoveryInfo{StackDir: filepath.Join(p.dir, "stacks", name)}, true } func (p *floorProvider) RecoverStackSecrets(string, []string) map[string]string { return nil } func (p *floorProvider) RecreateStackDefinitionFromUnit(string, string, map[string]string) error { return nil } func (p *floorProvider) StartStackServices(string, []string) error { return nil } func (p *floorProvider) GetStackClassifiedBinds(string) ([]ClassifiedBind, bool) { return nil, false } type floorHarness struct { m *Manager prov *floorProvider events []unitEvent usage map[string]*UnitSpace dir string } // newFloorHarness injects the usage read, so the filesystem's occupancy is a test input rather than // something the test has to manufacture on a real disk. func newFloorHarness(t *testing.T, stacks ...string) *floorHarness { t.Helper() dir := t.TempDir() h := &floorHarness{ prov: &floorProvider{stacks: stacks, dir: dir}, usage: map[string]*UnitSpace{}, dir: dir, } h.m = &Manager{ logger: log.New(io.Discard, "", 0), systemDataPath: dir, stackProvider: h.prov, unitSpaceFn: func(name string) *UnitSpace { return h.usage[name] }, } h.m.SetUnitNotify(func(name string, err error, u *UnitSpace) { h.events = append(h.events, unitEvent{app: name, err: err.Error(), usage: u}) }) return h } func (h *floorHarness) setSpace(app string, usedPct, availGB float64) { h.usage[app] = &UnitSpace{ Path: h.dir, UsedPercent: usedPct, AvailGB: availGB, TotalGB: 100, UsedGB: usedPct, } } // --- Scenario D — the floor refuses, per app, and says so ---------------------------------------- func TestFloor_RefusesTheAppAndLeavesItsPreviousUnitByteIdentical(t *testing.T) { h := newFloorHarness(t, "homebox", "immich", "nextcloud") h.setSpace("homebox", 40, 60) h.setSpace("immich", 98, 0.4) // below the floor on BOTH terms h.setSpace("nextcloud", 40, 60) // A previous unit exists for the app about to be refused. Checksum it before and after. unitDir := filepath.Join(h.dir, "felhom-data", "backups", "primary", "immich", "compose") if err := os.MkdirAll(unitDir, 0o755); err != nil { t.Fatal(err) } prev := filepath.Join(unitDir, "app.yaml") if err := os.WriteFile(prev, []byte("deployed: true\nenv:\n A: previous-good-value\n"), 0o600); err != nil { t.Fatal(err) } before := checksumFile(t, prev) h.m.captureAllRecoveryUnits() // The refused app must NOT have been attempted at all — the floor is checked BEFORE any write. for _, hit := range h.prov.infoHits { if hit == "immich" { t.Fatal("the refused app's recovery info was read — the capture was ATTEMPTED rather than " + "refused up front, so a write could have started and failed partway") } } if after := checksumFile(t, prev); after != before { t.Fatalf("the previous unit changed (%s → %s) — a refused capture must leave the last good "+ "copy byte-identical", before, after) } if _, err := os.Stat(prev); err != nil { t.Fatalf("the previous unit is gone: %v — the floor REFUSES, it never deletes", err) } // Exactly one alert, for the refused app, carrying the space figures. if len(h.events) != 1 { t.Fatalf("got %d alerts, want exactly 1: %+v", len(h.events), h.events) } e := h.events[0] if e.app != "immich" { t.Fatalf("alert names %q, want immich", e.app) } if e.usage == nil || e.usage.AvailGB != 0.4 { t.Fatalf("the alert carries no/incorrect space figures: %+v", e.usage) } if !strings.Contains(e.err, "reserve") { t.Fatalf("the alert message %q does not say it was a reserve refusal — an operator would read "+ "it as a broken capture rather than a deliberate hold", e.err) } // The other two must have been captured normally — one app's refusal must not silence its siblings. got := strings.Join(h.prov.infoHits, ",") if !strings.Contains(got, "homebox") || !strings.Contains(got, "nextcloud") { t.Fatalf("attempted=%v — the loop did not continue past the refusal", h.prov.infoHits) } } // Nothing may be deleted to make room, under any threshold. Nothing on this filesystem is // generational, so "the oldest" is always a DIFFERENT app's only local copy. func TestFloor_NeverDeletesAnotherAppsUnit(t *testing.T) { h := newFloorHarness(t, "immich", "nextcloud") h.setSpace("immich", 99, 0.1) h.setSpace("nextcloud", 99, 0.1) other := filepath.Join(h.dir, "felhom-data", "backups", "primary", "nextcloud") if err := os.MkdirAll(other, 0o755); err != nil { t.Fatal(err) } keep := filepath.Join(other, "manifest.json") if err := os.WriteFile(keep, []byte(`{"app_name":"nextcloud"}`), 0o644); err != nil { t.Fatal(err) } before := checksumFile(t, keep) h.m.captureAllRecoveryUnits() if _, err := os.Stat(keep); err != nil { t.Fatalf("another app's unit was DELETED to make room: %v — nothing here is generational, so "+ "pruning could only destroy an app's only local copy", err) } if after := checksumFile(t, keep); after != before { t.Fatal("another app's unit was modified while the filesystem was under the floor") } } // --- Scenario E — the floor is not a wall by another name ---------------------------------------- // The floor is about the FILESYSTEM's remaining headroom, never the unit's size. A per-unit cap would // be R-163 rebuilt inside one volume. func TestFloor_LargeUnitWithAmpleSpaceIsCaptured(t *testing.T) { h := newFloorHarness(t, "immich") // A huge app on a huge, mostly-empty filesystem: 40% used, 600 GB free. h.usage["immich"] = &UnitSpace{Path: h.dir, UsedPercent: 40, AvailGB: 600, TotalGB: 1000, UsedGB: 400} h.m.captureAllRecoveryUnits() if len(h.events) != 0 { t.Fatalf("a capture was refused on a filesystem with 600 GB free (%+v) — the floor has become "+ "a per-unit size cap, which is exactly the ceiling R-165 removed", h.events) } if len(h.prov.infoHits) != 1 || h.prov.infoHits[0] != "immich" { t.Fatalf("attempted=%v, want [immich] — the capture was not even tried", h.prov.infoHits) } } // The old 20 G ceiling must not survive anywhere: a unit far larger than the retired partition is // captured when the filesystem has room. func TestFloor_TheOld20GCeilingIsGone(t *testing.T) { h := newFloorHarness(t, "immich") // 180 GB free, and the app's own data is 120 GB — SIX TIMES the retired 20 G area. The figure is // deliberately far above 20 so that a literal `UsedGB > 20` cap cannot survive this test: a // fixture sitting exactly on the old boundary would pass under the very shape it forbids. h.usage["immich"] = &UnitSpace{Path: h.dir, UsedPercent: 40, AvailGB: 180, TotalGB: 300, UsedGB: 120} h.m.captureAllRecoveryUnits() if len(h.events) != 0 { t.Fatalf("refused with 180 GB free: %+v — a fixed per-area limit survives somewhere", h.events) } } // --- Group E — the floor sits BELOW the critical warning band ------------------------------------- // A floor that fires before its own warning is a silent failure wearing a threshold: the customer // would get a refusal with no prior notice that anything was wrong. The customer's `disk_critical` // must always come first. func TestFloorSitsBelowTheCriticalWarningBand(t *testing.T) { if FloorUsedPercent <= fillwatch.CritUsedPercent { t.Fatalf("FloorUsedPercent (%.1f) must be strictly ABOVE fillwatch.CritUsedPercent (%.1f) — "+ "otherwise a capture can be refused before the customer was ever warned that the disk was "+ "filling, which is a silent failure wearing a threshold", FloorUsedPercent, fillwatch.CritUsedPercent) } if FloorFreeGiB >= fillwatch.CritFreeGiB { t.Fatalf("FloorFreeGiB (%.1f) must be strictly BELOW fillwatch.CritFreeGiB (%.1f) — the "+ "free-byte term needs the same ordering as the percentage term, or the free-byte path "+ "refuses before it warns", FloorFreeGiB, fillwatch.CritFreeGiB) } // And below the WARNING band too, transitively — stated explicitly so the chain is readable. if FloorUsedPercent <= fillwatch.WarnUsedPercent || FloorFreeGiB >= fillwatch.WarnFreeGiB { t.Fatal("the floor is not beyond the warning band — the customer must be warned, then warned " + "critically, and only then can a capture be refused") } // Both terms must be able to refuse INDEPENDENTLY — that is why there are two. estGiB=0 is the // history-less case, which exercises the headroom term alone. if _, r := (&Manager{}).floorVerdict(&UnitSpace{UsedPercent: 50, AvailGB: 0.5}, 0); r != floorHeadroom { t.Fatal("a filesystem with 0.5 GiB free at only 50% used was NOT refused — the free-byte term " + "does not trip on its own, so a very large volume can run out without the floor engaging") } if _, r := (&Manager{}).floorVerdict(&UnitSpace{UsedPercent: 98, AvailGB: 40}, 0); r != floorHeadroom { t.Fatal("a filesystem 98% used was NOT refused — the percentage term does not trip on its own") } if _, r := (&Manager{}).floorVerdict(&UnitSpace{UsedPercent: 50, AvailGB: 50}, 0); r != floorAdmit { t.Fatal("a healthy filesystem was refused") } } // --- §8.4 — a nil usage read neither refuses nor warns -------------------------------------------- func TestFloor_UnreadableFilesystemNeitherRefusesNorWarns(t *testing.T) { h := newFloorHarness(t, "immich") // No entry → the injected reader returns nil, which is what system.GetDiskUsage does on error. h.m.captureAllRecoveryUnits() if len(h.events) != 0 { t.Fatalf("an UNREADABLE filesystem produced %d alert(s): %+v — an absent, unmounted or "+ "unreadable filesystem is the drive gate's business and already has its own alert; "+ "refusing here would block every capture on a box whose drive merely blipped", len(h.events), h.events) } if len(h.prov.infoHits) != 1 { t.Fatalf("the capture was not attempted on an unreadable read (attempted=%v) — a nil reading "+ "must not refuse", h.prov.infoHits) } } // ErrCaptureFloor must be matchable, so a caller can tell a deliberate refusal from a broken capture. func TestErrCaptureFloor_IsMatchable(t *testing.T) { h := newFloorHarness(t, "immich") h.setSpace("immich", 99, 0.2) h.m.captureAllRecoveryUnits() if len(h.events) != 1 { t.Fatalf("want 1 event, got %d", len(h.events)) } // The seam hands a string, so assert on the sentinel's own text being present and distinct. if !errors.Is(errWrapForTest(), ErrCaptureFloor) { t.Fatal("ErrCaptureFloor does not survive wrapping") } if !strings.Contains(h.events[0].err, "Refused") && !strings.Contains(h.events[0].err, "refused") { t.Fatalf("the alert %q does not identify itself as a refusal", h.events[0].err) } } func errWrapForTest() error { return errors.Join(ErrCaptureFloor, errors.New("ctx")) } func checksumFile(t *testing.T, path string) string { t.Helper() f, err := os.Open(path) if err != nil { t.Fatal(err) } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { t.Fatal(err) } return hex.EncodeToString(h.Sum(nil)) }