felhom-controller/controller/internal/backup/capture_floor_test.go

package backup

import (
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-controller/internal/fillwatch"
)

// R-165 / decision B2 — the capture floor that replaces the `mp1` bulkhead.
//
// Before the merge, the 20 G backup partition kept a runaway capture from reaching
// `/var/lib/docker`, because it was a different filesystem. After the merge it is the same one, and a
// full Docker data-root is a stopped box. These pin the replacement.

// floorProvider lists stacks and always resolves recovery info — the floor must refuse BEFORE any of
// that is consulted, so a capture that gets as far as GetStackRecoveryInfo has already lost.
type floorProvider struct {
	stacks   []string
	dir      string
	infoHits []string // records every app whose recovery info was read = a capture that was ATTEMPTED
}

func (p *floorProvider) GetStackComposePath(string) (string, bool) { return "", false }
func (p *floorProvider) ListDeployedStacks() []StackSummary {
	out := make([]StackSummary, 0, len(p.stacks))
	for _, s := range p.stacks {
		out = append(out, StackSummary{Name: s})
	}
	return out
}
func (p *floorProvider) GetStackHDDMounts(string) []string { return nil }
func (p *floorProvider) GetStackHDDPath(string) string     { return "" }
func (p *floorProvider) GetImportRoot() string             { return "" }
func (p *floorProvider) GetDockerVolumes(string) []string  { return nil }
func (p *floorProvider) StopStack(string) error            { return nil }
func (p *floorProvider) StartStack(string) error           { return nil }
func (p *floorProvider) RefreshAndIsRunning(string) bool    { return true }
func (p *floorProvider) GetStackRecoveryInfo(name string) (RecoveryInfo, bool) {
	p.infoHits = append(p.infoHits, name)
	return RecoveryInfo{StackDir: filepath.Join(p.dir, "stacks", name)}, true
}
func (p *floorProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
func (p *floorProvider) RecreateStackDefinitionFromUnit(string, string, map[string]string) error {
	return nil
}
func (p *floorProvider) StartStackServices(string, []string) error { return nil }
func (p *floorProvider) GetStackClassifiedBinds(string) ([]ClassifiedBind, bool) {
	return nil, false
}

type floorHarness struct {
	m      *Manager
	prov   *floorProvider
	events []unitEvent
	usage  map[string]*UnitSpace
	dir    string
}

// newFloorHarness injects the usage read, so the filesystem's occupancy is a test input rather than
// something the test has to manufacture on a real disk.
func newFloorHarness(t *testing.T, stacks ...string) *floorHarness {
	t.Helper()
	dir := t.TempDir()
	h := &floorHarness{
		prov:  &floorProvider{stacks: stacks, dir: dir},
		usage: map[string]*UnitSpace{},
		dir:   dir,
	}
	h.m = &Manager{
		logger:         log.New(io.Discard, "", 0),
		systemDataPath: dir,
		stackProvider:  h.prov,
		unitSpaceFn:    func(name string) *UnitSpace { return h.usage[name] },
	}
	h.m.SetUnitNotify(func(name string, err error, u *UnitSpace) {
		h.events = append(h.events, unitEvent{app: name, err: err.Error(), usage: u})
	})
	return h
}

func (h *floorHarness) setSpace(app string, usedPct, availGB float64) {
	h.usage[app] = &UnitSpace{
		Path: h.dir, UsedPercent: usedPct, AvailGB: availGB,
		TotalGB: 100, UsedGB: usedPct,
	}
}

// --- Scenario D — the floor refuses, per app, and says so ----------------------------------------

func TestFloor_RefusesTheAppAndLeavesItsPreviousUnitByteIdentical(t *testing.T) {
	h := newFloorHarness(t, "homebox", "immich", "nextcloud")
	h.setSpace("homebox", 40, 60)
	h.setSpace("immich", 98, 0.4) // below the floor on BOTH terms
	h.setSpace("nextcloud", 40, 60)

	// A previous unit exists for the app about to be refused. Checksum it before and after.
	unitDir := filepath.Join(h.dir, "felhom-data", "backups", "primary", "immich", "compose")
	if err := os.MkdirAll(unitDir, 0o755); err != nil {
		t.Fatal(err)
	}
	prev := filepath.Join(unitDir, "app.yaml")
	if err := os.WriteFile(prev, []byte("deployed: true\nenv:\n  A: previous-good-value\n"), 0o600); err != nil {
		t.Fatal(err)
	}
	before := checksumFile(t, prev)

	h.m.captureAllRecoveryUnits()

	// The refused app must NOT have been attempted at all — the floor is checked BEFORE any write.
	for _, hit := range h.prov.infoHits {
		if hit == "immich" {
			t.Fatal("the refused app's recovery info was read — the capture was ATTEMPTED rather than " +
				"refused up front, so a write could have started and failed partway")
		}
	}

	if after := checksumFile(t, prev); after != before {
		t.Fatalf("the previous unit changed (%s → %s) — a refused capture must leave the last good "+
			"copy byte-identical", before, after)
	}
	if _, err := os.Stat(prev); err != nil {
		t.Fatalf("the previous unit is gone: %v — the floor REFUSES, it never deletes", err)
	}

	// Exactly one alert, for the refused app, carrying the space figures.
	if len(h.events) != 1 {
		t.Fatalf("got %d alerts, want exactly 1: %+v", len(h.events), h.events)
	}
	e := h.events[0]
	if e.app != "immich" {
		t.Fatalf("alert names %q, want immich", e.app)
	}
	if e.usage == nil || e.usage.AvailGB != 0.4 {
		t.Fatalf("the alert carries no/incorrect space figures: %+v", e.usage)
	}
	if !strings.Contains(e.err, "reserve") {
		t.Fatalf("the alert message %q does not say it was a reserve refusal — an operator would read "+
			"it as a broken capture rather than a deliberate hold", e.err)
	}

	// The other two must have been captured normally — one app's refusal must not silence its siblings.
	got := strings.Join(h.prov.infoHits, ",")
	if !strings.Contains(got, "homebox") || !strings.Contains(got, "nextcloud") {
		t.Fatalf("attempted=%v — the loop did not continue past the refusal", h.prov.infoHits)
	}
}

// Nothing may be deleted to make room, under any threshold. Nothing on this filesystem is
// generational, so "the oldest" is always a DIFFERENT app's only local copy.
func TestFloor_NeverDeletesAnotherAppsUnit(t *testing.T) {
	h := newFloorHarness(t, "immich", "nextcloud")
	h.setSpace("immich", 99, 0.1)
	h.setSpace("nextcloud", 99, 0.1)

	other := filepath.Join(h.dir, "felhom-data", "backups", "primary", "nextcloud")
	if err := os.MkdirAll(other, 0o755); err != nil {
		t.Fatal(err)
	}
	keep := filepath.Join(other, "manifest.json")
	if err := os.WriteFile(keep, []byte(`{"app_name":"nextcloud"}`), 0o644); err != nil {
		t.Fatal(err)
	}
	before := checksumFile(t, keep)

	h.m.captureAllRecoveryUnits()

	if _, err := os.Stat(keep); err != nil {
		t.Fatalf("another app's unit was DELETED to make room: %v — nothing here is generational, so "+
			"pruning could only destroy an app's only local copy", err)
	}
	if after := checksumFile(t, keep); after != before {
		t.Fatal("another app's unit was modified while the filesystem was under the floor")
	}
}

// --- Scenario E — the floor is not a wall by another name ----------------------------------------

// The floor is about the FILESYSTEM's remaining headroom, never the unit's size. A per-unit cap would
// be R-163 rebuilt inside one volume.
func TestFloor_LargeUnitWithAmpleSpaceIsCaptured(t *testing.T) {
	h := newFloorHarness(t, "immich")
	// A huge app on a huge, mostly-empty filesystem: 40% used, 600 GB free.
	h.usage["immich"] = &UnitSpace{Path: h.dir, UsedPercent: 40, AvailGB: 600, TotalGB: 1000, UsedGB: 400}

	h.m.captureAllRecoveryUnits()

	if len(h.events) != 0 {
		t.Fatalf("a capture was refused on a filesystem with 600 GB free (%+v) — the floor has become "+
			"a per-unit size cap, which is exactly the ceiling R-165 removed", h.events)
	}
	if len(h.prov.infoHits) != 1 || h.prov.infoHits[0] != "immich" {
		t.Fatalf("attempted=%v, want [immich] — the capture was not even tried", h.prov.infoHits)
	}
}

// The old 20 G ceiling must not survive anywhere: a unit far larger than the retired partition is
// captured when the filesystem has room.
func TestFloor_TheOld20GCeilingIsGone(t *testing.T) {
	h := newFloorHarness(t, "immich")
	// 180 GB free, and the app's own data is 120 GB — SIX TIMES the retired 20 G area. The figure is
	// deliberately far above 20 so that a literal `UsedGB > 20` cap cannot survive this test: a
	// fixture sitting exactly on the old boundary would pass under the very shape it forbids.
	h.usage["immich"] = &UnitSpace{Path: h.dir, UsedPercent: 40, AvailGB: 180, TotalGB: 300, UsedGB: 120}
	h.m.captureAllRecoveryUnits()
	if len(h.events) != 0 {
		t.Fatalf("refused with 180 GB free: %+v — a fixed per-area limit survives somewhere", h.events)
	}
}

// --- Group E — the floor sits BELOW the critical warning band -------------------------------------

// A floor that fires before its own warning is a silent failure wearing a threshold: the customer
// would get a refusal with no prior notice that anything was wrong. The customer's `disk_critical`
// must always come first.
func TestFloorSitsBelowTheCriticalWarningBand(t *testing.T) {
	if FloorUsedPercent <= fillwatch.CritUsedPercent {
		t.Fatalf("FloorUsedPercent (%.1f) must be strictly ABOVE fillwatch.CritUsedPercent (%.1f) — "+
			"otherwise a capture can be refused before the customer was ever warned that the disk was "+
			"filling, which is a silent failure wearing a threshold",
			FloorUsedPercent, fillwatch.CritUsedPercent)
	}
	if FloorFreeGiB >= fillwatch.CritFreeGiB {
		t.Fatalf("FloorFreeGiB (%.1f) must be strictly BELOW fillwatch.CritFreeGiB (%.1f) — the "+
			"free-byte term needs the same ordering as the percentage term, or the free-byte path "+
			"refuses before it warns", FloorFreeGiB, fillwatch.CritFreeGiB)
	}
	// And below the WARNING band too, transitively — stated explicitly so the chain is readable.
	if FloorUsedPercent <= fillwatch.WarnUsedPercent || FloorFreeGiB >= fillwatch.WarnFreeGiB {
		t.Fatal("the floor is not beyond the warning band — the customer must be warned, then warned " +
			"critically, and only then can a capture be refused")
	}

	// Both terms must be able to refuse INDEPENDENTLY — that is why there are two.
	if _, blocked := (&Manager{}).floorVerdict(&UnitSpace{UsedPercent: 50, AvailGB: 0.5}); !blocked {
		t.Fatal("a filesystem with 0.5 GiB free at only 50% used was NOT refused — the free-byte term " +
			"does not trip on its own, so a very large volume can run out without the floor engaging")
	}
	if _, blocked := (&Manager{}).floorVerdict(&UnitSpace{UsedPercent: 98, AvailGB: 40}); !blocked {
		t.Fatal("a filesystem 98% used was NOT refused — the percentage term does not trip on its own")
	}
	if _, blocked := (&Manager{}).floorVerdict(&UnitSpace{UsedPercent: 50, AvailGB: 50}); blocked {
		t.Fatal("a healthy filesystem was refused")
	}
}

// --- §8.4 — a nil usage read neither refuses nor warns --------------------------------------------

func TestFloor_UnreadableFilesystemNeitherRefusesNorWarns(t *testing.T) {
	h := newFloorHarness(t, "immich")
	// No entry → the injected reader returns nil, which is what system.GetDiskUsage does on error.

	h.m.captureAllRecoveryUnits()

	if len(h.events) != 0 {
		t.Fatalf("an UNREADABLE filesystem produced %d alert(s): %+v — an absent, unmounted or "+
			"unreadable filesystem is the drive gate's business and already has its own alert; "+
			"refusing here would block every capture on a box whose drive merely blipped", len(h.events), h.events)
	}
	if len(h.prov.infoHits) != 1 {
		t.Fatalf("the capture was not attempted on an unreadable read (attempted=%v) — a nil reading "+
			"must not refuse", h.prov.infoHits)
	}
}

// ErrCaptureFloor must be matchable, so a caller can tell a deliberate refusal from a broken capture.
func TestErrCaptureFloor_IsMatchable(t *testing.T) {
	h := newFloorHarness(t, "immich")
	h.setSpace("immich", 99, 0.2)
	h.m.captureAllRecoveryUnits()
	if len(h.events) != 1 {
		t.Fatalf("want 1 event, got %d", len(h.events))
	}
	// The seam hands a string, so assert on the sentinel's own text being present and distinct.
	if !errors.Is(errWrapForTest(), ErrCaptureFloor) {
		t.Fatal("ErrCaptureFloor does not survive wrapping")
	}
	if !strings.Contains(h.events[0].err, "Refused") && !strings.Contains(h.events[0].err, "refused") {
		t.Fatalf("the alert %q does not identify itself as a refusal", h.events[0].err)
	}
}

func errWrapForTest() error { return errors.Join(ErrCaptureFloor, errors.New("ctx")) }

func checksumFile(t *testing.T, path string) string {
	t.Helper()
	f, err := os.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		t.Fatal(err)
	}
	return hex.EncodeToString(h.Sum(nil))
}