felhom-agent/internal/localapi/disks_bind_liveness_test.go

package localapi

import (
	"context"
	"io"
	"log/slog"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
)

// R-117 — BoundUnderParent must mean THE BIND ACTUALLY WORKS, not "a mount by that name exists".
//
// THE BUG THESE PIN. GuestSeesMount (intermediary.go) and isHostMountpoint both parse a mountinfo line
// and then test only fields[4], the mount POINT. Field 3 — major:minor — sits in the same parsed slice
// and was discarded. So after a drive is detached and returned, the raw host mount heals onto the NEW
// device via its fs-UUID-keyed unit while the bind still names the OLD one, and BOTH existing terms stay
// true. Measured live: raw on 8:32 /dev/sdc, bind on 8:16 /dev/sdb with `shutdown`, BoundUnderParent
// TRUE, EIO on every read and write, and the controller's gate taking its Return branch — restarting the
// customer's apps onto that namespace and emailing backup_target_restored, with no alarm on any channel
// (felhom.eu audits/SPIKE-r117-bind-liveness-2026-07-30.md §3.3, §5.2).
//
// AND THE HALF THAT EMITS NOTHING AT ALL (spike §9, filed R-117a): a device that fails WITHOUT
// disappearing leaves the raw mount active, the devnos EQUAL, and the drive never Disconnected — so the
// gate produces neither a Stop nor a Return action and nothing is emitted, indefinitely. A devno
// comparison alone reads stale-device=false there, which is why term 3 checks the filesystem's own abort
// flags too. TestDisks_BindLiveness_AbortedFilesystemReadsAbsent is that case; a fix that shipped only
// the devno comparison would pass every other test in this file.
//
// WHY THE FIXTURES ARE REAL. Each mountinfo body below is the captured output of the spike run, not a
// hand-written line. R-116's fix shipped green and inert because its fixture supplied a MountPath
// production never supplies. These tests redirect procSelfMountinfo at a fixture file, so the REAL
// parser (hostMountEntries), the REAL predicate (bindLiveness) and the REAL /disks handler all run —
// the data is injected, the logic is not.
//
// RED-PROOFS (each verified to land, see REPORT.md): dropping `&& s.bindUsable(...)` from either /disks
// construction site fails StaleBindReadsAbsent / UnionPath_StaleBindReadsAbsent and
// AbortedFilesystemReadsAbsent; dropping the abort check so only the device comparison remains fails
// AbortedFilesystemReadsAbsent and AbortedWins_WhenDevnosAgree ALONE — that is the P1-only fix, and it is
// the one worth fearing; dropping `emergency_ro` from abortTokensByFS fails only the emergency_ro subtest;
// and returning BindLive instead of BindUnknown for an unreadable table fails UnknownIsTreatedAsPresent.
//
// A THIRD ordering trap, caught by TestBindLiveness_Verdicts during development and worth naming because
// it reports correctly while breaking the repair: reading the abort flag BEFORE comparing devices
// classifies the real return state as BindAborted, since its stale bind carries `shutdown` as well as a
// different device. BoundUnderParent still reads absent — every test in Group A still passes — but
// AttachDrive then refuses the re-bind that actually repairs it, so the self-heal never runs. The verdict
// must answer "would a re-bind help", which means reading the abort flag of the RAW mount (the re-bind's
// target) in the stale case, and of the bind itself only when the devices already agree.

// ── fixtures, from the spike's captures ─────────────────────────────────────────────────────────
//
// Devices, super options, optional-field tags and root paths are verbatim. ONE substitution: the spike
// ran against a SCRATCH shared parent (/mnt/r117-drives) so it could not disturb the live
// /mnt/felhom-drives peer group, whereas the code under test derives the stable path itself, as
// StableParentDir + "/" + DriveNameFromRaw(raw). So /mnt/r117-drives/sd becomes
// /mnt/felhom-drives/r117sd. Substituting anything else would make these fixtures describe a path
// production never produces — which is precisely how R-116's fix shipped green and inert.

// mountinfoHealthy is the HEALTHY state (spike §3.3): raw and bind on the SAME device, no abort token.
const mountinfoHealthy = `748 33 8:16 / /mnt/r117sd rw,relatime shared:450 - ext4 /dev/sdb rw,stripe=512
755 118 8:16 /felhom-data /mnt/felhom-drives/r117sd rw,relatime shared:450 - ext4 /dev/sdb rw,stripe=512
`

// mountinfoStaleBind is the R-117 RETURN state (spike §3.3 / §5.2): the drive came back as /dev/sdc
// (8:32) and the raw mount healed onto it, while the bind still names /dev/sdb (8:16) and carries
// `shutdown`. Both pre-R-117 terms read true here.
const mountinfoStaleBind = `755 118 8:16 /felhom-data /mnt/felhom-drives/r117sd rw,relatime shared:450 - ext4 /dev/sdb rw,stripe=512,shutdown
814 33 8:32 / /mnt/r117sd rw,relatime shared:485 - ext4 /dev/sdc rw,stripe=512
`

// mountinfoAborted is the STEADY-STATE state (spike §9): the device errored in place, so it NEVER LEFT.
// Raw and bind are the SAME device — the device comparison cannot see this — and ext4 has done an
// emergency remount-ro. Today this state emits nothing on any channel.
//
// SECOND substitution, and it is the one that nearly made this test decoration. The spike produced this
// state on a dm device (dm is the only mechanism that can make a device error WITHOUT disappearing), so
// the capture reads 252:11 /dev/mapper/r117cel. Transposed here onto the USB drive shape, because
// RoleForStorage derives role="system" for a /dev/mapper backing device — and a system-role row never
// enters the block that computes BoundUnderParent, so the field stays false by DEFAULT and the assertion
// below passes without term 3 ever running. It did exactly that until RP1 failed to fail (see REPORT.md).
// An in-place abort on a USB drive is the realistic customer case anyway (a link reset that recovers the
// link after ext4 has already given up); only the super options and the matching devnos carry the claim.
const mountinfoAborted = `748 33 8:16 / /mnt/r117sd rw,relatime shared:450 - ext4 /dev/sdb rw,stripe=512,emergency_ro
755 118 8:16 /felhom-data /mnt/felhom-drives/r117sd rw,relatime shared:450 - ext4 /dev/sdb rw,stripe=512,emergency_ro
`

// mountinfoAbortedShutdown is the same in-place shape with the OTHER ext4 abort token, the one a device
// removal sets. Both were measured; a check for only `shutdown` passes mountinfoAborted and a check for
// only `emergency_ro` passes this — which is why abortTokensByFS carries both, and why RP4 exists.
const mountinfoAbortedShutdown = `748 33 8:16 / /mnt/r117sd rw,relatime shared:450 - ext4 /dev/sdb rw,stripe=512,shutdown
755 118 8:16 /felhom-data /mnt/felhom-drives/r117sd rw,relatime shared:450 - ext4 /dev/sdb rw,stripe=512,shutdown
`

// mountinfoUnknownFS is healthy-looking but on a filesystem whose abort vocabulary we have not measured.
// The honest verdict is UNKNOWN — which must be treated as PRESENT, not as live and not as absent.
const mountinfoUnknownFS = `748 33 8:16 / /mnt/r117sd rw,relatime shared:450 - btrfs /dev/sdb rw
755 118 8:16 /felhom-data /mnt/felhom-drives/r117sd rw,relatime shared:450 - btrfs /dev/sdb rw
`

// useMountinfo points the REAL parsers at a fixture for the duration of one test.
func useMountinfo(t *testing.T, body string) {
	t.Helper()
	p := filepath.Join(t.TempDir(), "mountinfo")
	if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
		t.Fatal(err)
	}
	prev := procSelfMountinfo
	procSelfMountinfo = p
	t.Cleanup(func() { procSelfMountinfo = prev })
}

// livenessServer builds a /disks server over one Observe target (or one registry drive) whose raw mount
// is `raw` and stable guest path derives from it. The two PRE-R-117 terms are forced TRUE — that is the
// whole point: they were both true in the measured defect, so term 3 is the only thing that can save us.
func livenessServer(t *testing.T, obs []hub.StorageTarget, known []storage.KnownTarget) *Server {
	t.Helper()
	opts := Options{
		ListenAddr: "127.0.0.1:0",
		Guests:     &fakeGuestsCfg{},
		Backups:    &fakeBackups{},
		Store:      &fakeStore{},
		Storage:    fakeStorage{targets: obs},
		Tokens:     staticTokens{"A": 8200},
		Disks:      &fakeDiskOps{probe: storage.DeviceProbe{Probed: true, HasFilesystem: true, FSType: "ext4"}},
		DiskGate:   &fakeGate{},
		HostReader: sysOnSDA(),
		Logger:     slog.New(slog.NewTextHandler(io.Discard, nil)),
	}
	if known != nil {
		opts.DriveTargets = fakeKnownTargets{drives: known}
	}
	srv, err := NewServer(opts)
	if err != nil {
		t.Fatal(err)
	}
	srv.baseCtx = context.Background()
	// Terms 1 and 2 TRUE — the measured defect's own conditions. livenessCheck is left nil so the real
	// bindLiveness runs against the fixture.
	srv.boundCheck = func(string) bool { return true }
	srv.deviceCheck = func(string) bool { return true }
	srv.resolveStorageDevice = func(string) (string, error) { return "/dev/sdb1", nil }
	return srv
}

var obsSD = []hub.StorageTarget{
	{Name: "sd", Type: hub.StorageTypeUSB, BackingDevice: "/dev/sdb", MountPath: "/mnt/r117sd", State: hub.StorageStateAttached},
}
var knownSD = []storage.KnownTarget{
	{Name: "sd", Type: hub.StorageTypeUSB, MountPath: "/mnt/r117sd", DurableID: "uuid:71e1", UUID: "71e1"},
}

// ── Group A — the consequence: a dead namespace reads ABSENT ────────────────────────────────────

// TestDisks_BindLiveness_StaleBindReadsAbsent is R-117 case (a), through the real /disks handler.
// It asserts the CONSEQUENCE — what the controller reads off the wire — not that a comparison happened.
func TestDisks_BindLiveness_StaleBindReadsAbsent(t *testing.T) {
	useMountinfo(t, mountinfoStaleBind)
	di := diskByMount(t, livenessServer(t, obsSD, nil), "/mnt/r117sd")
	if di.BoundUnderParent {
		t.Error("BoundUnderParent reports PRESENT over a stale bind (R-117). The bind names 8:16 /dev/sdb " +
			"while the raw mount is 8:32 /dev/sdc; every access through it returns EIO. The controller's " +
			"gate would take its Return branch (controller intermediary.go:258,:299) and restart the " +
			"customer's apps onto a dead namespace, then email backup_target_restored.")
	}
}

// TestDisks_BindLiveness_AbortedFilesystemReadsAbsent is R-117a, the steady-state half — and the test a
// devno-only fix would fail. The device NEVER LEFT, so raw and bind agree on 252:11.
func TestDisks_BindLiveness_AbortedFilesystemReadsAbsent(t *testing.T) {
	for _, c := range []struct{ name, body string }{
		{"emergency_ro (errors=remount-ro fired in place)", mountinfoAborted},
		{"shutdown (forced abort)", mountinfoAbortedShutdown},
	} {
		t.Run(c.name, func(t *testing.T) {
			useMountinfo(t, c.body)
			// GUARD, earned: assert the row is the shape production emits BEFORE asserting the field.
			// A system-role row has no GuestPath, never runs the conjunction, and reports
			// BoundUnderParent=false by default — passing this test while proving nothing.
			di := diskByMount(t, livenessServer(t, obsSD, nil), "/mnt/r117sd")
			if di.Role != "user-data" || di.GuestPath == "" {
				t.Fatalf("fixture does not reproduce the production row shape: role=%q guest_path=%q — "+
					"the conjunction never runs on such a row, so any assertion below is vacuous",
					di.Role, di.GuestPath)
			}
			if di.BoundUnderParent {
				t.Error("BoundUnderParent reports PRESENT over an ABORTED filesystem (R-117a). The devnos " +
					"MATCH (the device never disappeared), so the device comparison cannot see this — only " +
					"the filesystem's own abort token can. Today this state emits NOTHING on any channel: " +
					"the drive is never Disconnected, so the gate produces neither a Stop nor a Return.")
			}
		})
	}
}

// TestDisks_BindLiveness_UnionPath_AbortedReadsAbsent — the union path for the steady-state half. It
// matters more than the Observe one here: this row's Role is hardcoded user-data and its State hardcoded
// attached, so the conjunction is the ONLY thing on the row that can report the abort.
func TestDisks_BindLiveness_UnionPath_AbortedReadsAbsent(t *testing.T) {
	useMountinfo(t, mountinfoAborted)
	di := diskByMount(t, livenessServer(t, nil, knownSD), "/mnt/r117sd")
	if di.BoundUnderParent {
		t.Error("union-path drive reports PRESENT over an ABORTED filesystem (R-117a) — and its Role and " +
			"State are both hardcoded on this row, so nothing else can contradict it")
	}
}

// The union path carries no PVE dir-storage and hardcodes State:"attached", so these terms are the only
// device truth on the row — R-113's reasoning, and it applies to term 3 identically.
func TestDisks_BindLiveness_UnionPath_StaleBindReadsAbsent(t *testing.T) {
	useMountinfo(t, mountinfoStaleBind)
	di := diskByMount(t, livenessServer(t, nil, knownSD), "/mnt/r117sd")
	if di.BoundUnderParent {
		t.Error("union-path drive reports PRESENT over a stale bind (R-117) — and its State is hardcoded " +
			"attached, so nothing else on the row can contradict it")
	}
}

// ── Group B — no false negatives ────────────────────────────────────────────────────────────────

func TestDisks_BindLiveness_HealthyReadsPresent(t *testing.T) {
	useMountinfo(t, mountinfoHealthy)
	for _, c := range []struct {
		name  string
		obs   []hub.StorageTarget
		known []storage.KnownTarget
	}{
		{"observe", obsSD, nil},
		{"union", nil, knownSD},
	} {
		t.Run(c.name, func(t *testing.T) {
			di := diskByMount(t, livenessServer(t, c.obs, c.known), "/mnt/r117sd")
			if !di.BoundUnderParent {
				t.Error("a healthy drive reads ABSENT — a false absent STOPS a working customer's apps, " +
					"which is strictly worse than the bug being fixed")
			}
		})
	}
}

// ── Group C — cannot tell must never mean absent ────────────────────────────────────────────────

// TestDisks_BindLiveness_UnknownIsTreatedAsPresent pins the rule in every way it can be reached. The
// workspace's false-invariant table records newestArchiveOn promising exactly this over a signature that
// could not express it; Usable() is the one place it lives, so this is the test that keeps it honest.
func TestDisks_BindLiveness_UnknownIsTreatedAsPresent(t *testing.T) {
	t.Run("unreadable mount table", func(t *testing.T) {
		prev := procSelfMountinfo
		procSelfMountinfo = filepath.Join(t.TempDir(), "does-not-exist")
		t.Cleanup(func() { procSelfMountinfo = prev })
		if got := bindLiveness("/mnt/felhom-drives/r117sd", "/mnt/r117sd"); got != BindUnknown {
			t.Errorf("unreadable /proc gave %v, want BindUnknown", got)
		}
		di := diskByMount(t, livenessServer(t, obsSD, nil), "/mnt/r117sd")
		if !di.BoundUnderParent {
			t.Error("an unreadable mount table made the drive read ABSENT — cannot-tell must never stop apps")
		}
	})

	t.Run("no raw mount entry to compare against", func(t *testing.T) {
		// Only the bind is in the table. devicePresent is the term that answers device absence; this one
		// must abstain rather than double-count it.
		useMountinfo(t, `755 118 8:16 /felhom-data /mnt/felhom-drives/r117sd rw,relatime shared:450 - ext4 /dev/sdb rw
`)
		if got := bindLiveness("/mnt/felhom-drives/r117sd", "/mnt/r117sd"); got != BindUnknown {
			t.Errorf("missing raw entry gave %v, want BindUnknown", got)
		}
	})

	t.Run("empty paths", func(t *testing.T) {
		if got := bindLiveness("", "/mnt/r117sd"); got != BindUnknown {
			t.Errorf("empty stable gave %v, want BindUnknown", got)
		}
		if got := bindLiveness("/mnt/felhom-drives/r117sd", ""); got != BindUnknown {
			t.Errorf("empty raw gave %v, want BindUnknown", got)
		}
	})

	t.Run("filesystem whose abort vocabulary is unmeasured", func(t *testing.T) {
		useMountinfo(t, mountinfoUnknownFS)
		if got := bindLiveness("/mnt/felhom-drives/r117sd", "/mnt/r117sd"); got != BindUnknown {
			t.Errorf("btrfs bind gave %v, want BindUnknown — we cannot read its abort state, so we must "+
				"not claim LIVE either", got)
		}
		di := diskByMount(t, livenessServer(t, obsSD, nil), "/mnt/r117sd")
		if !di.BoundUnderParent {
			t.Error("an unmeasured filesystem read ABSENT — that would stop apps on every non-ext4 drive")
		}
	})

	t.Run("Usable is the single place the rule lives", func(t *testing.T) {
		for _, c := range []struct {
			l    BindLiveness
			want bool
		}{
			{BindLive, true},
			{BindUnknown, true}, // the rule
			{BindStaleDevice, false},
			{BindAborted, false},
		} {
			if got := c.l.Usable(); got != c.want {
				t.Errorf("%v.Usable() = %v, want %v", c.l, got, c.want)
			}
		}
	})
}

// ── Group D — the verdict itself, including the ordering that matters ───────────────────────────

func TestBindLiveness_Verdicts(t *testing.T) {
	for _, c := range []struct {
		name, body  string
		stable, raw string
		want        BindLiveness
	}{
		{"healthy", mountinfoHealthy, "/mnt/felhom-drives/r117sd", "/mnt/r117sd", BindLive},
		{"stale device (case a)", mountinfoStaleBind, "/mnt/felhom-drives/r117sd", "/mnt/r117sd", BindStaleDevice},
		{"aborted in place, emergency_ro (case b)", mountinfoAborted, "/mnt/felhom-drives/r117sd", "/mnt/r117sd", BindAborted},
		{"aborted in place, shutdown", mountinfoAbortedShutdown, "/mnt/felhom-drives/r117sd", "/mnt/r117sd", BindAborted},
	} {
		t.Run(c.name, func(t *testing.T) {
			useMountinfo(t, c.body)
			if got := bindLiveness(c.stable, c.raw); got != c.want {
				t.Errorf("bindLiveness = %v, want %v", got, c.want)
			}
		})
	}
}

// TestBindLiveness_AbortedWins pins the ORDERING, which is load-bearing and not obvious: in the measured
// stale-bind state the filesystem ALSO carries `shutdown`, so both P1 and P2 apply. The verdict must be
// BindAborted-or-BindStaleDevice — never live — but more importantly the in-place state, where ONLY P2
// applies, must not fall through to a devno comparison that reads equal. This test fails if P1 is checked
// before P2 and returns early.
func TestBindLiveness_AbortedWins_WhenDevnosAgree(t *testing.T) {
	useMountinfo(t, mountinfoAborted)
	got := bindLiveness("/mnt/felhom-drives/r117sd", "/mnt/r117sd")
	if got.Usable() {
		t.Fatalf("bindLiveness = %v (usable) — the devnos agree because the device never left, so a "+
			"P1-first implementation reads this as LIVE and ships R-117's silent half intact", got)
	}
	if got != BindAborted {
		t.Errorf("bindLiveness = %v, want BindAborted (the abort token is the only signal here)", got)
	}
}

// ── Group E — the parser, on a real captured table ──────────────────────────────────────────────

// TestHostMountEntries_ParsesDevnoAndSuperOpts pins the field extraction R-117 turned on. The optional
// fields run (shared:NNN master:NNN) is variable-length, so the " - " separator — not a fixed index — is
// what locates fstype and the super options.
func TestHostMountEntries_ParsesDevnoAndSuperOpts(t *testing.T) {
	// A guest-side line with BOTH optional-field tags, the longest real shape (spike §5.2).
	useMountinfo(t, `759 1176 8:16 /felhom-data /mnt/felhom-drives/r117sd rw,relatime shared:459 master:450 - ext4 /dev/sdb rw,stripe=512,shutdown
`)
	got := hostMountEntries("/mnt/felhom-drives/r117sd")
	if len(got) != 1 {
		t.Fatalf("got %d entries, want 1", len(got))
	}
	e := got[0]
	if e.Devno != "8:16" {
		t.Errorf("Devno = %q, want 8:16 — this is the field R-117 was lost for want of reading", e.Devno)
	}
	if e.Root != "/felhom-data" {
		t.Errorf("Root = %q, want /felhom-data", e.Root)
	}
	if e.FSType != "ext4" {
		t.Errorf("FSType = %q, want ext4 (located via the ' - ' separator, not a fixed index)", e.FSType)
	}
	if !strings.Contains(e.SuperOpts, "shutdown") {
		t.Errorf("SuperOpts = %q, want it to carry `shutdown`", e.SuperOpts)
	}
}

// countHostMounts and isHostMountpoint were rewritten onto hostMountEntries; the double-bind convergence
// AttachDrive depends on must survive that (REUSE.md: a boolean could not converge stacked binds).
func TestHostMountEntries_CountsStackedBinds(t *testing.T) {
	useMountinfo(t, `755 118 8:16 /felhom-data /mnt/felhom-drives/r117sd rw,relatime - ext4 /dev/sdb rw
756 118 8:16 /felhom-data /mnt/felhom-drives/r117sd rw,relatime - ext4 /dev/sdb rw
`)
	if n := countHostMounts("/mnt/felhom-drives/r117sd"); n != 2 {
		t.Errorf("countHostMounts = %d, want 2 — AttachDrive's normalize leg needs the count, not a bool", n)
	}
	if !isHostMountpoint("/mnt/felhom-drives/r117sd") {
		t.Error("isHostMountpoint = false over two stacked binds")
	}
	if isHostMountpoint("/mnt/nope") {
		t.Error("isHostMountpoint = true for a path with no entry")
	}
	if n := countHostMounts("/mnt/nope"); n != 0 {
		t.Errorf("countHostMounts = %d for an absent path, want 0", n)
	}
}