felhom-agent/internal/localapi/intermediary.go

package localapi

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"strconv"
	"strings"
)

// Intermediary-mount model (replaces the per-drive `pct set -mpN` bind). A SINGLE permanent parent bind
// `/mnt/felhom-drives` is set into the guest once (at provision/migration); the host keeps that dir a
// SHARED mount, and the agent mounts/unmounts each drive's felhom-data namespace UNDERNEATH it host-side
// (`mount --bind /mnt/<name>/felhom-data /mnt/felhom-drives/<name>`). Mount propagation (host `shared` →
// guest `slave`) carries the change into the RUNNING guest live — no `pct`, no reboot, and the parent
// bind source never disappears (so the guest is inherently C1-immune). Confinement holds: only the
// felhom-data subtree crosses in, never the customer's other top-level dirs. See
// felhom.eu/documentation/audits/SPIKE-intermediary-mount-2026-06-15.md.

// StableParentDir is the permanent host dir bound once into the guest; drives are swapped underneath it.
const StableParentDir = "/mnt/felhom-drives"

// sharedParentScript re-establishes the shared parent on every HOST boot. It MUST run before
// pve-guests.service so the guest's parent bind inherits the shared peer group as `slave` (if the guest
// starts first, its bind is `private` and drive swaps don't propagate until a guest restart).
const sharedParentScriptPath = "/usr/local/sbin/felhom-shared-parent.sh"
const sharedParentScript = `#!/bin/sh
# felhom stable drive parent: a SHARED bind so the agent can swap backing drives underneath it and the
# guest sees the change live (no restart). MUST run before pve-guests so the guest's parent bind inherits
# the shared peer group (slave). Installed + enabled by felhom-agent. Idempotent.
set -e
mkdir -p ` + StableParentDir + `
# Isolate + share ONLY when first creating the self-bind (a fresh boot). The self-bind inherits the root
# mount's shared peer group, so make-private detaches it (else binds under it DOUBLE via the root peer),
# then make-shared gives it its own group whose only slave is the guest's parent bind. Re-running this on
# an existing parent would churn the peer-group id and orphan the guest's slave — so guard on mountpoint.
if ! mountpoint -q ` + StableParentDir + `; then
  mount --bind ` + StableParentDir + ` ` + StableParentDir + `
  mount --make-private ` + StableParentDir + `
  mount --make-shared ` + StableParentDir + `
fi
`

const sharedParentUnitPath = "/etc/systemd/system/felhom-shared-parent.service"

// sharedParentUnit MUST run before pve-guests so the guest's parent bind inherits the shared peer group.
// WantedBy=pve-guests.service makes pve-guests itself PULL IT IN (and Before= orders it first) — a plain
// WantedBy=multi-user.target proved unreliable (the unit wasn't pulled into the boot transaction; it
// never ran before pve-guests). local-fs.target ordering ensures /mnt is available.
const sharedParentUnit = `[Unit]
Description=Felhom stable drive parent (shared bind for live drive hot-swap)
After=local-fs.target
Before=pve-guests.service
ConditionPathExists=` + sharedParentScriptPath + `

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=` + sharedParentScriptPath + `

[Install]
WantedBy=pve-guests.service multi-user.target
`

// StablePathForRaw maps a drive's RAW host mount (/mnt/<name>) to its stable in-guest path
// (/mnt/felhom-drives/<name>). The basename is the drive name — the single source of truth both repos
// derive the guest path from. Returns "" if `where` is not a /mnt/<name> path.
func StablePathForRaw(where string) string {
	name := DriveNameFromRaw(where)
	if name == "" {
		return ""
	}
	return StableParentDir + "/" + name
}

// DriveNameFromRaw returns the drive name from a raw /mnt/<name> host mount (the basename), or "" if the
// path isn't a single-component /mnt/<name>.
func DriveNameFromRaw(where string) string {
	if !strings.HasPrefix(where, "/mnt/") {
		return ""
	}
	name := strings.TrimPrefix(where, "/mnt/")
	if name == "" || strings.ContainsAny(name, "/ \t") {
		return ""
	}
	return name
}

// EnsureSharedParent makes the host stable parent a SHARED mount and installs+enables the boot-time
// systemd unit that re-establishes it before pve-guests. Idempotent: it only binds when the dir isn't
// already a mountpoint (re-binding would stack), and always (re-)marks it shared (a no-op when already
// shared). Best-effort install of the unit (a host-reboot-persistence concern) — a failed install does
// not stop the live setup. Called at agent startup and at provision.
func (b *GuestBinder) EnsureSharedParent(ctx context.Context) error {
	if err := b.run(ctx, "mkdir", "-p", StableParentDir); err != nil {
		return fmt.Errorf("shared-parent: mkdir %s: %w", StableParentDir, err)
	}
	// Isolate + share the parent ONLY when first creating the self-bind. The self-bind inherits the root
	// mount's shared peer group, so make-PRIVATE detaches it (else binds under it double via the root
	// peer); make-SHARED then gives it its OWN group whose only slave is the guest's parent bind. This
	// must NOT run on every reconcile: re-doing make-private+make-shared churns the peer-group id and
	// ORPHANS the guest's already-established slave (propagation silently dies). On a fresh host boot the
	// parent isn't a mountpoint → this runs once, before pve-guests, so the guest slaves the right group.
	if !isHostMountpoint(StableParentDir) {
		if err := b.run(ctx, "mount", "--bind", StableParentDir, StableParentDir); err != nil {
			return fmt.Errorf("shared-parent: self-bind: %w", err)
		}
		if err := b.run(ctx, "mount", "--make-private", StableParentDir); err != nil {
			return fmt.Errorf("shared-parent: make-private: %w", err)
		}
		if err := b.run(ctx, "mount", "--make-shared", StableParentDir); err != nil {
			return fmt.Errorf("shared-parent: make-shared: %w", err)
		}
	}
	// (Re)install the boot-persistence files only when the on-disk SCRIPT or UNIT differs from what we
	// ship (or is missing) — EnsureSharedParent runs on a periodic reconcile, so re-writing files +
	// daemon-reload every tick would be wasteful; the common case (both current) is two cheap reads.
	//
	// F2-a: this MUST compare the SCRIPT too, not just the unit. The v0.36.6 make-private fix changed
	// only the script (the unit was unchanged), so the earlier unit-only gate never redeployed it —
	// leaving hosts running the pre-fix script (no make-private), whose self-bind stays in root's shared
	// peer group and DOUBLES every drive bind. Comparing both files closes that deploy gap.
	if sharedParentInstallStale(sharedParentUnitPath, sharedParentScriptPath) {
		if ierr := b.installSharedParentUnit(ctx); ierr != nil {
			b.logger.Warn("shared-parent: boot-persistence (re)install failed (live setup OK; survives until host reboot)", "err", ierr)
		}
	}
	return nil
}

// stageTemp writes content to a fresh random-named temp file (os.CreateTemp pattern — `*` is replaced
// by a random string) and returns its path. Caller removes it after the privileged `install`.
func stageTemp(pattern, content string) (string, error) {
	f, err := os.CreateTemp("", pattern)
	if err != nil {
		return "", err
	}
	name := f.Name()
	if _, err := f.WriteString(content); err != nil {
		f.Close()
		os.Remove(name)
		return "", err
	}
	if err := f.Close(); err != nil {
		os.Remove(name)
		return "", err
	}
	return name, nil
}

// sharedParentInstallStale reports whether the on-disk boot script OR unit is missing or differs from
// what this build ships — the trigger to (re)install both. Comparing BOTH (not the unit alone) is the
// F2-a fix: a script-only change must still redeploy. Pure (path args) so it is unit-testable.
func sharedParentInstallStale(unitPath, scriptPath string) bool {
	if cur, err := os.ReadFile(unitPath); err != nil || string(cur) != sharedParentUnit {
		return true
	}
	if cur, err := os.ReadFile(scriptPath); err != nil || string(cur) != sharedParentScript {
		return true
	}
	return false
}

// installSharedParentUnit writes the script + unit (from agent-written temps) and enables the unit so the
// shared parent is re-established on every host boot before pve-guests. Idempotent. The temps are
// RANDOM-named os.CreateTemp files (audit B1): a fixed, predictable /tmp name could be pre-created by
// another local user and rewritten between our write and root's install (TOCTOU into a root-executed
// boot script). The final modes come from `install -m`, so the 0600 temps are fine.
func (b *GuestBinder) installSharedParentUnit(ctx context.Context) error {
	tmpScript, err := stageTemp("felhom-shared-parent-*.sh", sharedParentScript)
	if err != nil {
		return fmt.Errorf("write temp script: %w", err)
	}
	defer os.Remove(tmpScript)
	if err := b.run(ctx, "install", "-m", "0755", "--", tmpScript, sharedParentScriptPath); err != nil {
		return fmt.Errorf("install script: %w", err)
	}
	tmpUnit, err := stageTemp("felhom-shared-parent-*.service", sharedParentUnit)
	if err != nil {
		return fmt.Errorf("write temp unit: %w", err)
	}
	defer os.Remove(tmpUnit)
	if err := b.run(ctx, "install", "-m", "0644", "--", tmpUnit, sharedParentUnitPath); err != nil {
		return fmt.Errorf("install unit: %w", err)
	}
	if err := b.run(ctx, "systemctl", "daemon-reload"); err != nil {
		return fmt.Errorf("daemon-reload: %w", err)
	}
	if err := b.run(ctx, "systemctl", "enable", "felhom-shared-parent.service"); err != nil {
		return fmt.Errorf("enable unit: %w", err)
	}
	return nil
}

// AttachDrive binds a drive's felhom-data namespace under the stable parent so it appears live in the
// guest at the returned stable path (via propagation — no pct, no reboot). `where` is the drive's RAW
// host PVE mount (/mnt/<name>); only `<where>/felhom-data` crosses into the guest (confinement). The
// stable per-drive dir is created HOST-ROOT-owned (fail-closed when nothing is mounted under it); the
// felhom-data namespace is created+chowned to the guest base so the in-guest controller owns it.
//
// GUEST-REBOOT SAFETY (the load-bearing subtlety): a guest's parent bind is NON-RECURSIVE, so on a guest
// reboot it does NOT carry the pre-existing drive submount, and mount propagation only delivers mount
// events created AFTER the guest's bind exists. So "the host already has the bind" is NOT sufficient —
// the GUEST may not see it. AttachDrive therefore checks whether vmid's guest actually sees the stable
// path; if the host has the bind but the guest does not (the post-guest-reboot case), it FORCE re-binds
// (umount + mount) to fire a fresh propagation event into the current guest namespace. Idempotent when
// the guest already sees it.
func (b *GuestBinder) AttachDrive(ctx context.Context, vmid int, where string) (string, error) {
	b.mountMu.Lock() // serialize vs a concurrent DetachDrive/AttachDrive (no double-bind TOCTOU)
	defer b.mountMu.Unlock()
	stable := StablePathForRaw(where)
	if stable == "" {
		return "", fmt.Errorf("guest-attach: %q is not a /mnt/<name> mount", where)
	}
	src := where + "/" + felhomDataNS
	// Ensure the namespace exists + is owned by the guest base (same as the legacy AttachBind).
	if err := b.run(ctx, "mkdir", "-p", src); err != nil {
		return "", fmt.Errorf("guest-attach: namespace %s: %w", src, err)
	}
	if err := b.run(ctx, "chown", guestMappedRoot, src); err != nil {
		return "", fmt.Errorf("guest-attach: chown namespace %s: %w", src, err)
	}
	// The stable mountpoint dir stays HOST-ROOT-owned (fail-closed) — create it, never chown it.
	if err := b.run(ctx, "mkdir", "-p", stable); err != nil {
		return "", fmt.Errorf("guest-attach: stable dir %s: %w", stable, err)
	}
	// NORMALIZE to EXACTLY ONE bind. The target is usable only when there is exactly one bind AND the
	// guest sees it; in that case this is a no-op. Otherwise (zero binds, a post-reboot bind the guest
	// can't see, OR stacked duplicate binds from an earlier race/operator action) we strip ALL existing
	// binds (bounded loop) and lay down exactly one fresh bind — which also re-fires propagation into the
	// current guest namespace. Counting (countHostMounts) rather than a boolean isHostMountpoint is what
	// makes this converge a double-bind to one (the old umount-one+mount-one never did).
	n := countHostMounts(stable)
	if n == 1 && b.GuestSeesMount(ctx, vmid, stable) {
		// R-117: "one bind + the guest sees it" is NOT liveness. Both of those are path-presence tests, so
		// this early return declared a namespace that EIO'd on every call "fully live" and defeated the
		// three call sites that already invoke this repair — the 20 s reconcile ticker, agent startup, and
		// the controller's Return branch BEFORE it restarts the apps (spike §8.2). The verdict decides:
		switch lv := bindLiveness(stable, where); lv {
		case BindStaleDevice:
			// Case (a). The raw mount has healed onto the returning device; re-binding this stale shell
			// onto it REPAIRS the namespace live, with no guest restart (proven, spike §8.1). Fall through
			// to the normalize+rebind below. WARN not INFO-per-tick: this fires once, then it is fixed.
			b.logger.Warn("guest-attach: bind is STALE — it names a different device than the raw mount; re-binding",
				"vmid", vmid, "where", where, "stable", stable, "verdict", lv.String())
		case BindAborted:
			// Case (b), the Q7 steady-state case. The raw mount is the SAME aborted superblock, so a
			// re-bind produces a fresh bind to a still-dead filesystem — and because this runs every 20 s
			// it would be an infinite silent retry: exactly the silence Q7 found, with more CPU. Leave the
			// mount alone and let the truth travel in BoundUnderParent, which now reads false, so the
			// drive gate stops the apps and raises the alarm. Clearing an aborted filesystem needs a
			// remount or a fsck — an operator decision, never an automatic one (R-117a).
			//
			// DEBUG, not WARN: this repeats every tick, and the operator-facing signal is the /disks
			// payload plus the customer alarm. Per logging-conventions, INFO is for state changes.
			b.logger.Debug("guest-attach: filesystem under the bind has ABORTED — not re-binding (a re-bind cannot clear it); reported not-live instead",
				"vmid", vmid, "where", where, "stable", stable, "verdict", lv.String())
			return stable, nil
		default: // BindLive, BindUnknown — genuinely live, or we cannot tell. Unchanged behaviour.
			return stable, nil
		}
	}
	for i := 0; i < 16 && countHostMounts(stable) > 0; i++ {
		if err := b.run(ctx, "umount", stable); err != nil {
			b.logger.Warn("guest-attach: normalize umount failed (continuing)", "stable", stable, "err", err)
			break
		}
	}
	if err := b.run(ctx, "mount", "--bind", src, stable); err != nil {
		return "", fmt.Errorf("guest-attach: bind %s -> %s: %w", src, stable, err)
	}
	b.logger.Info("guest-attach: drive bound under shared parent (normalized to one bind, live)", "vmid", vmid, "where", where, "stable", stable, "prior_binds", n)
	return stable, nil
}

// countHostMounts returns how many times `path` appears as a mount target in /proc/self/mountinfo (i.e.
// how many stacked binds are at it). 0 = not mounted; >1 = stacked duplicates. Used to normalize to one.
func countHostMounts(path string) int { return len(hostMountEntries(path)) }

// GuestSeesMount reports whether vmid's guest currently has `path` as a mount target in ITS mount
// namespace (read from /proc/<guest-init-pid>/mountinfo). This is the GUEST-side truth the host-side
// isHostMountpoint can't see — the signal that distinguishes "bound on the host" from "live in the
// guest" after a guest reboot. A resolution/read error → false (treat as not-seen → AttachDrive re-binds,
// which is safe). The controller's BoundUnderParent report keys on this.
func (b *GuestBinder) GuestSeesMount(ctx context.Context, vmid int, path string) bool {
	pid := b.guestInitPID(ctx, vmid)
	if pid == "" {
		return false
	}
	data, err := os.ReadFile(procGuestMountinfo(pid))
	if err != nil {
		return false
	}
	for _, line := range strings.Split(string(data), "\n") {
		f := strings.Fields(line)
		if len(f) >= 5 && f[4] == path {
			return true
		}
	}
	return false
}

// guestInitPID returns the guest's PID-1 host PID (`lxc-info -n <vmid> -p -H`), or "" on error.
func (b *GuestBinder) guestInitPID(ctx context.Context, vmid int) string {
	out, _, err := b.runner.Run(ctx, "lxc-info", "-n", strconv.Itoa(vmid), "-p", "-H")
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(out))
}

// GuestBootID returns a token that CHANGES on every guest boot (host reboot or guest reboot) but is
// STABLE across a controller-only restart: "<host-btime>-<guest-init-starttime>". The controller persists
// the last-seen value and, when it changes, DETERMINISTICALLY recreates drive-backed apps (they may have
// auto-started on the empty stable bind before the agent re-propagated the drive). host-btime (epoch of
// the host boot, /proc/stat) changes on a host reboot; the guest init's starttime (field 22 of
// /proc/<pid>/stat — ticks since host boot, unique per process launch) changes on a guest reboot. ""
// on any read error (the controller then keeps its last-seen → no spurious recreate).
func (b *GuestBinder) GuestBootID(ctx context.Context, vmid int) string {
	pid := b.guestInitPID(ctx, vmid)
	if pid == "" {
		return ""
	}
	start := procStarttime(pid)
	if start == "" {
		return ""
	}
	bt := hostBtime()
	if bt == "" {
		bt = "0"
	}
	return bt + "-" + start
}

// procStarttime returns field 22 (starttime) of /proc/<pid>/stat. The comm field (2) can contain spaces
// and parentheses, so we split AFTER the last ')': field 22 is index 19 of the post-comm fields
// (field 3 = state is index 0). "" on any error.
func procStarttime(pid string) string {
	data, err := os.ReadFile("/proc/" + pid + "/stat")
	if err != nil {
		return ""
	}
	return starttimeFromStat(string(data))
}

// starttimeFromStat is the pure parser: field 22 (starttime) of a /proc/<pid>/stat body. The comm field
// (2) can contain spaces and parentheses, so split AFTER the LAST ')': field 22 is index 19 of the
// post-comm fields (field 3 = state is index 0). "" on a malformed line.
func starttimeFromStat(s string) string {
	rp := strings.LastIndexByte(s, ')')
	if rp < 0 || rp+2 > len(s) {
		return ""
	}
	fields := strings.Fields(s[rp+1:]) // fields[0] == state (field 3)
	if len(fields) < 20 {
		return ""
	}
	return fields[19] // field 22 (starttime)
}

// hostBtime returns the host boot time (epoch seconds) from /proc/stat's "btime" line. "" on error.
func hostBtime() string {
	data, err := os.ReadFile("/proc/stat")
	if err != nil {
		return ""
	}
	for _, line := range strings.Split(string(data), "\n") {
		if strings.HasPrefix(line, "btime ") {
			return strings.TrimSpace(strings.TrimPrefix(line, "btime "))
		}
	}
	return ""
}

// DetachDrive unmounts a drive's felhom-data from the stable parent (propagates OUT of the guest live),
// leaving the bare HOST-ROOT-owned stable dir → fail-closed (the guest can't write to it even as root,
// since host uid 0 is unmapped). No pct, no reboot. Idempotent: a non-mountpoint is a no-op.
func (b *GuestBinder) DetachDrive(ctx context.Context, where string) error {
	b.mountMu.Lock() // serialize vs a concurrent AttachDrive (so detach can't race a re-bind)
	defer b.mountMu.Unlock()
	stable := StablePathForRaw(where)
	if stable == "" {
		return fmt.Errorf("guest-detach: %q is not a /mnt/<name> mount", where)
	}
	// Loop-umount: a stable path can carry MORE THAN ONE stacked bind (e.g. an operator-applied bind on
	// top of the agent's, or a rare attach race). Detach must remove ALL layers, else eject leaves a
	// lower bind exposing data → fail-close broken. Bounded to avoid an infinite loop.
	for i := 0; i < 16 && isHostMountpoint(stable); i++ {
		if err := b.run(ctx, "umount", stable); err != nil {
			return fmt.Errorf("guest-detach: umount %s (layer %d): %w", stable, i, err)
		}
	}
	if isHostMountpoint(stable) {
		return fmt.Errorf("guest-detach: %s still a mountpoint after 16 umounts", stable)
	}
	b.logger.Info("guest-detach: drive fully unmounted from shared parent (live, fail-closed)", "where", where, "stable", stable)
	return nil
}

// isHostMountpoint reports whether path is currently a mount target in the host's mount table
// (/proc/self/mountinfo). Pure read — used for idempotency (skip re-binding) and the BoundUnderParent
// report. A read error → false (treat as not-mounted; AttachDrive then (re)binds, which is safe).
func isHostMountpoint(path string) bool { return len(hostMountEntries(path)) > 0 }

// procSelfMountinfo is the host mount table every predicate in this file reads. It is a package var
// ONLY so a test can point the REAL parsers at a captured fixture — production never reassigns it, and a
// test that does must restore it (t.Cleanup). Injecting the DATA rather than the verdict is what keeps
// the R-117 tests non-hollow: the parser, the predicate and the /disks handler all run for real.
var procSelfMountinfo = "/proc/self/mountinfo"

// procGuestMountinfo resolves a guest's init PID to its mount-table path. A package var for the same
// single reason as procSelfMountinfo: so a test can point the REAL GuestSeesMount at a captured guest
// mount table. Production never reassigns it.
var procGuestMountinfo = func(pid string) string { return "/proc/" + pid + "/mountinfo" }

// mountEntry is the parsed subset of a mountinfo line the liveness predicate needs. Field numbers are
// the kernel's 1-based numbering (proc(5) "/proc/<pid>/mountinfo"): 3 = major:minor, 4 = root within the
// filesystem, 5 = mount point; after the " - " separator come fstype, source and the per-superblock
// options. Mount points containing spaces are octal-escaped by the kernel, so strings.Fields is safe.
type mountEntry struct {
	// Devno is field 3, the backing device as major:minor. THIS is the field R-117 was lost for want of
	// reading: it sat in the same parsed slice as the mount point and was discarded.
	Devno string
	// Root is field 4 — which subtree of the filesystem is mounted (e.g. /felhom-data for our binds).
	Root string
	// FSType is the filesystem driver, needed to know whether SuperOpts' vocabulary is one we can read.
	FSType string
	// SuperOpts is the per-superblock option list — where ext4 records that it has stopped serving I/O.
	SuperOpts string
}

// hostMountEntries returns every entry in the host mount table whose mount point is `path`. There is
// more than one when binds are stacked (the double-bind case AttachDrive normalizes). A read error
// yields nil — callers treat that as "not mounted"/"cannot tell", never as a positive.
//
// Pure /proc read, NO BLOCK I/O, per CLAUDE.md's health-check rule: a probe that touches a wedged
// device enters uninterruptible sleep and survives SIGKILL (measured, R-117 spike §6.3).
func hostMountEntries(path string) []mountEntry {
	f, err := os.Open(procSelfMountinfo)
	if err != nil {
		return nil
	}
	defer f.Close()
	var out []mountEntry
	sc := bufio.NewScanner(f)
	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
	for sc.Scan() {
		fields := strings.Fields(sc.Text())
		if len(fields) < 5 || fields[4] != path {
			continue
		}
		e := mountEntry{Devno: fields[2], Root: fields[3]}
		// The optional-fields run is variable-length; the " - " separator terminates it.
		for i := 5; i < len(fields); i++ {
			if fields[i] != "-" {
				continue
			}
			if len(fields) > i+1 {
				e.FSType = fields[i+1]
			}
			if len(fields) > i+3 {
				e.SuperOpts = fields[i+3]
			}
			break
		}
		out = append(out, e)
	}
	return out
}

// BindLiveness is the THREE-state answer to "is the bind at the stable path actually usable?".
//
// Three states and not a bool, deliberately. The R-117 fix must be able to say "cannot tell", and the
// cost of getting that wrong is asymmetric: reporting a live drive absent STOPS a customer's apps. The
// workspace's false-invariant table records `newestArchiveOn` promising "errors degrade to unknown,
// never to no-backup" over a (value, bool) signature that made it unrepresentable — the comment was a
// wish. Read every verdict through Usable() and no caller can repeat that.
type BindLiveness int

const (
	// BindUnknown — liveness could not be established (unreadable /proc, no raw mount to compare
	// against, or a filesystem whose abort vocabulary we have not measured). TREATED AS PRESENT by
	// Usable(), the same rule devicePresent applies to an empty path (disks.go).
	BindUnknown BindLiveness = iota
	// BindLive — the bind names the same device as the raw mount and its filesystem has not aborted.
	BindLive
	// BindStaleDevice — R-117 case (a), the detach/return case. The bind still references the superblock
	// of the drive that went away, while the raw mount has healed onto the returning device via its
	// fs-UUID-keyed unit. Every access through the bind fails. RE-BINDING REPAIRS THIS.
	BindStaleDevice
	// BindAborted — R-117 case (b), the Q7 steady-state case. The filesystem under the bind has given up:
	// ext4 sets `shutdown` when its device vanished, `emergency_ro` when errors=remount-ro fired in place.
	// The raw mount is the SAME aborted superblock, so RE-BINDING CANNOT REPAIR THIS — it must surface as
	// not-live so the drive gate stops the apps and alarms. See AttachDrive's switch.
	BindAborted
)

// Usable is the ONLY sanctioned way to turn a verdict into a yes/no, so the unknown-is-present rule
// lives in exactly one place. Pinned by TestBindLiveness_UnknownIsTreatedAsPresent.
func (l BindLiveness) Usable() bool { return l == BindLive || l == BindUnknown }

func (l BindLiveness) String() string {
	switch l {
	case BindLive:
		return "live"
	case BindStaleDevice:
		return "stale-device"
	case BindAborted:
		return "filesystem-aborted"
	default:
		return "unknown"
	}
}

// abortTokensByFS maps a filesystem driver to the per-superblock option tokens it sets when it has
// stopped serving I/O. BOTH ext4 tokens are load-bearing and BOTH were measured (R-117 spike §4):
// `shutdown` when the device was removed, `emergency_ro` when errors=remount-ro fired with the device
// still present. A check for only `shutdown` passes the ENTIRE Q7 state, which is the silent half.
//
// ext2/ext3 are served by the ext4 driver on this kernel, so they emit the same tokens. Anything else is
// a customer-supplied filesystem whose vocabulary we have not measured — it yields UNKNOWN, never LIVE
// (the agent itself only ever formats ext4).
var abortTokensByFS = map[string][]string{
	"ext4": {"shutdown", "emergency_ro"},
	"ext3": {"shutdown", "emergency_ro"},
	"ext2": {"shutdown", "emergency_ro"},
}

// fsAborted reports whether the entry's filesystem has aborted, and whether we could tell at all.
// `known` false means the fstype is not in abortTokensByFS — the caller must degrade to BindUnknown
// rather than infer health from the absence of a token it does not know how to look for.
func fsAborted(e mountEntry) (aborted, known bool) {
	toks, ok := abortTokensByFS[e.FSType]
	if !ok {
		return false, false
	}
	for _, opt := range strings.Split(e.SuperOpts, ",") {
		for _, t := range toks {
			if opt == t {
				return true, true
			}
		}
	}
	return false, true
}

// bindLiveness is the R-117 predicate: does the bind at `stable` actually work? `raw` is the drive's RAW
// host mount (/mnt/<name>). Reads /proc only — NO block I/O, per CLAUDE.md's health-check rule.
//
// WHY THIS EXISTS. GuestSeesMount and isHostMountpoint both compare only field 5 (the mount point) of a
// mountinfo line, so both answer "does a mount by that name exist" and neither can see that the bind and
// the raw mount name DIFFERENT devices. Measured live: raw on 8:32 /dev/sdc while the bind read
// 8:16 /dev/sdb with `shutdown`, BoundUnderParent true, EIO on every read and write, and the gate
// restarting the customer's apps onto it (R-117 spike §5.2).
//
// HOST-SIDE ONLY, deliberately: the host bind and the guest's view of it are the same mount in one
// propagation peer group and carry identical devno and super options (measured, spike §5.2), so this
// needs no lxc-info fork. Guest VISIBILITY is a different question and stays with GuestSeesMount.
func bindLiveness(stable, raw string) BindLiveness {
	if stable == "" || raw == "" {
		return BindUnknown // nothing to compare — never claim absent
	}
	binds := hostMountEntries(stable)
	if len(binds) == 0 {
		return BindUnknown // nothing bound here; that is isHostMountpoint's question, not this one
	}
	rawEntries := hostMountEntries(raw)
	if len(rawEntries) == 0 {
		return BindUnknown // the raw mount is gone — devicePresent already reports that as absent
	}
	// The two cases are distinguished by WHETHER THE DEVICES AGREE, and the abort flag is read off a
	// DIFFERENT entry in each. Getting this backwards is a live trap, caught here by
	// TestBindLiveness_Verdicts: in the real return state the stale bind carries `shutdown` AND names a
	// different device, so an abort-first rule classifies it BindAborted — which reports correctly but
	// refuses the re-bind that actually repairs it. The question a verdict must answer for AttachDrive is
	// not "is something aborted" but "would a re-bind help".
	rawEntry := rawEntries[0]
	for _, b := range binds {
		if b.Devno != rawEntry.Devno {
			// P1 — case (a). The bind references a superblock that is NOT the one the raw mount now has:
			// the drive went away and came back, and the raw mount healed onto it via its fs-UUID-keyed
			// unit. Sound rather than heuristic — a stale bind pins the dead superblock, which keeps the
			// old device index allocated, which FORCES the returning device onto a different number
			// (measured both ways, including the control test where releasing the bind let the letter be
			// reused, spike §3.4).
			//
			// Whether a re-bind repairs it depends on the RAW mount, which is what a re-bind would point
			// at — not on the stale bind's own abort flag.
			if aborted, known := fsAborted(rawEntry); known && aborted {
				return BindAborted // re-binding would land on another dead filesystem
			}
			return BindStaleDevice // re-binding lands on the healthy returning device: repairable
		}
	}
	// Same superblock on both sides, so a re-bind is a no-op by construction. Only the filesystem's own
	// abort state can tell us anything — and this is R-117's steady-state half (spike §9), where the
	// device NEVER LEFT so the devnos above agree and P1 alone reads healthy.
	for _, b := range binds {
		if aborted, known := fsAborted(b); known && aborted {
			return BindAborted
		}
	}
	// Devices agree and nothing aborted. If we cannot read this filesystem's abort vocabulary we must not
	// call it live — say unknown, which Usable() treats as present.
	for _, b := range binds {
		if _, known := fsAborted(b); !known {
			return BindUnknown
		}
	}
	return BindLive
}