felhom-agent/internal/localapi/intermediary.go

package localapi

import (
	"bufio"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

// Intermediary-mount model (replaces the per-drive `pct set -mpN` bind). A SINGLE permanent parent bind
// `/mnt/felhom-drives` is set into the guest once (at provision/migration); the host keeps that dir a
// SHARED mount, and the agent mounts/unmounts each drive's felhom-data namespace UNDERNEATH it host-side
// (`mount --bind /mnt/<name>/felhom-data /mnt/felhom-drives/<name>`). Mount propagation (host `shared` →
// guest `slave`) carries the change into the RUNNING guest live — no `pct`, no reboot, and the parent
// bind source never disappears (so the guest is inherently C1-immune). Confinement holds: only the
// felhom-data subtree crosses in, never the customer's other top-level dirs. See
// felhom.eu/documentation/audits/SPIKE-intermediary-mount-2026-06-15.md.

// StableParentDir is the permanent host dir bound once into the guest; drives are swapped underneath it.
const StableParentDir = "/mnt/felhom-drives"

// sharedParentScript re-establishes the shared parent on every HOST boot. It MUST run before
// pve-guests.service so the guest's parent bind inherits the shared peer group as `slave` (if the guest
// starts first, its bind is `private` and drive swaps don't propagate until a guest restart).
const sharedParentScriptPath = "/usr/local/sbin/felhom-shared-parent.sh"
const sharedParentScript = `#!/bin/sh
# felhom stable drive parent: a SHARED bind so the agent can swap backing drives underneath it and the
# guest sees the change live (no restart). MUST run before pve-guests so the guest's parent bind inherits
# the shared peer group (slave). Installed + enabled by felhom-agent. Idempotent.
set -e
mkdir -p ` + StableParentDir + `
# Isolate + share ONLY when first creating the self-bind (a fresh boot). The self-bind inherits the root
# mount's shared peer group, so make-private detaches it (else binds under it DOUBLE via the root peer),
# then make-shared gives it its own group whose only slave is the guest's parent bind. Re-running this on
# an existing parent would churn the peer-group id and orphan the guest's slave — so guard on mountpoint.
if ! mountpoint -q ` + StableParentDir + `; then
  mount --bind ` + StableParentDir + ` ` + StableParentDir + `
  mount --make-private ` + StableParentDir + `
  mount --make-shared ` + StableParentDir + `
fi
`

const sharedParentUnitPath = "/etc/systemd/system/felhom-shared-parent.service"

// sharedParentUnit MUST run before pve-guests so the guest's parent bind inherits the shared peer group.
// WantedBy=pve-guests.service makes pve-guests itself PULL IT IN (and Before= orders it first) — a plain
// WantedBy=multi-user.target proved unreliable (the unit wasn't pulled into the boot transaction; it
// never ran before pve-guests). local-fs.target ordering ensures /mnt is available.
const sharedParentUnit = `[Unit]
Description=Felhom stable drive parent (shared bind for live drive hot-swap)
After=local-fs.target
Before=pve-guests.service
ConditionPathExists=` + sharedParentScriptPath + `

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=` + sharedParentScriptPath + `

[Install]
WantedBy=pve-guests.service multi-user.target
`

// StablePathForRaw maps a drive's RAW host mount (/mnt/<name>) to its stable in-guest path
// (/mnt/felhom-drives/<name>). The basename is the drive name — the single source of truth both repos
// derive the guest path from. Returns "" if `where` is not a /mnt/<name> path.
func StablePathForRaw(where string) string {
	name := DriveNameFromRaw(where)
	if name == "" {
		return ""
	}
	return StableParentDir + "/" + name
}

// DriveNameFromRaw returns the drive name from a raw /mnt/<name> host mount (the basename), or "" if the
// path isn't a single-component /mnt/<name>.
func DriveNameFromRaw(where string) string {
	if !strings.HasPrefix(where, "/mnt/") {
		return ""
	}
	name := strings.TrimPrefix(where, "/mnt/")
	if name == "" || strings.ContainsAny(name, "/ \t") {
		return ""
	}
	return name
}

// EnsureSharedParent makes the host stable parent a SHARED mount and installs+enables the boot-time
// systemd unit that re-establishes it before pve-guests. Idempotent: it only binds when the dir isn't
// already a mountpoint (re-binding would stack), and always (re-)marks it shared (a no-op when already
// shared). Best-effort install of the unit (a host-reboot-persistence concern) — a failed install does
// not stop the live setup. Called at agent startup and at provision.
func (b *GuestBinder) EnsureSharedParent(ctx context.Context) error {
	if err := b.run(ctx, "mkdir", "-p", StableParentDir); err != nil {
		return fmt.Errorf("shared-parent: mkdir %s: %w", StableParentDir, err)
	}
	// Isolate + share the parent ONLY when first creating the self-bind. The self-bind inherits the root
	// mount's shared peer group, so make-PRIVATE detaches it (else binds under it double via the root
	// peer); make-SHARED then gives it its OWN group whose only slave is the guest's parent bind. This
	// must NOT run on every reconcile: re-doing make-private+make-shared churns the peer-group id and
	// ORPHANS the guest's already-established slave (propagation silently dies). On a fresh host boot the
	// parent isn't a mountpoint → this runs once, before pve-guests, so the guest slaves the right group.
	if !isHostMountpoint(StableParentDir) {
		if err := b.run(ctx, "mount", "--bind", StableParentDir, StableParentDir); err != nil {
			return fmt.Errorf("shared-parent: self-bind: %w", err)
		}
		if err := b.run(ctx, "mount", "--make-private", StableParentDir); err != nil {
			return fmt.Errorf("shared-parent: make-private: %w", err)
		}
		if err := b.run(ctx, "mount", "--make-shared", StableParentDir); err != nil {
			return fmt.Errorf("shared-parent: make-shared: %w", err)
		}
	}
	// Install the boot-persistence unit only when missing OR its content differs from what we ship (so a
	// unit-template fix deploys) — EnsureSharedParent runs on a periodic reconcile, and re-writing files +
	// daemon-reload every tick would be wasteful, so the common case (unchanged) is a cheap read.
	if cur, err := os.ReadFile(sharedParentUnitPath); err != nil || string(cur) != sharedParentUnit {
		if ierr := b.installSharedParentUnit(ctx); ierr != nil {
			b.logger.Warn("shared-parent: boot-persistence unit install failed (live setup OK; survives until host reboot)", "err", ierr)
		}
	}
	return nil
}

// installSharedParentUnit writes the script + unit (from agent-written temps) and enables the unit so the
// shared parent is re-established on every host boot before pve-guests. Idempotent.
func (b *GuestBinder) installSharedParentUnit(ctx context.Context) error {
	tmpScript := filepath.Join(os.TempDir(), "felhom-shared-parent.sh")
	if err := os.WriteFile(tmpScript, []byte(sharedParentScript), 0o755); err != nil {
		return fmt.Errorf("write temp script: %w", err)
	}
	defer os.Remove(tmpScript)
	if err := b.run(ctx, "install", "-m", "0755", "--", tmpScript, sharedParentScriptPath); err != nil {
		return fmt.Errorf("install script: %w", err)
	}
	tmpUnit := filepath.Join(os.TempDir(), "felhom-shared-parent.service")
	if err := os.WriteFile(tmpUnit, []byte(sharedParentUnit), 0o644); err != nil {
		return fmt.Errorf("write temp unit: %w", err)
	}
	defer os.Remove(tmpUnit)
	if err := b.run(ctx, "install", "-m", "0644", "--", tmpUnit, sharedParentUnitPath); err != nil {
		return fmt.Errorf("install unit: %w", err)
	}
	if err := b.run(ctx, "systemctl", "daemon-reload"); err != nil {
		return fmt.Errorf("daemon-reload: %w", err)
	}
	if err := b.run(ctx, "systemctl", "enable", "felhom-shared-parent.service"); err != nil {
		return fmt.Errorf("enable unit: %w", err)
	}
	return nil
}

// AttachDrive binds a drive's felhom-data namespace under the stable parent so it appears live in the
// guest at the returned stable path (via propagation — no pct, no reboot). `where` is the drive's RAW
// host PVE mount (/mnt/<name>); only `<where>/felhom-data` crosses into the guest (confinement). The
// stable per-drive dir is created HOST-ROOT-owned (fail-closed when nothing is mounted under it); the
// felhom-data namespace is created+chowned to the guest base so the in-guest controller owns it.
//
// GUEST-REBOOT SAFETY (the load-bearing subtlety): a guest's parent bind is NON-RECURSIVE, so on a guest
// reboot it does NOT carry the pre-existing drive submount, and mount propagation only delivers mount
// events created AFTER the guest's bind exists. So "the host already has the bind" is NOT sufficient —
// the GUEST may not see it. AttachDrive therefore checks whether vmid's guest actually sees the stable
// path; if the host has the bind but the guest does not (the post-guest-reboot case), it FORCE re-binds
// (umount + mount) to fire a fresh propagation event into the current guest namespace. Idempotent when
// the guest already sees it.
func (b *GuestBinder) AttachDrive(ctx context.Context, vmid int, where string) (string, error) {
	b.mountMu.Lock() // serialize vs a concurrent DetachDrive/AttachDrive (no double-bind TOCTOU)
	defer b.mountMu.Unlock()
	stable := StablePathForRaw(where)
	if stable == "" {
		return "", fmt.Errorf("guest-attach: %q is not a /mnt/<name> mount", where)
	}
	src := where + "/" + felhomDataNS
	// Ensure the namespace exists + is owned by the guest base (same as the legacy AttachBind).
	if err := b.run(ctx, "mkdir", "-p", src); err != nil {
		return "", fmt.Errorf("guest-attach: namespace %s: %w", src, err)
	}
	if err := b.run(ctx, "chown", guestMappedRoot, src); err != nil {
		return "", fmt.Errorf("guest-attach: chown namespace %s: %w", src, err)
	}
	// The stable mountpoint dir stays HOST-ROOT-owned (fail-closed) — create it, never chown it.
	if err := b.run(ctx, "mkdir", "-p", stable); err != nil {
		return "", fmt.Errorf("guest-attach: stable dir %s: %w", stable, err)
	}
	// NORMALIZE to EXACTLY ONE bind. The target is usable only when there is exactly one bind AND the
	// guest sees it; in that case this is a no-op. Otherwise (zero binds, a post-reboot bind the guest
	// can't see, OR stacked duplicate binds from an earlier race/operator action) we strip ALL existing
	// binds (bounded loop) and lay down exactly one fresh bind — which also re-fires propagation into the
	// current guest namespace. Counting (countHostMounts) rather than a boolean isHostMountpoint is what
	// makes this converge a double-bind to one (the old umount-one+mount-one never did).
	n := countHostMounts(stable)
	if n == 1 && b.GuestSeesMount(ctx, vmid, stable) {
		return stable, nil // exactly one bind + guest-visible → fully live, no-op
	}
	for i := 0; i < 16 && countHostMounts(stable) > 0; i++ {
		if err := b.run(ctx, "umount", stable); err != nil {
			b.logger.Warn("guest-attach: normalize umount failed (continuing)", "stable", stable, "err", err)
			break
		}
	}
	if err := b.run(ctx, "mount", "--bind", src, stable); err != nil {
		return "", fmt.Errorf("guest-attach: bind %s -> %s: %w", src, stable, err)
	}
	b.logger.Info("guest-attach: drive bound under shared parent (normalized to one bind, live)", "vmid", vmid, "where", where, "stable", stable, "prior_binds", n)
	return stable, nil
}

// countHostMounts returns how many times `path` appears as a mount target in /proc/self/mountinfo (i.e.
// how many stacked binds are at it). 0 = not mounted; >1 = stacked duplicates. Used to normalize to one.
func countHostMounts(path string) int {
	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return 0
	}
	defer f.Close()
	n := 0
	sc := bufio.NewScanner(f)
	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
	for sc.Scan() {
		fields := strings.Fields(sc.Text())
		if len(fields) >= 5 && fields[4] == path {
			n++
		}
	}
	return n
}

// GuestSeesMount reports whether vmid's guest currently has `path` as a mount target in ITS mount
// namespace (read from /proc/<guest-init-pid>/mountinfo). This is the GUEST-side truth the host-side
// isHostMountpoint can't see — the signal that distinguishes "bound on the host" from "live in the
// guest" after a guest reboot. A resolution/read error → false (treat as not-seen → AttachDrive re-binds,
// which is safe). The controller's BoundUnderParent report keys on this.
func (b *GuestBinder) GuestSeesMount(ctx context.Context, vmid int, path string) bool {
	pid := b.guestInitPID(ctx, vmid)
	if pid == "" {
		return false
	}
	data, err := os.ReadFile("/proc/" + pid + "/mountinfo")
	if err != nil {
		return false
	}
	for _, line := range strings.Split(string(data), "\n") {
		f := strings.Fields(line)
		if len(f) >= 5 && f[4] == path {
			return true
		}
	}
	return false
}

// guestInitPID returns the guest's PID-1 host PID (`lxc-info -n <vmid> -p -H`), or "" on error.
func (b *GuestBinder) guestInitPID(ctx context.Context, vmid int) string {
	out, _, err := b.runner.Run(ctx, "lxc-info", "-n", strconv.Itoa(vmid), "-p", "-H")
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(out))
}

// GuestBootID returns a token that CHANGES on every guest boot (host reboot or guest reboot) but is
// STABLE across a controller-only restart: "<host-btime>-<guest-init-starttime>". The controller persists
// the last-seen value and, when it changes, DETERMINISTICALLY recreates drive-backed apps (they may have
// auto-started on the empty stable bind before the agent re-propagated the drive). host-btime (epoch of
// the host boot, /proc/stat) changes on a host reboot; the guest init's starttime (field 22 of
// /proc/<pid>/stat — ticks since host boot, unique per process launch) changes on a guest reboot. ""
// on any read error (the controller then keeps its last-seen → no spurious recreate).
func (b *GuestBinder) GuestBootID(ctx context.Context, vmid int) string {
	pid := b.guestInitPID(ctx, vmid)
	if pid == "" {
		return ""
	}
	start := procStarttime(pid)
	if start == "" {
		return ""
	}
	bt := hostBtime()
	if bt == "" {
		bt = "0"
	}
	return bt + "-" + start
}

// procStarttime returns field 22 (starttime) of /proc/<pid>/stat. The comm field (2) can contain spaces
// and parentheses, so we split AFTER the last ')': field 22 is index 19 of the post-comm fields
// (field 3 = state is index 0). "" on any error.
func procStarttime(pid string) string {
	data, err := os.ReadFile("/proc/" + pid + "/stat")
	if err != nil {
		return ""
	}
	return starttimeFromStat(string(data))
}

// starttimeFromStat is the pure parser: field 22 (starttime) of a /proc/<pid>/stat body. The comm field
// (2) can contain spaces and parentheses, so split AFTER the LAST ')': field 22 is index 19 of the
// post-comm fields (field 3 = state is index 0). "" on a malformed line.
func starttimeFromStat(s string) string {
	rp := strings.LastIndexByte(s, ')')
	if rp < 0 || rp+2 > len(s) {
		return ""
	}
	fields := strings.Fields(s[rp+1:]) // fields[0] == state (field 3)
	if len(fields) < 20 {
		return ""
	}
	return fields[19] // field 22 (starttime)
}

// hostBtime returns the host boot time (epoch seconds) from /proc/stat's "btime" line. "" on error.
func hostBtime() string {
	data, err := os.ReadFile("/proc/stat")
	if err != nil {
		return ""
	}
	for _, line := range strings.Split(string(data), "\n") {
		if strings.HasPrefix(line, "btime ") {
			return strings.TrimSpace(strings.TrimPrefix(line, "btime "))
		}
	}
	return ""
}

// DetachDrive unmounts a drive's felhom-data from the stable parent (propagates OUT of the guest live),
// leaving the bare HOST-ROOT-owned stable dir → fail-closed (the guest can't write to it even as root,
// since host uid 0 is unmapped). No pct, no reboot. Idempotent: a non-mountpoint is a no-op.
func (b *GuestBinder) DetachDrive(ctx context.Context, where string) error {
	b.mountMu.Lock() // serialize vs a concurrent AttachDrive (so detach can't race a re-bind)
	defer b.mountMu.Unlock()
	stable := StablePathForRaw(where)
	if stable == "" {
		return fmt.Errorf("guest-detach: %q is not a /mnt/<name> mount", where)
	}
	// Loop-umount: a stable path can carry MORE THAN ONE stacked bind (e.g. an operator-applied bind on
	// top of the agent's, or a rare attach race). Detach must remove ALL layers, else eject leaves a
	// lower bind exposing data → fail-close broken. Bounded to avoid an infinite loop.
	for i := 0; i < 16 && isHostMountpoint(stable); i++ {
		if err := b.run(ctx, "umount", stable); err != nil {
			return fmt.Errorf("guest-detach: umount %s (layer %d): %w", stable, i, err)
		}
	}
	if isHostMountpoint(stable) {
		return fmt.Errorf("guest-detach: %s still a mountpoint after 16 umounts", stable)
	}
	b.logger.Info("guest-detach: drive fully unmounted from shared parent (live, fail-closed)", "where", where, "stable", stable)
	return nil
}

// isHostMountpoint reports whether path is currently a mount target in the host's mount table
// (/proc/self/mountinfo). Pure read — used for idempotency (skip re-binding) and the BoundUnderParent
// report. A read error → false (treat as not-mounted; AttachDrive then (re)binds, which is safe).
func isHostMountpoint(path string) bool {
	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return false
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
	for sc.Scan() {
		// mountinfo field 5 (0-indexed 4) is the mount point.
		fields := strings.Fields(sc.Text())
		if len(fields) >= 5 && fields[4] == path {
			return true
		}
	}
	return false
}