controller v0.71.0: fix guest-reboot boot-race stranding drive-backed apps

Sub-cause: on guest pct reboot, in-guest dockerd auto-starts unless-stopped
drive-backed apps ~18s BEFORE the agent re-binds the drive; the create-time
volume bind fails (mkdir /mnt/felhom-drives/<drive>/userdata: permission denied)
and RestartCount=0 means it's never retried -> stuck Exited. The existing
recovery (processGuestBootChange) RAN but raced the rebind: it sampled the
agent's BoundUnderParent once during fast startup (not live yet), recreated
nothing, and persisted the new boot-id -> burned its one-shot. The periodic gate
never recovered them either (first observation after the rebind -> no transition).

Fix (harden the existing mechanism, no parallel one): processGuestBootChange now
gates on the REAL live in-guest bind. driveBindLive checks whether
/mnt/felhom-drives/<drive> is an actual mountpoint in the controller's own /mnt
(rslave) /proc/self/mountinfo -- true only once the agent's bind propagated,
exactly when docker can recreate the app. pollLiveBinds waits for that (bounded
~120s, poll 2s; rebind lands ~18s) and only then recreates via the normal
pipeline, including stuck-Exited create-time-failure apps (shouldRecreateOnBoot
is state-independent). Single-flight; absent-after-window drives left to the gate;
host-reboot path unaffected; guest-only reboot path now covered.

Tests: pollLiveBinds waits through the rebind then reports live (recreate fires);
never-live drive stays absent; pre-fix companion (single early sample misses the
not-yet-live bind). Red-proofed against a no-wait single-sample.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

controller/internal/web/intermediary_test.go

@@ -2,6 +2,7 @@ package web
 
 import (
 	"testing"
+	"time"
 
 	"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
@@ -75,6 +76,69 @@ func TestDefaultPromotionTarget(t *testing.T) {
 	}
 }
 
+// TestPollLiveBinds_WaitsForLateBind is the boot-race fix's core: the readiness gate must WAIT for the
+// drive bind to actually go live (the agent re-binds it ~18s post-boot) and only THEN report it present,
+// so shouldRecreateOnBoot fires and the create-time-failed app is recreated.
+//
+// COMPANION (must fail pre-fix): the old readiness was a SINGLE early sample of the agent's
+// BoundUnderParent — taken before the ~18s rebind, so it read the bind ABSENT, recreated nothing, and
+// burned the boot-id one-shot. TestSingleEarlySample_MissesLateBind_Companion pins that broken outcome;
+// and if pollLiveBinds is reverted to a no-wait single check, this test fails its "did it wait" assertion.
+func TestPollLiveBinds_WaitsForLateBind(t *testing.T) {
+	flash := "/mnt/felhom-drives/felhom-flash"
+	var nowT time.Duration // fake monotonic clock since boot
+	now := func() time.Time { return time.Unix(0, 0).Add(nowT) }
+	sleep := func(d time.Duration) { nowT += d }
+	const goLive = 18 * time.Second // the real agent rebind lag
+	bindLive := func(string) bool { return nowT >= goLive }
+
+	live := pollLiveBinds([]string{flash}, bindLive, sleep, now, bootBindWait, bootBindPoll)
+
+	if !live[flash] {
+		t.Fatalf("poll must detect the bind once live; got %v", live)
+	}
+	if nowT < goLive { // proves it WAITED through the rebind window (a single sample would return at t=0)
+		t.Fatalf("poll returned at t=%s before the bind went live at %s — it did not wait (regression)", nowT, goLive)
+	}
+	if !shouldRecreateOnBoot(true, flash, live) {
+		t.Fatalf("with the live bind present, the drive-backed app MUST be recreated")
+	}
+}
+
+// TestSingleEarlySample_MissesLateBind_Companion is the explicit pre-fix companion: a single sample of
+// the bind at boot (t=0), before the ~18s rebind, reads it ABSENT → the app is NOT recreated and stays
+// Exited. This is exactly what stranded paperless-webserver et al. live.
+func TestSingleEarlySample_MissesLateBind_Companion(t *testing.T) {
+	flash := "/mnt/felhom-drives/felhom-flash"
+	bindLiveAtBoot := func(string) bool { return false } // not yet live at the boot instant
+	oldPresent := map[string]bool{flash: bindLiveAtBoot(flash)}
+	if shouldRecreateOnBoot(true, flash, oldPresent) {
+		t.Fatalf("companion: a single early sample reads the not-yet-live bind as absent and must MISS it")
+	}
+}
+
+// TestPollLiveBinds_TimeoutLeavesAbsent: a drive that never comes live within the window stays absent,
+// so its apps are NOT recreated here (left to the normal drive gate) — no spurious recreate/loop.
+func TestPollLiveBinds_TimeoutLeavesAbsent(t *testing.T) {
+	usb := "/mnt/felhom-drives/felhom-usb"
+	var nowT time.Duration
+	now := func() time.Time { return time.Unix(0, 0).Add(nowT) }
+	sleep := func(d time.Duration) { nowT += d }
+	bindLive := func(string) bool { return false } // never live
+
+	live := pollLiveBinds([]string{usb}, bindLive, sleep, now, bootBindWait, bootBindPoll)
+
+	if live[usb] {
+		t.Fatalf("a never-live bind must remain absent after the bounded wait")
+	}
+	if nowT < bootBindWait {
+		t.Fatalf("poll must run to the deadline for an absent drive, t=%s", nowT)
+	}
+	if shouldRecreateOnBoot(true, usb, live) {
+		t.Fatalf("an absent drive's app must NOT be recreated here (the gate owns drive-absent)")
+	}
+}
+
 func TestStablePathForName(t *testing.T) {
 	if got := stablePathForName("felhom-usb"); got != "/mnt/felhom-drives/felhom-usb" {
 		t.Errorf("stablePathForName = %q", got)