Files
felhom-controller/controller/internal/web/intermediary_test.go
T
admin 25e5cb5850 controller v0.71.0: fix guest-reboot boot-race stranding drive-backed apps
Sub-cause: on guest pct reboot, in-guest dockerd auto-starts unless-stopped
drive-backed apps ~18s BEFORE the agent re-binds the drive; the create-time
volume bind fails (mkdir /mnt/felhom-drives/<drive>/userdata: permission denied)
and RestartCount=0 means it's never retried -> stuck Exited. The existing
recovery (processGuestBootChange) RAN but raced the rebind: it sampled the
agent's BoundUnderParent once during fast startup (not live yet), recreated
nothing, and persisted the new boot-id -> burned its one-shot. The periodic gate
never recovered them either (first observation after the rebind -> no transition).

Fix (harden the existing mechanism, no parallel one): processGuestBootChange now
gates on the REAL live in-guest bind. driveBindLive checks whether
/mnt/felhom-drives/<drive> is an actual mountpoint in the controller's own /mnt
(rslave) /proc/self/mountinfo -- true only once the agent's bind propagated,
exactly when docker can recreate the app. pollLiveBinds waits for that (bounded
~120s, poll 2s; rebind lands ~18s) and only then recreates via the normal
pipeline, including stuck-Exited create-time-failure apps (shouldRecreateOnBoot
is state-independent). Single-flight; absent-after-window drives left to the gate;
host-reboot path unaffected; guest-only reboot path now covered.

Tests: pollLiveBinds waits through the rebind then reports live (recreate fires);
never-live drive stays absent; pre-fix companion (single early sample misses the
not-yet-live bind). Red-proofed against a no-wait single-sample.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-16 15:46:51 +02:00

197 lines
9.5 KiB
Go

package web
import (
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
func TestAgentWhere(t *testing.T) {
cases := map[string]string{
"/mnt/felhom-drives/felhom-usb": "/mnt/felhom-usb", // stable → raw
"/mnt/felhom-usb": "/mnt/felhom-usb", // legacy raw → raw (idempotent)
"/mnt/felhom-drives/x": "/mnt/x",
}
for in, want := range cases {
if got := agentWhere(in); got != want {
t.Errorf("agentWhere(%q) = %q, want %q", in, got, want)
}
}
}
// TestShouldRecreateOnBoot pins the DETERMINISTIC boot-id recreate decision: recreate EVERY deployed
// drive-backed present app, independent of its current container state.
//
// COMPANION GUARD: the pre-fix logic (the old `stackStartedRecently`, and even a `State!=stopped` filter)
// MISSED an app that was Stopped/Exited at the one-shot instant — docker hadn't auto-restarted it yet
// after the boot. The "exited" and "stopped" cases below are `true` here: a state-filtered impl returns
// false for them and the app stays down (exactly what happened live: 5 apps exited after a host reboot).
func TestShouldRecreateOnBoot(t *testing.T) {
present := map[string]bool{"/mnt/felhom-drives/felhom-flash": true}
cases := []struct {
name string
deployed bool
hdd string
want bool
}{
{"deployed+present (recreate regardless of state)", true, "/mnt/felhom-drives/felhom-flash", true},
{"drive absent (gate handles)", true, "/mnt/felhom-drives/felhom-usb", false},
{"SSD path never", true, "/mnt/sys_drive/felhom-data", false},
{"app.yaml not deployed", false, "/mnt/felhom-drives/felhom-flash", false},
{"no HDD_PATH (SSD-resident)", true, "", false},
}
for _, c := range cases {
if got := shouldRecreateOnBoot(c.deployed, c.hdd, present); got != c.want {
t.Errorf("%s: shouldRecreateOnBoot = %v, want %v", c.name, got, c.want)
}
}
}
// TestDefaultPromotionTarget pins M1 (never leave zero default).
//
// COMPANION GUARD: the pre-fix decommission blanked the default and promoted nothing — equivalent to this
// always returning ("", false). The "promote another" + "block when only drive" cases below fail that.
func TestDefaultPromotionTarget(t *testing.T) {
flash := "/mnt/felhom-drives/felhom-flash"
usb := "/mnt/felhom-drives/felhom-usb"
// decommissioning a NON-default → no action.
paths := []settings.StoragePath{{Path: flash, IsDefault: true, Schedulable: true}, {Path: usb, Schedulable: true}}
if tgt, blk := defaultPromotionTarget(paths, usb, ""); tgt != "" || blk {
t.Fatalf("non-default decommission: got (%q,%v), want (\"\",false)", tgt, blk)
}
// decommissioning the DEFAULT with another usable → promote it.
if tgt, blk := defaultPromotionTarget(paths, flash, ""); tgt != usb || blk {
t.Fatalf("default decommission: got (%q,%v), want (%q,false)", tgt, blk, usb)
}
// prefer the migrate target when valid.
if tgt, _ := defaultPromotionTarget(paths, flash, usb); tgt != usb {
t.Fatalf("should prefer migrate target %q, got %q", usb, tgt)
}
// the ONLY usable drive (other is decommissioned) → BLOCK.
only := []settings.StoragePath{{Path: flash, IsDefault: true, Schedulable: true}, {Path: usb, Decommissioned: true}}
if tgt, blk := defaultPromotionTarget(only, flash, ""); tgt != "" || !blk {
t.Fatalf("only-drive decommission: got (%q,%v), want (\"\",true)", tgt, blk)
}
}
// TestPollLiveBinds_WaitsForLateBind is the boot-race fix's core: the readiness gate must WAIT for the
// drive bind to actually go live (the agent re-binds it ~18s post-boot) and only THEN report it present,
// so shouldRecreateOnBoot fires and the create-time-failed app is recreated.
//
// COMPANION (must fail pre-fix): the old readiness was a SINGLE early sample of the agent's
// BoundUnderParent — taken before the ~18s rebind, so it read the bind ABSENT, recreated nothing, and
// burned the boot-id one-shot. TestSingleEarlySample_MissesLateBind_Companion pins that broken outcome;
// and if pollLiveBinds is reverted to a no-wait single check, this test fails its "did it wait" assertion.
func TestPollLiveBinds_WaitsForLateBind(t *testing.T) {
flash := "/mnt/felhom-drives/felhom-flash"
var nowT time.Duration // fake monotonic clock since boot
now := func() time.Time { return time.Unix(0, 0).Add(nowT) }
sleep := func(d time.Duration) { nowT += d }
const goLive = 18 * time.Second // the real agent rebind lag
bindLive := func(string) bool { return nowT >= goLive }
live := pollLiveBinds([]string{flash}, bindLive, sleep, now, bootBindWait, bootBindPoll)
if !live[flash] {
t.Fatalf("poll must detect the bind once live; got %v", live)
}
if nowT < goLive { // proves it WAITED through the rebind window (a single sample would return at t=0)
t.Fatalf("poll returned at t=%s before the bind went live at %s — it did not wait (regression)", nowT, goLive)
}
if !shouldRecreateOnBoot(true, flash, live) {
t.Fatalf("with the live bind present, the drive-backed app MUST be recreated")
}
}
// TestSingleEarlySample_MissesLateBind_Companion is the explicit pre-fix companion: a single sample of
// the bind at boot (t=0), before the ~18s rebind, reads it ABSENT → the app is NOT recreated and stays
// Exited. This is exactly what stranded paperless-webserver et al. live.
func TestSingleEarlySample_MissesLateBind_Companion(t *testing.T) {
flash := "/mnt/felhom-drives/felhom-flash"
bindLiveAtBoot := func(string) bool { return false } // not yet live at the boot instant
oldPresent := map[string]bool{flash: bindLiveAtBoot(flash)}
if shouldRecreateOnBoot(true, flash, oldPresent) {
t.Fatalf("companion: a single early sample reads the not-yet-live bind as absent and must MISS it")
}
}
// TestPollLiveBinds_TimeoutLeavesAbsent: a drive that never comes live within the window stays absent,
// so its apps are NOT recreated here (left to the normal drive gate) — no spurious recreate/loop.
func TestPollLiveBinds_TimeoutLeavesAbsent(t *testing.T) {
usb := "/mnt/felhom-drives/felhom-usb"
var nowT time.Duration
now := func() time.Time { return time.Unix(0, 0).Add(nowT) }
sleep := func(d time.Duration) { nowT += d }
bindLive := func(string) bool { return false } // never live
live := pollLiveBinds([]string{usb}, bindLive, sleep, now, bootBindWait, bootBindPoll)
if live[usb] {
t.Fatalf("a never-live bind must remain absent after the bounded wait")
}
if nowT < bootBindWait {
t.Fatalf("poll must run to the deadline for an absent drive, t=%s", nowT)
}
if shouldRecreateOnBoot(true, usb, live) {
t.Fatalf("an absent drive's app must NOT be recreated here (the gate owns drive-absent)")
}
}
func TestStablePathForName(t *testing.T) {
if got := stablePathForName("felhom-usb"); got != "/mnt/felhom-drives/felhom-usb" {
t.Errorf("stablePathForName = %q", got)
}
}
// TestPlanDriveGates pins the gate's pure decision across the four meaningful states.
//
// COMPANION GUARD: a trivial impl that gates every absent path regardless of the disconnected flag would
// re-stop an already-disconnected drive (and never return it); one that ignores presence would never
// gate. Both fail here.
func TestPlanDriveGates(t *testing.T) {
paths := []settings.StoragePath{
{Path: "/mnt/felhom-drives/usb"}, // present + connected → no action
{Path: "/mnt/felhom-drives/flash"}, // ABSENT + connected → STOP
{Path: "/mnt/felhom-drives/back", Disconnected: true}, // present + disconnected → RETURN
{Path: "/mnt/felhom-drives/gone", Disconnected: true}, // ABSENT + disconnected → no action (steady)
{Path: "/mnt/felhom-drives/dead", Decommissioned: true}, // decommissioned → never touched
{Path: "/mnt/sys_drive/felhom-data"}, // INTERNAL SSD (absent from agent) → never gated
}
disks := []agentapi.DiskInfo{
// present = BoundUnderParent (the usable-in-guest signal), not merely State==attached.
{MountPath: "/mnt/usb", GuestPath: "/mnt/felhom-drives/usb", State: "attached", BoundUnderParent: true},
{MountPath: "/mnt/back", GuestPath: "/mnt/felhom-drives/back", State: "attached", BoundUnderParent: true},
// flash reports State=attached but NOT bound under parent yet → still treated ABSENT (the
// reboot-ordering case: raw drive mounted, agent hasn't bound it under the parent yet).
{MountPath: "/mnt/flash", GuestPath: "/mnt/felhom-drives/flash", State: "attached", BoundUnderParent: false},
// gone + dead report NO present disk
}
actions := map[string]gateAction{}
for _, a := range planDriveGates(paths, disks) {
actions[a.Path] = a
}
if len(actions) != 2 {
t.Fatalf("expected exactly 2 actions (stop flash, return back), got %d: %+v", len(actions), actions)
}
if a, ok := actions["/mnt/felhom-drives/flash"]; !ok || !a.Stop || a.Return {
t.Errorf("flash should STOP (absent+connected): %+v", a)
}
if a, ok := actions["/mnt/felhom-drives/back"]; !ok || !a.Return || a.Stop || a.Raw != "/mnt/back" {
t.Errorf("back should RETURN with raw /mnt/back (present+disconnected): %+v", a)
}
if _, gated := actions["/mnt/felhom-drives/usb"]; gated {
t.Errorf("usb (present+connected) must not be gated")
}
if _, acted := actions["/mnt/felhom-drives/gone"]; acted {
t.Errorf("gone (absent+already-disconnected) is steady — no action")
}
if _, acted := actions["/mnt/felhom-drives/dead"]; acted {
t.Errorf("decommissioned drive must never be gated")
}
if _, acted := actions["/mnt/sys_drive/felhom-data"]; acted {
t.Errorf("internal SSD/system path must NEVER be gated (only /mnt/felhom-drives/ externals)")
}
}