6ea25388d7
processGuestBootChange recreated the drive-backed app stacks but never re-synced FileBrowser (base-infra, no HDD_PATH), so its drive mounts went stale after a reboot. Now, AFTER pollLiveBinds confirms the live binds and the apps are recreated, trigger go s.SyncFileBrowserMounts() so FileBrowser converges against the now-live drives. Refactored into pure recreateDriveBackedApps(stacks, present, recreate, syncFB). Tests: FB sync runs once after recreate (red-proofed companion); runs even when nothing recreated. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
234 lines
11 KiB
Go
234 lines
11 KiB
Go
package web
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
func TestAgentWhere(t *testing.T) {
|
|
cases := map[string]string{
|
|
"/mnt/felhom-drives/felhom-usb": "/mnt/felhom-usb", // stable → raw
|
|
"/mnt/felhom-usb": "/mnt/felhom-usb", // legacy raw → raw (idempotent)
|
|
"/mnt/felhom-drives/x": "/mnt/x",
|
|
}
|
|
for in, want := range cases {
|
|
if got := agentWhere(in); got != want {
|
|
t.Errorf("agentWhere(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestShouldRecreateOnBoot pins the DETERMINISTIC boot-id recreate decision: recreate EVERY deployed
|
|
// drive-backed present app, independent of its current container state.
|
|
//
|
|
// COMPANION GUARD: the pre-fix logic (the old `stackStartedRecently`, and even a `State!=stopped` filter)
|
|
// MISSED an app that was Stopped/Exited at the one-shot instant — docker hadn't auto-restarted it yet
|
|
// after the boot. The "exited" and "stopped" cases below are `true` here: a state-filtered impl returns
|
|
// false for them and the app stays down (exactly what happened live: 5 apps exited after a host reboot).
|
|
func TestShouldRecreateOnBoot(t *testing.T) {
|
|
present := map[string]bool{"/mnt/felhom-drives/felhom-flash": true}
|
|
cases := []struct {
|
|
name string
|
|
deployed bool
|
|
hdd string
|
|
want bool
|
|
}{
|
|
{"deployed+present (recreate regardless of state)", true, "/mnt/felhom-drives/felhom-flash", true},
|
|
{"drive absent (gate handles)", true, "/mnt/felhom-drives/felhom-usb", false},
|
|
{"SSD path never", true, "/mnt/sys_drive/felhom-data", false},
|
|
{"app.yaml not deployed", false, "/mnt/felhom-drives/felhom-flash", false},
|
|
{"no HDD_PATH (SSD-resident)", true, "", false},
|
|
}
|
|
for _, c := range cases {
|
|
if got := shouldRecreateOnBoot(c.deployed, c.hdd, present); got != c.want {
|
|
t.Errorf("%s: shouldRecreateOnBoot = %v, want %v", c.name, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestDefaultPromotionTarget pins M1 (never leave zero default).
|
|
//
|
|
// COMPANION GUARD: the pre-fix decommission blanked the default and promoted nothing — equivalent to this
|
|
// always returning ("", false). The "promote another" + "block when only drive" cases below fail that.
|
|
func TestDefaultPromotionTarget(t *testing.T) {
|
|
flash := "/mnt/felhom-drives/felhom-flash"
|
|
usb := "/mnt/felhom-drives/felhom-usb"
|
|
// decommissioning a NON-default → no action.
|
|
paths := []settings.StoragePath{{Path: flash, IsDefault: true, Schedulable: true}, {Path: usb, Schedulable: true}}
|
|
if tgt, blk := defaultPromotionTarget(paths, usb, ""); tgt != "" || blk {
|
|
t.Fatalf("non-default decommission: got (%q,%v), want (\"\",false)", tgt, blk)
|
|
}
|
|
// decommissioning the DEFAULT with another usable → promote it.
|
|
if tgt, blk := defaultPromotionTarget(paths, flash, ""); tgt != usb || blk {
|
|
t.Fatalf("default decommission: got (%q,%v), want (%q,false)", tgt, blk, usb)
|
|
}
|
|
// prefer the migrate target when valid.
|
|
if tgt, _ := defaultPromotionTarget(paths, flash, usb); tgt != usb {
|
|
t.Fatalf("should prefer migrate target %q, got %q", usb, tgt)
|
|
}
|
|
// the ONLY usable drive (other is decommissioned) → BLOCK.
|
|
only := []settings.StoragePath{{Path: flash, IsDefault: true, Schedulable: true}, {Path: usb, Decommissioned: true}}
|
|
if tgt, blk := defaultPromotionTarget(only, flash, ""); tgt != "" || !blk {
|
|
t.Fatalf("only-drive decommission: got (%q,%v), want (\"\",true)", tgt, blk)
|
|
}
|
|
}
|
|
|
|
// TestPollLiveBinds_WaitsForLateBind is the boot-race fix's core: the readiness gate must WAIT for the
|
|
// drive bind to actually go live (the agent re-binds it ~18s post-boot) and only THEN report it present,
|
|
// so shouldRecreateOnBoot fires and the create-time-failed app is recreated.
|
|
//
|
|
// COMPANION (must fail pre-fix): the old readiness was a SINGLE early sample of the agent's
|
|
// BoundUnderParent — taken before the ~18s rebind, so it read the bind ABSENT, recreated nothing, and
|
|
// burned the boot-id one-shot. TestSingleEarlySample_MissesLateBind_Companion pins that broken outcome;
|
|
// and if pollLiveBinds is reverted to a no-wait single check, this test fails its "did it wait" assertion.
|
|
func TestPollLiveBinds_WaitsForLateBind(t *testing.T) {
|
|
flash := "/mnt/felhom-drives/felhom-flash"
|
|
var nowT time.Duration // fake monotonic clock since boot
|
|
now := func() time.Time { return time.Unix(0, 0).Add(nowT) }
|
|
sleep := func(d time.Duration) { nowT += d }
|
|
const goLive = 18 * time.Second // the real agent rebind lag
|
|
bindLive := func(string) bool { return nowT >= goLive }
|
|
|
|
live := pollLiveBinds([]string{flash}, bindLive, sleep, now, bootBindWait, bootBindPoll)
|
|
|
|
if !live[flash] {
|
|
t.Fatalf("poll must detect the bind once live; got %v", live)
|
|
}
|
|
if nowT < goLive { // proves it WAITED through the rebind window (a single sample would return at t=0)
|
|
t.Fatalf("poll returned at t=%s before the bind went live at %s — it did not wait (regression)", nowT, goLive)
|
|
}
|
|
if !shouldRecreateOnBoot(true, flash, live) {
|
|
t.Fatalf("with the live bind present, the drive-backed app MUST be recreated")
|
|
}
|
|
}
|
|
|
|
// TestSingleEarlySample_MissesLateBind_Companion is the explicit pre-fix companion: a single sample of
|
|
// the bind at boot (t=0), before the ~18s rebind, reads it ABSENT → the app is NOT recreated and stays
|
|
// Exited. This is exactly what stranded paperless-webserver et al. live.
|
|
func TestSingleEarlySample_MissesLateBind_Companion(t *testing.T) {
|
|
flash := "/mnt/felhom-drives/felhom-flash"
|
|
bindLiveAtBoot := func(string) bool { return false } // not yet live at the boot instant
|
|
oldPresent := map[string]bool{flash: bindLiveAtBoot(flash)}
|
|
if shouldRecreateOnBoot(true, flash, oldPresent) {
|
|
t.Fatalf("companion: a single early sample reads the not-yet-live bind as absent and must MISS it")
|
|
}
|
|
}
|
|
|
|
// TestPollLiveBinds_TimeoutLeavesAbsent: a drive that never comes live within the window stays absent,
|
|
// so its apps are NOT recreated here (left to the normal drive gate) — no spurious recreate/loop.
|
|
func TestPollLiveBinds_TimeoutLeavesAbsent(t *testing.T) {
|
|
usb := "/mnt/felhom-drives/felhom-usb"
|
|
var nowT time.Duration
|
|
now := func() time.Time { return time.Unix(0, 0).Add(nowT) }
|
|
sleep := func(d time.Duration) { nowT += d }
|
|
bindLive := func(string) bool { return false } // never live
|
|
|
|
live := pollLiveBinds([]string{usb}, bindLive, sleep, now, bootBindWait, bootBindPoll)
|
|
|
|
if live[usb] {
|
|
t.Fatalf("a never-live bind must remain absent after the bounded wait")
|
|
}
|
|
if nowT < bootBindWait {
|
|
t.Fatalf("poll must run to the deadline for an absent drive, t=%s", nowT)
|
|
}
|
|
if shouldRecreateOnBoot(true, usb, live) {
|
|
t.Fatalf("an absent drive's app must NOT be recreated here (the gate owns drive-absent)")
|
|
}
|
|
}
|
|
|
|
// TestRecreateDriveBackedApps_SyncsFileBrowserAfterRecreate (Task B): FileBrowser must be re-synced
|
|
// AFTER the drive-backed apps are recreated (so it syncs against live binds). COMPANION: the pre-fix
|
|
// boot-recreate path never synced FileBrowser — red-proofed by dropping the syncFB() call.
|
|
func TestRecreateDriveBackedApps_SyncsFileBrowserAfterRecreate(t *testing.T) {
|
|
flash := "/mnt/felhom-drives/felhom-flash"
|
|
present := map[string]bool{flash: true}
|
|
stacks := []bootStack{
|
|
{name: "romm", deployed: true, hdd: flash}, // drive-backed, live → recreate
|
|
{name: "actualbudget", deployed: true, hdd: "/mnt/sys_drive/felhom-data"}, // SSD → not recreated
|
|
{name: "stranded", deployed: true, hdd: "/mnt/felhom-drives/felhom-usb"}, // drive-backed, bind NOT live → skipped
|
|
}
|
|
var seq []string
|
|
recreate := func(bs bootStack) { seq = append(seq, "recreate:"+bs.name) }
|
|
syncFB := func() { seq = append(seq, "syncFB") }
|
|
|
|
recreated, skipped := recreateDriveBackedApps(stacks, present, recreate, syncFB)
|
|
|
|
if recreated != 1 || skipped != 1 {
|
|
t.Fatalf("recreated=%d skipped=%d, want 1/1", recreated, skipped)
|
|
}
|
|
// FileBrowser sync MUST be invoked, and AFTER every recreate.
|
|
if len(seq) != 2 || seq[0] != "recreate:romm" || seq[len(seq)-1] != "syncFB" {
|
|
t.Fatalf("FileBrowser sync must run once, AFTER the recreate; seq=%v", seq)
|
|
}
|
|
}
|
|
|
|
// TestRecreateDriveBackedApps_SyncsEvenWithNoRecreate: FileBrowser is re-synced to reflect the live
|
|
// binds even when no app needed recreating (so its mounts never go stale).
|
|
func TestRecreateDriveBackedApps_SyncsEvenWithNoRecreate(t *testing.T) {
|
|
stacks := []bootStack{{name: "x", deployed: true, hdd: "/mnt/sys_drive/felhom-data"}} // SSD only
|
|
synced := false
|
|
recreateDriveBackedApps(stacks, map[string]bool{}, func(bootStack) { t.Fatal("should not recreate") }, func() { synced = true })
|
|
if !synced {
|
|
t.Fatal("FileBrowser sync must run even when nothing was recreated")
|
|
}
|
|
}
|
|
|
|
func TestStablePathForName(t *testing.T) {
|
|
if got := stablePathForName("felhom-usb"); got != "/mnt/felhom-drives/felhom-usb" {
|
|
t.Errorf("stablePathForName = %q", got)
|
|
}
|
|
}
|
|
|
|
// TestPlanDriveGates pins the gate's pure decision across the four meaningful states.
|
|
//
|
|
// COMPANION GUARD: a trivial impl that gates every absent path regardless of the disconnected flag would
|
|
// re-stop an already-disconnected drive (and never return it); one that ignores presence would never
|
|
// gate. Both fail here.
|
|
func TestPlanDriveGates(t *testing.T) {
|
|
paths := []settings.StoragePath{
|
|
{Path: "/mnt/felhom-drives/usb"}, // present + connected → no action
|
|
{Path: "/mnt/felhom-drives/flash"}, // ABSENT + connected → STOP
|
|
{Path: "/mnt/felhom-drives/back", Disconnected: true}, // present + disconnected → RETURN
|
|
{Path: "/mnt/felhom-drives/gone", Disconnected: true}, // ABSENT + disconnected → no action (steady)
|
|
{Path: "/mnt/felhom-drives/dead", Decommissioned: true}, // decommissioned → never touched
|
|
{Path: "/mnt/sys_drive/felhom-data"}, // INTERNAL SSD (absent from agent) → never gated
|
|
}
|
|
disks := []agentapi.DiskInfo{
|
|
// present = BoundUnderParent (the usable-in-guest signal), not merely State==attached.
|
|
{MountPath: "/mnt/usb", GuestPath: "/mnt/felhom-drives/usb", State: "attached", BoundUnderParent: true},
|
|
{MountPath: "/mnt/back", GuestPath: "/mnt/felhom-drives/back", State: "attached", BoundUnderParent: true},
|
|
// flash reports State=attached but NOT bound under parent yet → still treated ABSENT (the
|
|
// reboot-ordering case: raw drive mounted, agent hasn't bound it under the parent yet).
|
|
{MountPath: "/mnt/flash", GuestPath: "/mnt/felhom-drives/flash", State: "attached", BoundUnderParent: false},
|
|
// gone + dead report NO present disk
|
|
}
|
|
actions := map[string]gateAction{}
|
|
for _, a := range planDriveGates(paths, disks) {
|
|
actions[a.Path] = a
|
|
}
|
|
if len(actions) != 2 {
|
|
t.Fatalf("expected exactly 2 actions (stop flash, return back), got %d: %+v", len(actions), actions)
|
|
}
|
|
if a, ok := actions["/mnt/felhom-drives/flash"]; !ok || !a.Stop || a.Return {
|
|
t.Errorf("flash should STOP (absent+connected): %+v", a)
|
|
}
|
|
if a, ok := actions["/mnt/felhom-drives/back"]; !ok || !a.Return || a.Stop || a.Raw != "/mnt/back" {
|
|
t.Errorf("back should RETURN with raw /mnt/back (present+disconnected): %+v", a)
|
|
}
|
|
if _, gated := actions["/mnt/felhom-drives/usb"]; gated {
|
|
t.Errorf("usb (present+connected) must not be gated")
|
|
}
|
|
if _, acted := actions["/mnt/felhom-drives/gone"]; acted {
|
|
t.Errorf("gone (absent+already-disconnected) is steady — no action")
|
|
}
|
|
if _, acted := actions["/mnt/felhom-drives/dead"]; acted {
|
|
t.Errorf("decommissioned drive must never be gated")
|
|
}
|
|
if _, acted := actions["/mnt/sys_drive/felhom-data"]; acted {
|
|
t.Errorf("internal SSD/system path must NEVER be gated (only /mnt/felhom-drives/ externals)")
|
|
}
|
|
}
|