e2de234325
Live diagnosis of drive-backed apps stuck Exited after a pct reboot pinned THREE sub-causes, fixed together (hardening the existing processGuestBootChange, not a parallel mechanism): 1. Agent-path blocker (live root cause): agentClient() returned "agent not configured" (cfg.LocalAPI.Endpoint empty), so processGuestBootChange AND the whole drive gate bailed at the first guard. bootstrap.json had a complete local_api block, but MaybeIngest returned immediately on "already configured" so a controller.yaml seeded before local_api existed never got the agent path. Fix: MaybeIngest now calls ensureLocalAPI on the already-configured path, merging local_api from bootstrap.json into the existing controller.yaml when missing (no hub re-pull, config preserved; idempotent + fail-safe). 2. Boot-race readiness gate: processGuestBootChange sampled BoundUnderParent once during fast startup, racing the ~18s rebind, recreated nothing, burned its boot-id one-shot. Fix: gate on the REAL live in-guest bind -- driveBindLive checks /mnt/felhom-drives/<drive> is a mountpoint in the controller's own /mnt rslave /proc/self/mountinfo; pollLiveBinds waits for it (bounded ~120s) before recreating via the normal pipeline. shouldRecreateOnBoot stays state-independent so stuck-Exited create-time-failure apps are included. 3. Single-shot fragility: processGuestBootChange ran only once at startup; a briefly-unreachable agent right after a guest reboot stranded recovery. Fix: driveGateLoop runs it every periodic tick too (idempotent, boot-id gated). Tests (non-hollow, pre-fix companions, red-proofed): pollLiveBinds waits then reports live / never-live stays absent / single early sample misses; ensureLocalAPI merges local_api into a configured controller.yaml that lacks it / no-ops when present. Live-accepted with repeated pct reboot 9201. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
313 lines
12 KiB
Go
313 lines
12 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
|
)
|
|
|
|
func testLogger() *log.Logger { return log.New(io.Discard, "", 0) }
|
|
|
|
// A valid v2 bootstrap: only customer.id + hub.url + hub.retrieval_password + the per-guest local_api.
|
|
const goodBootstrapV2 = `{
|
|
"schema": "felhom.bootstrap/v2",
|
|
"customer": {"id": "cust-8200"},
|
|
"hub": {"url": "https://hub.felhom.eu", "retrieval_password": "five-word-passphrase-here"},
|
|
"local_api": {"endpoint": "192.168.0.162:8443", "fingerprint": "ab12", "token": "PERGUESTTOKEN"}
|
|
}`
|
|
|
|
// hubYAML is what the hub's /api/v1/config/{id} returns: a full controller.yaml carrying the
|
|
// CUSTOMER-scoped hub key + identity + assets, but NO local_api (the hub can't know per-guest
|
|
// Proxmox internals). Includes an unmodeled field (`assets.source_url`) to prove map-level merge
|
|
// preserves it.
|
|
const hubYAML = `# Felhom Controller Configuration
|
|
customer:
|
|
id: cust-8200
|
|
name: Teszt Ügyfél
|
|
domain: cust8200.felhom.eu
|
|
email: a@b.hu
|
|
hub:
|
|
enabled: true
|
|
url: https://hub.felhom.eu
|
|
api_key: CUSTKEY_FROM_HUB
|
|
assets:
|
|
source_url: https://hub.felhom.eu/assets
|
|
sync_enabled: true
|
|
web:
|
|
session_secret: deadbeef
|
|
`
|
|
|
|
func writeBootstrap(t *testing.T, dir, content string) (bpath, cfgPath string) {
|
|
t.Helper()
|
|
bpath = filepath.Join(dir, "bootstrap.json")
|
|
cfgPath = filepath.Join(dir, "controller.yaml")
|
|
if err := os.WriteFile(bpath, []byte(content), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("FELHOM_BOOTSTRAP_PATH", bpath)
|
|
return bpath, cfgPath
|
|
}
|
|
|
|
// PULL+MERGE: an unconfigured controller pulls the hub yaml and merges in the per-guest local_api.
|
|
// The written controller.yaml must carry BOTH the hub's customer key/identity/assets AND the
|
|
// bootstrap's local_api — and must NOT contain a host key.
|
|
func TestMaybeIngest_PullsAndMerges(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_, cfgPath := writeBootstrap(t, dir, goodBootstrapV2)
|
|
|
|
var calls int
|
|
var gotURL, gotID, gotPass string
|
|
pull := func(hubURL, customerID, pass string) (string, error) {
|
|
calls++
|
|
gotURL, gotID, gotPass = hubURL, customerID, pass
|
|
return hubYAML, nil
|
|
}
|
|
|
|
got := MaybeIngest(cfgPath, config.Default(), testLogger(), pull)
|
|
|
|
// pull was called once with the bootstrap's values
|
|
if calls != 1 || gotURL != "https://hub.felhom.eu" || gotID != "cust-8200" || gotPass != "five-word-passphrase-here" {
|
|
t.Fatalf("pull args wrong: calls=%d url=%q id=%q pass=%q", calls, gotURL, gotID, gotPass)
|
|
}
|
|
// returned cfg carries the hub's CUSTOMER key + identity (from the pull)
|
|
if got.Hub.APIKey != "CUSTKEY_FROM_HUB" || !got.Hub.Enabled || got.Hub.URL != "https://hub.felhom.eu" {
|
|
t.Fatalf("hub not from pulled config: %+v", got.Hub)
|
|
}
|
|
if got.Customer.ID != "cust-8200" || got.Customer.Domain != "cust8200.felhom.eu" {
|
|
t.Fatalf("customer not from pulled config: %+v", got.Customer)
|
|
}
|
|
// AND the per-guest local_api merged in from the bootstrap
|
|
if got.LocalAPI.Endpoint != "192.168.0.162:8443" || got.LocalAPI.Token != "PERGUESTTOKEN" || got.LocalAPI.Fingerprint != "ab12" {
|
|
t.Fatalf("local_api not merged from bootstrap: %+v", got.LocalAPI)
|
|
}
|
|
// unmodeled hub field preserved (forward-compat: map-level merge)
|
|
if got.Assets.SourceURL != "https://hub.felhom.eu/assets" {
|
|
t.Fatalf("assets.source_url not preserved through merge: %+v", got.Assets)
|
|
}
|
|
// the written file must reload configured, carry the customer key, and NOT carry a host key
|
|
raw, err := os.ReadFile(cfgPath)
|
|
if err != nil {
|
|
t.Fatalf("controller.yaml not written: %v", err)
|
|
}
|
|
s := string(raw)
|
|
if !strings.Contains(s, "CUSTKEY_FROM_HUB") {
|
|
t.Fatalf("written controller.yaml missing customer key:\n%s", s)
|
|
}
|
|
if !strings.Contains(s, "PERGUESTTOKEN") || !strings.Contains(s, "192.168.0.162:8443") {
|
|
t.Fatalf("written controller.yaml missing merged local_api:\n%s", s)
|
|
}
|
|
if strings.Contains(s, "host_id") || strings.Contains(s, "HOSTKEY") {
|
|
t.Fatalf("written controller.yaml leaked a host key/id:\n%s", s)
|
|
}
|
|
reloaded, err := config.LoadPermissive(cfgPath)
|
|
if err != nil || reloaded.Customer.ID != "cust-8200" || reloaded.Hub.APIKey != "CUSTKEY_FROM_HUB" {
|
|
t.Fatalf("written controller.yaml does not reload configured: %v / %+v", err, reloaded)
|
|
}
|
|
}
|
|
|
|
// IDEMPOTENT: an already-configured controller is never clobbered, and pull is NEVER invoked.
|
|
func TestMaybeIngest_DoesNotClobberConfigured_NoPull(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_, cfgPath := writeBootstrap(t, dir, goodBootstrapV2)
|
|
|
|
existing := config.Default()
|
|
existing.Customer.ID = "already-here"
|
|
existing.Customer.Domain = "existing.felhom.eu"
|
|
|
|
pulled := false
|
|
pull := func(string, string, string) (string, error) { pulled = true; return hubYAML, nil }
|
|
|
|
got := MaybeIngest(cfgPath, existing, testLogger(), pull)
|
|
if pulled {
|
|
t.Fatal("pull was invoked on an already-configured controller")
|
|
}
|
|
if got.Customer.ID != "already-here" {
|
|
t.Fatalf("configured controller was clobbered: %+v", got.Customer)
|
|
}
|
|
if _, err := os.Stat(cfgPath); err == nil {
|
|
t.Fatal("controller.yaml written despite an already-configured controller")
|
|
}
|
|
}
|
|
|
|
// FIX (v0.71.0): an ALREADY-configured controller.yaml that LACKS local_api still gets the per-guest
|
|
// agent path merged from bootstrap.json — without it agentClient() returns "agent not configured" and
|
|
// the whole drive gate + guest-reboot recovery silently die. COMPANION: the pre-fix MaybeIngest
|
|
// returned immediately on "already configured", so LocalAPI.Endpoint stayed empty (this test fails
|
|
// against that). The hub is NEVER re-pulled (the existing config is preserved verbatim).
|
|
func TestMaybeIngest_ConfiguredMissingLocalAPI_Merges(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_, cfgPath := writeBootstrap(t, dir, goodBootstrapV2) // bootstrap HAS local_api
|
|
|
|
const configuredNoLocalAPI = `customer:
|
|
id: cust-8200
|
|
domain: cust8200.felhom.eu
|
|
hub:
|
|
enabled: true
|
|
url: https://hub.felhom.eu
|
|
api_key: CUSTKEY
|
|
`
|
|
if err := os.WriteFile(cfgPath, []byte(configuredNoLocalAPI), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
existing, err := config.LoadPermissive(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if existing.LocalAPI.Endpoint != "" {
|
|
t.Fatalf("precondition: existing config should lack local_api, got %q", existing.LocalAPI.Endpoint)
|
|
}
|
|
|
|
pulled := false
|
|
pull := func(string, string, string) (string, error) { pulled = true; return hubYAML, nil }
|
|
|
|
got := MaybeIngest(cfgPath, existing, testLogger(), pull)
|
|
|
|
if pulled {
|
|
t.Fatal("must NOT re-pull the hub config for an already-configured controller")
|
|
}
|
|
if got.LocalAPI.Endpoint != "192.168.0.162:8443" || got.LocalAPI.Token != "PERGUESTTOKEN" || got.LocalAPI.Fingerprint != "ab12" {
|
|
t.Fatalf("local_api not merged into the already-configured controller (the live boot-recovery blocker): %+v", got.LocalAPI)
|
|
}
|
|
if got.Customer.ID != "cust-8200" || got.Hub.APIKey != "CUSTKEY" {
|
|
t.Fatalf("merging local_api must preserve the existing config: %+v / %+v", got.Customer, got.Hub)
|
|
}
|
|
raw, _ := os.ReadFile(cfgPath)
|
|
if !strings.Contains(string(raw), "192.168.0.162:8443") || !strings.Contains(string(raw), "CUSTKEY") {
|
|
t.Fatalf("controller.yaml must persist local_api + keep the customer key:\n%s", raw)
|
|
}
|
|
}
|
|
|
|
// IDEMPOTENT: a configured controller that ALREADY has local_api is untouched (no re-merge, no pull,
|
|
// no rewrite).
|
|
func TestMaybeIngest_ConfiguredWithLocalAPI_NoOp(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_, cfgPath := writeBootstrap(t, dir, goodBootstrapV2)
|
|
existing := config.Default()
|
|
existing.Customer.ID = "cust-8200"
|
|
existing.LocalAPI.Endpoint = "already:9999"
|
|
pulled := false
|
|
pull := func(string, string, string) (string, error) { pulled = true; return hubYAML, nil }
|
|
|
|
got := MaybeIngest(cfgPath, existing, testLogger(), pull)
|
|
if pulled {
|
|
t.Fatal("must not pull when already configured")
|
|
}
|
|
if got.LocalAPI.Endpoint != "already:9999" {
|
|
t.Fatalf("existing local_api must be preserved, got %q", got.LocalAPI.Endpoint)
|
|
}
|
|
if _, err := os.Stat(cfgPath); err == nil {
|
|
t.Fatal("controller.yaml must not be rewritten when local_api already present")
|
|
}
|
|
}
|
|
|
|
// FAIL-SAFE (transient): a persistently-unreachable hub is retried, then leaves cfg in setup mode
|
|
// (no controller.yaml). Asserts the retry count (1 initial + len(pullRetryDelays)).
|
|
func TestMaybeIngest_TransientRetriesThenSetup(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_, cfgPath := writeBootstrap(t, dir, goodBootstrapV2)
|
|
|
|
// shrink the backoff so the test is fast
|
|
orig := pullRetryDelays
|
|
pullRetryDelays = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
|
|
defer func() { pullRetryDelays = orig }()
|
|
|
|
calls := 0
|
|
pull := func(string, string, string) (string, error) {
|
|
calls++
|
|
return "", fmt.Errorf("%w: dial tcp: timeout", ErrPullTransient)
|
|
}
|
|
|
|
got := MaybeIngest(cfgPath, config.Default(), testLogger(), pull)
|
|
if got.Customer.ID != "" {
|
|
t.Fatalf("seeded despite a failing hub pull: %+v", got.Customer)
|
|
}
|
|
if _, err := os.Stat(cfgPath); err == nil {
|
|
t.Fatal("controller.yaml written despite a failing pull")
|
|
}
|
|
if want := 1 + len(pullRetryDelays); calls != want {
|
|
t.Fatalf("transient retry count: got %d, want %d", calls, want)
|
|
}
|
|
}
|
|
|
|
// FAIL-SAFE (permanent): an auth/not-found failure is NOT retried (fail fast), setup mode.
|
|
func TestMaybeIngest_PermanentNoRetry(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_, cfgPath := writeBootstrap(t, dir, goodBootstrapV2)
|
|
|
|
calls := 0
|
|
pull := func(string, string, string) (string, error) {
|
|
calls++
|
|
return "", errors.New("authentication failed") // permanent (not wrapped with ErrPullTransient)
|
|
}
|
|
|
|
got := MaybeIngest(cfgPath, config.Default(), testLogger(), pull)
|
|
if got.Customer.ID != "" {
|
|
t.Fatalf("seeded despite a permanent pull failure: %+v", got.Customer)
|
|
}
|
|
if calls != 1 {
|
|
t.Fatalf("permanent failure was retried: %d calls", calls)
|
|
}
|
|
}
|
|
|
|
// SCHEMA REJECT: a v1 (or any non-v2) schema is rejected → setup mode, no pull.
|
|
func TestMaybeIngest_RejectsNonV2Schema(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_, cfgPath := writeBootstrap(t, dir, `{"schema":"felhom.bootstrap/v1","customer":{"id":"x"},"hub":{"url":"u","retrieval_password":"p"},"local_api":{"endpoint":"e","fingerprint":"f","token":"t"}}`)
|
|
|
|
pulled := false
|
|
pull := func(string, string, string) (string, error) { pulled = true; return hubYAML, nil }
|
|
|
|
got := MaybeIngest(cfgPath, config.Default(), testLogger(), pull)
|
|
if pulled {
|
|
t.Fatal("pull invoked for a non-v2 schema")
|
|
}
|
|
if got.Customer.ID != "" {
|
|
t.Fatal("seeded from a non-v2 schema")
|
|
}
|
|
}
|
|
|
|
// MISSING REQUIRED FIELDS: a v2 bootstrap missing the retrieval passphrase (or local_api) is rejected.
|
|
func TestMaybeIngest_MissingRequiredStaysInSetup(t *testing.T) {
|
|
dir := t.TempDir()
|
|
_, cfgPath := writeBootstrap(t, dir, `{"schema":"felhom.bootstrap/v2","customer":{"id":"x"},"hub":{"url":"u"},"local_api":{"endpoint":"e","fingerprint":"f","token":"t"}}`)
|
|
|
|
pulled := false
|
|
pull := func(string, string, string) (string, error) { pulled = true; return hubYAML, nil }
|
|
got := MaybeIngest(cfgPath, config.Default(), testLogger(), pull)
|
|
if pulled {
|
|
t.Fatal("pull invoked despite a missing retrieval_password")
|
|
}
|
|
if got.Customer.ID != "" {
|
|
t.Fatal("seeded despite missing required fields")
|
|
}
|
|
}
|
|
|
|
// MALFORMED / ABSENT: never crash, stay in setup, no pull.
|
|
func TestMaybeIngest_MalformedAndAbsent(t *testing.T) {
|
|
dir := t.TempDir()
|
|
pulled := false
|
|
pull := func(string, string, string) (string, error) { pulled = true; return hubYAML, nil }
|
|
|
|
// malformed
|
|
_, cfgPath := writeBootstrap(t, dir, "{not json")
|
|
if got := MaybeIngest(cfgPath, config.Default(), testLogger(), pull); got.Customer.ID != "" {
|
|
t.Fatal("seeded from malformed bootstrap")
|
|
}
|
|
// absent
|
|
t.Setenv("FELHOM_BOOTSTRAP_PATH", filepath.Join(dir, "nope.json"))
|
|
if got := MaybeIngest(filepath.Join(dir, "c2.yaml"), config.Default(), testLogger(), pull); got.Customer.ID != "" {
|
|
t.Fatal("seeded with no bootstrap present")
|
|
}
|
|
if pulled {
|
|
t.Fatal("pull invoked for malformed/absent bootstrap")
|
|
}
|
|
}
|