diff --git a/CHANGELOG.md b/CHANGELOG.md index a8669e4..d7599fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,36 @@ ## Changelog +### v0.94.0 — pull-based config-refresh (re-pull controller.yaml + self-restart on a config change) (2026-06-30) + +Config delivery is now pull-based, riding the report ACK exactly like the Phase 2 version floor — the hub +never connects into the box. This replaces the hub's retired "Push Config" (companion hub change v0.26.0) +and is the mechanism by which an operator config edit reaches a running box. + +- **`internal/report/pusher.go`:** `PushResponse` gains `ConfigVersion int` (`json:"config_version"`). + 0 = the hub didn't advertise it (old hub / report-only customer) → no action. +- **`internal/report/config_refresh.go` (NEW) — `ConfigRefresher.Reconcile`.** The testable reconcile + (all side effects injected): on a config_version change vs. the last-applied version, **Refresh** (re-pull + `controller.yaml`) → **Record** → **Restart**. First-ever ACK (nothing recorded) records the baseline + WITHOUT restarting (the first-boot pull already has the current config); an unchanged version is a no-op + (no restart storm); a failed pull keeps the current config and does NOT record/restart (retried next + cycle); record-before-restart so the restarted process sees it applied and doesn't loop. +- **`internal/bootstrap/bootstrap.go` — `RefreshConfig`.** Re-pulls `controller.yaml` from the hub and + rewrites it, re-merging `local_api` from the same read-only `bootstrap.json` mount (no secret stashed + elsewhere). Reuses the existing `pullWithRetry`/`mergeLocalAPI`/`writeFileAtomic`. Overwrites + `controller.yaml` (hub = source of truth); NEVER touches `settings.json`; fail-safe (any failure leaves + the current config unchanged + returns an error). NOT first-boot-gated (unlike `MaybeIngest`). +- **`internal/settings/settings.go`:** `applied_config_version` + `GetAppliedConfigVersion` / + `SetAppliedConfigVersion` (persisted so the version survives the restart). +- **`internal/api/selfrestart.go`:** exported `GracefulSelfRestart` (the unexported one now calls it) so + the main.go reconcile reuses the one graceful-restart mechanism instead of reinventing an `os.Exit`. +- **`cmd/controller/main.go`:** wires the reconcile into `OnPushResponse` beside the floor reconcile — + same report cycle, no new timer, no agent involvement. The first-boot `MaybeIngest` never-clobber is + untouched (the refresh is a separate explicit re-pull). +- Tests: `Reconcile` (change→refresh+record+restart; **same-version no-op RED-PROOF**; baseline-no-restart; + failed-pull no-record/no-restart; zero-version no-op; record-fail skips restart) + `RefreshConfig` + (re-pull overwrites + re-merges local_api; failed pull leaves config unchanged; absent bootstrap errors + without writing). `go build/vet/test ./...` green. + ### v0.93.0 — NAS Part B: off-box backup target (restic-over-SFTP) (2026-06-30) Closes the NAS arc: back the app-data tier (each off-box app's recovery unit + DB dumps + volume tars) up to the customer's NAS as an **encrypted restic repo over SFTP** — the "1 off-site" leg of 3-2-1, distinct diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 34d9e63..dc33578 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -362,6 +362,20 @@ func main() { updater.SetFloor(resp.MinControllerVersion) updater.MaybeAutoUpdate() } + // Config-refresh (v0.26.0): the ACK also carries the per-customer config_version. On a + // change vs. the last-applied version, re-pull controller.yaml (re-merging local_api from + // bootstrap.json) and self-restart so the new config loads. Pull-based — the hub never + // connects into the box. Rides this same report cycle; no new timer, no agent. First-ever + // ACK records the baseline without restarting; a failed pull keeps the current config and + // retries next cycle; an unchanged version is a no-op (so no restart storm). + cr := &report.ConfigRefresher{ + Applied: sett.GetAppliedConfigVersion, + Record: sett.SetAppliedConfigVersion, + Refresh: func() error { return bootstrap.RefreshConfig(*configPath, logger, pull) }, + Restart: func() { api.GracefulSelfRestart(logger) }, + Logger: logger, + } + cr.Reconcile(resp.ConfigVersion) } // Wire hub push status into alert manager for dashboard alerts alertMgr.SetHubPushStatus(func() web.HubPushStatusData { diff --git a/controller/internal/api/selfrestart.go b/controller/internal/api/selfrestart.go index d69d8fe..1058959 100644 --- a/controller/internal/api/selfrestart.go +++ b/controller/internal/api/selfrestart.go @@ -16,6 +16,13 @@ const restartDelay = 500 * time.Millisecond // API token) and the manual restart button actually take effect: singletons such as the // Cloudflare client are built once at startup and are not reloaded in-process. func gracefulSelfRestart(logger *log.Logger) { + GracefulSelfRestart(logger) +} + +// GracefulSelfRestart is the exported entry point to the same graceful restart, so non-api callers +// (the config-refresh reconcile wired in main.go) reuse this one mechanism instead of reinventing an +// os.Exit path. See gracefulSelfRestart for the rationale. +func GracefulSelfRestart(logger *log.Logger) { go func() { time.Sleep(restartDelay) if logger != nil { diff --git a/controller/internal/bootstrap/bootstrap.go b/controller/internal/bootstrap/bootstrap.go index 7af095a..051ea63 100644 --- a/controller/internal/bootstrap/bootstrap.go +++ b/controller/internal/bootstrap/bootstrap.go @@ -166,6 +166,61 @@ func MaybeIngest(configPath string, cfg *config.Config, logger *log.Logger, pull return reloaded } +// RefreshConfig re-pulls controller.yaml from the hub and rewrites it, re-merging the per-guest +// local_api block — the config-refresh path (v0.26.0) invoked when the report ACK's config_version +// changes. Unlike MaybeIngest it is NOT idempotent and NOT first-boot-gated: it deliberately +// OVERWRITES the existing controller.yaml (the hub is the source of truth for it). It reads the +// credentials (customer id, hub url, retrieval passphrase, local_api) from the same read-only +// bootstrap.json mount the first-boot pull uses, so no secret is stashed elsewhere. +// +// Contract (acceptance §4, rule 4): +// - Source of truth: overwrites controller.yaml; NEVER touches settings.json (local state). +// - local_api: re-merged from bootstrap.json exactly as first boot does (the hub yaml lacks it). +// - Fail-safe: any failure (absent/invalid bootstrap, missing field, hub-unreachable pull, write +// error) returns an error and leaves the current controller.yaml UNCHANGED — the caller then +// keeps the current config and does not restart. A wizard-configured guest with no bootstrap.json +// returns an error here (nothing to pull from) and is simply left as-is. +func RefreshConfig(configPath string, logger *log.Logger, pull PullFunc) error { + bpath := Path() + data, err := os.ReadFile(bpath) + if err != nil { + return fmt.Errorf("read bootstrap %s: %w", bpath, err) + } + var b Bootstrap + if err := json.Unmarshal(data, &b); err != nil { + return fmt.Errorf("bootstrap %s not valid JSON: %w", bpath, err) + } + if b.Schema != SchemaV2 { + return fmt.Errorf("bootstrap unsupported schema %q (want %q)", b.Schema, SchemaV2) + } + if b.Customer.ID == "" || b.Hub.URL == "" || b.Hub.RetrievalPassword == "" { + return fmt.Errorf("bootstrap missing customer.id / hub.url / hub.retrieval_password") + } + if b.LocalAPI.Endpoint == "" || b.LocalAPI.Fingerprint == "" || b.LocalAPI.Token == "" { + return fmt.Errorf("bootstrap missing local_api.{endpoint,fingerprint,token}") + } + if pull == nil { + return fmt.Errorf("no pull function wired") + } + + pulled, err := pullWithRetry(pull, b.Hub.URL, b.Customer.ID, b.Hub.RetrievalPassword, logger) + if err != nil { + return fmt.Errorf("hub config pull failed: %w", err) + } + merged, err := mergeLocalAPI(pulled, b.LocalAPI) + if err != nil { + return fmt.Errorf("merge local_api: %w", err) + } + if err := writeFileAtomic(configPath, merged); err != nil { + return fmt.Errorf("write %s: %w", configPath, err) + } + if logger != nil { + logger.Printf("[INFO] config-refresh: re-pulled controller.yaml from %s for %s, merged local_api (%s)", + b.Hub.URL, b.Customer.ID, b.LocalAPI.Endpoint) + } + return nil +} + // pullWithRetry calls pull once, then retries on transient (ErrPullTransient) failures only, with // the pullRetryDelays backoff. Permanent failures (anything not ErrPullTransient) fail fast. func pullWithRetry(pull PullFunc, hubURL, customerID, password string, logger *log.Logger) (string, error) { diff --git a/controller/internal/bootstrap/config_refresh_test.go b/controller/internal/bootstrap/config_refresh_test.go new file mode 100644 index 0000000..73b0e4c --- /dev/null +++ b/controller/internal/bootstrap/config_refresh_test.go @@ -0,0 +1,95 @@ +package bootstrap + +import ( + "os" + "strings" + "testing" +) + +// RefreshConfig re-pulls the hub yaml and OVERWRITES controller.yaml, re-merging local_api from +// bootstrap.json. Unlike MaybeIngest it is not first-boot-gated — it clobbers an existing config (the +// hub is the source of truth for controller.yaml). +func TestRefreshConfig_RePullsAndMerges(t *testing.T) { + dir := t.TempDir() + _, cfgPath := writeBootstrap(t, dir, goodBootstrapV2) + + // Pre-existing (stale) controller.yaml that must be overwritten. + if err := os.WriteFile(cfgPath, []byte("customer:\n id: cust-8200\n name: OLD\n"), 0o600); err != nil { + t.Fatal(err) + } + + var calls int + var gotID, gotPass string + pull := func(hubURL, customerID, pass string) (string, error) { + calls++ + gotID, gotPass = customerID, pass + return hubYAML, nil + } + + if err := RefreshConfig(cfgPath, testLogger(), pull); err != nil { + t.Fatalf("RefreshConfig: %v", err) + } + if calls != 1 { + t.Errorf("pull called %d times, want 1", calls) + } + if gotID != "cust-8200" || gotPass != "five-word-passphrase-here" { + t.Errorf("pull args = (%q,%q), want (cust-8200, five-word-passphrase-here)", gotID, gotPass) + } + + out, err := os.ReadFile(cfgPath) + if err != nil { + t.Fatal(err) + } + s := string(out) + if strings.Contains(s, "OLD") { + t.Errorf("controller.yaml still contains stale content:\n%s", s) + } + if !strings.Contains(s, "CUSTKEY_FROM_HUB") { + t.Errorf("controller.yaml missing hub-pulled api_key:\n%s", s) + } + // local_api re-merged from bootstrap.json (the hub yaml lacks it). + if !strings.Contains(s, "PERGUESTTOKEN") || !strings.Contains(s, "192.168.0.162:8443") { + t.Errorf("controller.yaml missing re-merged local_api:\n%s", s) + } +} + +// Fail-safe: a failed pull leaves the existing controller.yaml UNCHANGED and returns an error (the +// caller then keeps the current config and does not restart). +func TestRefreshConfig_FailedPullLeavesConfigUnchanged(t *testing.T) { + dir := t.TempDir() + _, cfgPath := writeBootstrap(t, dir, goodBootstrapV2) + original := "customer:\n id: cust-8200\n name: CURRENT\n" + if err := os.WriteFile(cfgPath, []byte(original), 0o600); err != nil { + t.Fatal(err) + } + + pull := func(hubURL, customerID, pass string) (string, error) { + return "", ErrPullTransient // hub unreachable + } + if err := RefreshConfig(cfgPath, testLogger(), pull); err == nil { + t.Fatal("RefreshConfig should return an error on a failed pull") + } + out, _ := os.ReadFile(cfgPath) + if string(out) != original { + t.Errorf("controller.yaml changed despite failed pull:\n%s", string(out)) + } +} + +// Fail-safe: no bootstrap.json (e.g. a wizard-configured guest) → error, nothing written. +func TestRefreshConfig_NoBootstrapErrors(t *testing.T) { + dir := t.TempDir() + cfgPath := dir + "/controller.yaml" + t.Setenv("FELHOM_BOOTSTRAP_PATH", dir+"/does-not-exist.json") + + called := false + pull := func(hubURL, customerID, pass string) (string, error) { called = true; return hubYAML, nil } + if err := RefreshConfig(cfgPath, testLogger(), pull); err == nil { + t.Fatal("RefreshConfig should error when bootstrap.json is absent") + } + if called { + t.Error("pull should not be called when bootstrap.json is absent") + } + if _, err := os.Stat(cfgPath); !os.IsNotExist(err) { + t.Error("controller.yaml should not have been written") + } +} diff --git a/controller/internal/report/config_refresh.go b/controller/internal/report/config_refresh.go new file mode 100644 index 0000000..11d9e78 --- /dev/null +++ b/controller/internal/report/config_refresh.go @@ -0,0 +1,75 @@ +package report + +import "log" + +// ConfigRefresher reconciles the hub-advertised config_version (from the report ACK) against the +// controller's last-applied version and, on a change, re-pulls controller.yaml and self-restarts — +// the pull-based config-delivery path (the hub never connects into the box; this rides the existing +// report cycle exactly like the Phase 2 version floor). +// +// All side effects are injected so the reconcile is unit-testable without a real hub / filesystem / +// process exit: +// - Applied reads the persisted last-applied config_version (0 = none recorded yet). +// - Record persists a newly-applied config_version. +// - Refresh re-pulls controller.yaml from the hub and writes it (re-merging local_api). It must +// NOT touch settings.json. A hub-unreachable / write failure returns an error. +// - Restart triggers the graceful self-restart (process exit → Docker restart → fresh config). +type ConfigRefresher struct { + Applied func() int + Record func(int) error + Refresh func() error + Restart func() + Logger *log.Logger +} + +// Reconcile applies the config-refresh decision for one report ACK. Rules (acceptance B + safety §5): +// - ackVersion == 0 → no-op (hub didn't advertise; old hub / no config row). +// - no version recorded yet → record the baseline WITHOUT restarting (first-ever ACK; the +// first-boot pull already fetched the current config). +// - ackVersion == applied → no-op (no change; this is what prevents a restart storm — after a +// refresh, applied == ackVersion so the next report is a no-op). +// - ackVersion != applied → Refresh; on success Record THEN Restart (record-before-restart so +// the post-restart process sees it applied); on a failed Refresh keep the current config, do NOT +// record, do NOT restart — retried on the next report cycle. +func (cr *ConfigRefresher) Reconcile(ackVersion int) { + if ackVersion == 0 { + return // hub didn't advertise a config_version + } + applied := cr.Applied() + if applied == 0 { + // First-ever ACK carrying a config_version: record the baseline, do NOT restart (the box came + // up on the first-boot pull, which already has the current config). Mirrors the floor's + // first-run-records-baseline. + if err := cr.Record(ackVersion); err != nil { + cr.logf("[WARN] config-refresh: failed to record baseline config_version=%d: %v", ackVersion, err) + return + } + cr.logf("[INFO] config-refresh: baseline config_version=%d recorded (no restart)", ackVersion) + return + } + if ackVersion == applied { + return // no change + } + cr.logf("[INFO] config-refresh: hub config_version=%d != applied=%d — re-pulling controller.yaml", ackVersion, applied) + if err := cr.Refresh(); err != nil { + // Fail-safe: keep the current config, do NOT record, do NOT restart — retry next cycle. + cr.logf("[WARN] config-refresh: re-pull failed: %v — keeping current config, will retry next report", err) + return + } + // Record BEFORE restarting so the freshly-started process sees the version as applied and does not + // loop. (The restart is delayed, so the record persists first.) + if err := cr.Record(ackVersion); err != nil { + cr.logf("[WARN] config-refresh: applied config but failed to record version=%d: %v — skipping restart to avoid a loop", ackVersion, err) + return + } + cr.logf("[INFO] config-refresh: applied config_version=%d — self-restarting to load it", ackVersion) + if cr.Restart != nil { + cr.Restart() + } +} + +func (cr *ConfigRefresher) logf(format string, args ...interface{}) { + if cr.Logger != nil { + cr.Logger.Printf(format, args...) + } +} diff --git a/controller/internal/report/config_refresh_test.go b/controller/internal/report/config_refresh_test.go new file mode 100644 index 0000000..5bc4bac --- /dev/null +++ b/controller/internal/report/config_refresh_test.go @@ -0,0 +1,129 @@ +package report + +import ( + "errors" + "io" + "log" + "testing" +) + +// recorder collects the side effects a Reconcile would cause, so each case asserts exactly what +// happened (pull/record/restart) without a real hub, filesystem, or process exit. +type recorder struct { + applied int + recorded []int + refreshes int + refreshErr error + restarts int + recordErr error +} + +func (r *recorder) refresher() *ConfigRefresher { + return &ConfigRefresher{ + Applied: func() int { return r.applied }, + Record: func(v int) error { + if r.recordErr != nil { + return r.recordErr + } + r.recorded = append(r.recorded, v) + r.applied = v + return nil + }, + Refresh: func() error { + r.refreshes++ + return r.refreshErr + }, + Restart: func() { r.restarts++ }, + Logger: log.New(io.Discard, "", 0), + } +} + +// A version change re-pulls, records the new version, then restarts (record BEFORE restart). +func TestReconcile_VersionChange_RefreshRecordRestart(t *testing.T) { + r := &recorder{applied: 1} + r.refresher().Reconcile(2) + + if r.refreshes != 1 { + t.Errorf("refreshes = %d, want 1", r.refreshes) + } + if len(r.recorded) != 1 || r.recorded[0] != 2 { + t.Errorf("recorded = %v, want [2]", r.recorded) + } + if r.restarts != 1 { + t.Errorf("restarts = %d, want 1", r.restarts) + } +} + +// RED-PROOF for the no-restart-storm guard: when the ACK version == the applied version, Reconcile +// must do nothing — no refresh, no restart. (Drop the `ackVersion == applied` guard in Reconcile and +// this test fails: it would refresh + restart on every report.) +func TestReconcile_SameVersion_NoOp(t *testing.T) { + r := &recorder{applied: 5} + r.refresher().Reconcile(5) + + if r.refreshes != 0 { + t.Errorf("refreshes = %d, want 0 (unchanged version must not re-pull)", r.refreshes) + } + if r.restarts != 0 { + t.Errorf("restarts = %d, want 0 (unchanged version must NOT restart — restart storm)", r.restarts) + } + if len(r.recorded) != 0 { + t.Errorf("recorded = %v, want [] (nothing to record)", r.recorded) + } +} + +// First-ever ACK (nothing recorded yet): record the baseline WITHOUT restarting or re-pulling — the +// box already came up on the first-boot pull. +func TestReconcile_FirstRun_RecordsBaselineNoRestart(t *testing.T) { + r := &recorder{applied: 0} + r.refresher().Reconcile(3) + + if r.refreshes != 0 { + t.Errorf("refreshes = %d, want 0 (baseline must not re-pull)", r.refreshes) + } + if r.restarts != 0 { + t.Errorf("restarts = %d, want 0 (baseline must not restart)", r.restarts) + } + if len(r.recorded) != 1 || r.recorded[0] != 3 { + t.Errorf("recorded = %v, want [3] (baseline recorded)", r.recorded) + } +} + +// A failed pull keeps the current config: do NOT record, do NOT restart (retried next cycle). +func TestReconcile_FailedPull_NoRecordNoRestart(t *testing.T) { + r := &recorder{applied: 1, refreshErr: errors.New("hub unreachable")} + r.refresher().Reconcile(2) + + if r.refreshes != 1 { + t.Errorf("refreshes = %d, want 1 (attempted)", r.refreshes) + } + if r.restarts != 0 { + t.Errorf("restarts = %d, want 0 (failed pull must not restart)", r.restarts) + } + if len(r.recorded) != 0 { + t.Errorf("recorded = %v, want [] (failed pull must not record — version stays so it retries)", r.recorded) + } +} + +// ackVersion == 0 (old hub / report-only customer) is a no-op. +func TestReconcile_ZeroVersion_NoOp(t *testing.T) { + r := &recorder{applied: 4} + r.refresher().Reconcile(0) + if r.refreshes != 0 || r.restarts != 0 || len(r.recorded) != 0 { + t.Errorf("zero version should be a no-op; got refreshes=%d restarts=%d recorded=%v", r.refreshes, r.restarts, r.recorded) + } +} + +// If recording the applied version fails after a successful pull, skip the restart (avoid a loop: +// a restart without a recorded version would re-pull + restart forever). +func TestReconcile_RecordFails_SkipsRestart(t *testing.T) { + r := &recorder{applied: 1, recordErr: errors.New("disk full")} + r.refresher().Reconcile(2) + + if r.refreshes != 1 { + t.Errorf("refreshes = %d, want 1", r.refreshes) + } + if r.restarts != 0 { + t.Errorf("restarts = %d, want 0 (must not restart if the version couldn't be recorded)", r.restarts) + } +} diff --git a/controller/internal/report/pusher.go b/controller/internal/report/pusher.go index afe1050..9c6eb88 100644 --- a/controller/internal/report/pusher.go +++ b/controller/internal/report/pusher.go @@ -32,6 +32,11 @@ type PushResponse struct { // "update to latest" button — never the auto-target). MinControllerVersion string `json:"min_controller_version"` LatestVersion string `json:"latest_version"` + // ConfigVersion is the hub's per-customer config counter (v0.26.0). On a change vs. the + // last-applied version, the controller re-pulls controller.yaml + self-restarts (pull-based config + // delivery — the hub never connects into the box). 0 = the hub didn't advertise it (old hub, or a + // report-only customer with no config row) → the controller does nothing. + ConfigVersion int `json:"config_version"` } // Pusher sends reports to the central hub. diff --git a/controller/internal/settings/settings.go b/controller/internal/settings/settings.go index ed6f984..9d189a8 100644 --- a/controller/internal/settings/settings.go +++ b/controller/internal/settings/settings.go @@ -53,6 +53,11 @@ type Settings struct { HubVerifiedAt string `json:"hub_verified_at,omitempty"` // RFC3339 HubLastCheck string `json:"hub_last_check,omitempty"` // RFC3339 + // AppliedConfigVersion is the hub config_version this controller has last pulled + applied + // (v0.26.0 pull-based config-refresh). 0 = none recorded yet → the first report ACK records the + // baseline without restarting. A change vs. the ACK triggers a re-pull + self-restart. + AppliedConfigVersion int `json:"applied_config_version,omitempty"` + // Recovery credentials (saved from setup wizard input) RetrievalPassword string `json:"retrieval_password,omitempty"` @@ -1110,6 +1115,22 @@ func (s *Settings) SetHubVerified(verified bool, at time.Time) error { return s.save() } +// GetAppliedConfigVersion returns the last-applied hub config_version (0 = none recorded yet). +func (s *Settings) GetAppliedConfigVersion() int { + s.mu.RLock() + defer s.mu.RUnlock() + return s.AppliedConfigVersion +} + +// SetAppliedConfigVersion persists the config_version this controller has pulled + applied. Recorded +// BEFORE a config-refresh self-restart so the restarted process sees it applied and does not loop. +func (s *Settings) SetAppliedConfigVersion(v int) error { + s.mu.Lock() + defer s.mu.Unlock() + s.AppliedConfigVersion = v + return s.save() +} + // SetHubLastCheck updates the last Hub check timestamp without changing verification status. // GetLastGuestBootID returns the persisted last-seen guest boot-id ("" if never recorded). func (s *Settings) GetLastGuestBootID() string {