controller v0.94.0: pull-based config-refresh (re-pull + self-restart on config_version change)

PushResponse.ConfigVersion from the report ACK; ConfigRefresher reconciles vs.
the last-applied version (settings.applied_config_version) and on a change calls
bootstrap.RefreshConfig (re-pull controller.yaml + re-merge local_api) then
GracefulSelfRestart. First-run records baseline (no restart); unchanged = no-op
(no storm); failed pull keeps config + retries. Companion to hub v0.26.0.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HxLA1mZurFq9kt8hneFeCs

controller/cmd/controller/main.go

@@ -362,6 +362,20 @@ func main() {
 				updater.SetFloor(resp.MinControllerVersion)
 				updater.MaybeAutoUpdate()
 			}
+			// Config-refresh (v0.26.0): the ACK also carries the per-customer config_version. On a
+			// change vs. the last-applied version, re-pull controller.yaml (re-merging local_api from
+			// bootstrap.json) and self-restart so the new config loads. Pull-based — the hub never
+			// connects into the box. Rides this same report cycle; no new timer, no agent. First-ever
+			// ACK records the baseline without restarting; a failed pull keeps the current config and
+			// retries next cycle; an unchanged version is a no-op (so no restart storm).
+			cr := &report.ConfigRefresher{
+				Applied: sett.GetAppliedConfigVersion,
+				Record:  sett.SetAppliedConfigVersion,
+				Refresh: func() error { return bootstrap.RefreshConfig(*configPath, logger, pull) },
+				Restart: func() { api.GracefulSelfRestart(logger) },
+				Logger:  logger,
+			}
+			cr.Reconcile(resp.ConfigVersion)
 		}
 		// Wire hub push status into alert manager for dashboard alerts
 		alertMgr.SetHubPushStatus(func() web.HubPushStatusData {

controller/internal/api/selfrestart.go

@@ -16,6 +16,13 @@ const restartDelay = 500 * time.Millisecond
 // API token) and the manual restart button actually take effect: singletons such as the
 // Cloudflare client are built once at startup and are not reloaded in-process.
 func gracefulSelfRestart(logger *log.Logger) {
+	GracefulSelfRestart(logger)
+}
+
+// GracefulSelfRestart is the exported entry point to the same graceful restart, so non-api callers
+// (the config-refresh reconcile wired in main.go) reuse this one mechanism instead of reinventing an
+// os.Exit path. See gracefulSelfRestart for the rationale.
+func GracefulSelfRestart(logger *log.Logger) {
 	go func() {
 		time.Sleep(restartDelay)
 		if logger != nil {

controller/internal/bootstrap/bootstrap.go

@@ -166,6 +166,61 @@ func MaybeIngest(configPath string, cfg *config.Config, logger *log.Logger, pull
 	return reloaded
 }
 
+// RefreshConfig re-pulls controller.yaml from the hub and rewrites it, re-merging the per-guest
+// local_api block — the config-refresh path (v0.26.0) invoked when the report ACK's config_version
+// changes. Unlike MaybeIngest it is NOT idempotent and NOT first-boot-gated: it deliberately
+// OVERWRITES the existing controller.yaml (the hub is the source of truth for it). It reads the
+// credentials (customer id, hub url, retrieval passphrase, local_api) from the same read-only
+// bootstrap.json mount the first-boot pull uses, so no secret is stashed elsewhere.
+//
+// Contract (acceptance §4, rule 4):
+//   - Source of truth: overwrites controller.yaml; NEVER touches settings.json (local state).
+//   - local_api: re-merged from bootstrap.json exactly as first boot does (the hub yaml lacks it).
+//   - Fail-safe: any failure (absent/invalid bootstrap, missing field, hub-unreachable pull, write
+//     error) returns an error and leaves the current controller.yaml UNCHANGED — the caller then
+//     keeps the current config and does not restart. A wizard-configured guest with no bootstrap.json
+//     returns an error here (nothing to pull from) and is simply left as-is.
+func RefreshConfig(configPath string, logger *log.Logger, pull PullFunc) error {
+	bpath := Path()
+	data, err := os.ReadFile(bpath)
+	if err != nil {
+		return fmt.Errorf("read bootstrap %s: %w", bpath, err)
+	}
+	var b Bootstrap
+	if err := json.Unmarshal(data, &b); err != nil {
+		return fmt.Errorf("bootstrap %s not valid JSON: %w", bpath, err)
+	}
+	if b.Schema != SchemaV2 {
+		return fmt.Errorf("bootstrap unsupported schema %q (want %q)", b.Schema, SchemaV2)
+	}
+	if b.Customer.ID == "" || b.Hub.URL == "" || b.Hub.RetrievalPassword == "" {
+		return fmt.Errorf("bootstrap missing customer.id / hub.url / hub.retrieval_password")
+	}
+	if b.LocalAPI.Endpoint == "" || b.LocalAPI.Fingerprint == "" || b.LocalAPI.Token == "" {
+		return fmt.Errorf("bootstrap missing local_api.{endpoint,fingerprint,token}")
+	}
+	if pull == nil {
+		return fmt.Errorf("no pull function wired")
+	}
+
+	pulled, err := pullWithRetry(pull, b.Hub.URL, b.Customer.ID, b.Hub.RetrievalPassword, logger)
+	if err != nil {
+		return fmt.Errorf("hub config pull failed: %w", err)
+	}
+	merged, err := mergeLocalAPI(pulled, b.LocalAPI)
+	if err != nil {
+		return fmt.Errorf("merge local_api: %w", err)
+	}
+	if err := writeFileAtomic(configPath, merged); err != nil {
+		return fmt.Errorf("write %s: %w", configPath, err)
+	}
+	if logger != nil {
+		logger.Printf("[INFO] config-refresh: re-pulled controller.yaml from %s for %s, merged local_api (%s)",
+			b.Hub.URL, b.Customer.ID, b.LocalAPI.Endpoint)
+	}
+	return nil
+}
+
 // pullWithRetry calls pull once, then retries on transient (ErrPullTransient) failures only, with
 // the pullRetryDelays backoff. Permanent failures (anything not ErrPullTransient) fail fast.
 func pullWithRetry(pull PullFunc, hubURL, customerID, password string, logger *log.Logger) (string, error) {

controller/internal/bootstrap/config_refresh_test.go

@@ -0,0 +1,95 @@
+package bootstrap
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+// RefreshConfig re-pulls the hub yaml and OVERWRITES controller.yaml, re-merging local_api from
+// bootstrap.json. Unlike MaybeIngest it is not first-boot-gated — it clobbers an existing config (the
+// hub is the source of truth for controller.yaml).
+func TestRefreshConfig_RePullsAndMerges(t *testing.T) {
+	dir := t.TempDir()
+	_, cfgPath := writeBootstrap(t, dir, goodBootstrapV2)
+
+	// Pre-existing (stale) controller.yaml that must be overwritten.
+	if err := os.WriteFile(cfgPath, []byte("customer:\n  id: cust-8200\n  name: OLD\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	var calls int
+	var gotID, gotPass string
+	pull := func(hubURL, customerID, pass string) (string, error) {
+		calls++
+		gotID, gotPass = customerID, pass
+		return hubYAML, nil
+	}
+
+	if err := RefreshConfig(cfgPath, testLogger(), pull); err != nil {
+		t.Fatalf("RefreshConfig: %v", err)
+	}
+	if calls != 1 {
+		t.Errorf("pull called %d times, want 1", calls)
+	}
+	if gotID != "cust-8200" || gotPass != "five-word-passphrase-here" {
+		t.Errorf("pull args = (%q,%q), want (cust-8200, five-word-passphrase-here)", gotID, gotPass)
+	}
+
+	out, err := os.ReadFile(cfgPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(out)
+	if strings.Contains(s, "OLD") {
+		t.Errorf("controller.yaml still contains stale content:\n%s", s)
+	}
+	if !strings.Contains(s, "CUSTKEY_FROM_HUB") {
+		t.Errorf("controller.yaml missing hub-pulled api_key:\n%s", s)
+	}
+	// local_api re-merged from bootstrap.json (the hub yaml lacks it).
+	if !strings.Contains(s, "PERGUESTTOKEN") || !strings.Contains(s, "192.168.0.162:8443") {
+		t.Errorf("controller.yaml missing re-merged local_api:\n%s", s)
+	}
+}
+
+// Fail-safe: a failed pull leaves the existing controller.yaml UNCHANGED and returns an error (the
+// caller then keeps the current config and does not restart).
+func TestRefreshConfig_FailedPullLeavesConfigUnchanged(t *testing.T) {
+	dir := t.TempDir()
+	_, cfgPath := writeBootstrap(t, dir, goodBootstrapV2)
+	original := "customer:\n  id: cust-8200\n  name: CURRENT\n"
+	if err := os.WriteFile(cfgPath, []byte(original), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	pull := func(hubURL, customerID, pass string) (string, error) {
+		return "", ErrPullTransient // hub unreachable
+	}
+	if err := RefreshConfig(cfgPath, testLogger(), pull); err == nil {
+		t.Fatal("RefreshConfig should return an error on a failed pull")
+	}
+	out, _ := os.ReadFile(cfgPath)
+	if string(out) != original {
+		t.Errorf("controller.yaml changed despite failed pull:\n%s", string(out))
+	}
+}
+
+// Fail-safe: no bootstrap.json (e.g. a wizard-configured guest) → error, nothing written.
+func TestRefreshConfig_NoBootstrapErrors(t *testing.T) {
+	dir := t.TempDir()
+	cfgPath := dir + "/controller.yaml"
+	t.Setenv("FELHOM_BOOTSTRAP_PATH", dir+"/does-not-exist.json")
+
+	called := false
+	pull := func(hubURL, customerID, pass string) (string, error) { called = true; return hubYAML, nil }
+	if err := RefreshConfig(cfgPath, testLogger(), pull); err == nil {
+		t.Fatal("RefreshConfig should error when bootstrap.json is absent")
+	}
+	if called {
+		t.Error("pull should not be called when bootstrap.json is absent")
+	}
+	if _, err := os.Stat(cfgPath); !os.IsNotExist(err) {
+		t.Error("controller.yaml should not have been written")
+	}
+}

controller/internal/report/config_refresh.go

@@ -0,0 +1,75 @@
+package report
+
+import "log"
+
+// ConfigRefresher reconciles the hub-advertised config_version (from the report ACK) against the
+// controller's last-applied version and, on a change, re-pulls controller.yaml and self-restarts —
+// the pull-based config-delivery path (the hub never connects into the box; this rides the existing
+// report cycle exactly like the Phase 2 version floor).
+//
+// All side effects are injected so the reconcile is unit-testable without a real hub / filesystem /
+// process exit:
+//   - Applied  reads the persisted last-applied config_version (0 = none recorded yet).
+//   - Record   persists a newly-applied config_version.
+//   - Refresh  re-pulls controller.yaml from the hub and writes it (re-merging local_api). It must
+//              NOT touch settings.json. A hub-unreachable / write failure returns an error.
+//   - Restart  triggers the graceful self-restart (process exit → Docker restart → fresh config).
+type ConfigRefresher struct {
+	Applied func() int
+	Record  func(int) error
+	Refresh func() error
+	Restart func()
+	Logger  *log.Logger
+}
+
+// Reconcile applies the config-refresh decision for one report ACK. Rules (acceptance B + safety §5):
+//   - ackVersion == 0           → no-op (hub didn't advertise; old hub / no config row).
+//   - no version recorded yet   → record the baseline WITHOUT restarting (first-ever ACK; the
+//     first-boot pull already fetched the current config).
+//   - ackVersion == applied     → no-op (no change; this is what prevents a restart storm — after a
+//     refresh, applied == ackVersion so the next report is a no-op).
+//   - ackVersion != applied     → Refresh; on success Record THEN Restart (record-before-restart so
+//     the post-restart process sees it applied); on a failed Refresh keep the current config, do NOT
+//     record, do NOT restart — retried on the next report cycle.
+func (cr *ConfigRefresher) Reconcile(ackVersion int) {
+	if ackVersion == 0 {
+		return // hub didn't advertise a config_version
+	}
+	applied := cr.Applied()
+	if applied == 0 {
+		// First-ever ACK carrying a config_version: record the baseline, do NOT restart (the box came
+		// up on the first-boot pull, which already has the current config). Mirrors the floor's
+		// first-run-records-baseline.
+		if err := cr.Record(ackVersion); err != nil {
+			cr.logf("[WARN] config-refresh: failed to record baseline config_version=%d: %v", ackVersion, err)
+			return
+		}
+		cr.logf("[INFO] config-refresh: baseline config_version=%d recorded (no restart)", ackVersion)
+		return
+	}
+	if ackVersion == applied {
+		return // no change
+	}
+	cr.logf("[INFO] config-refresh: hub config_version=%d != applied=%d — re-pulling controller.yaml", ackVersion, applied)
+	if err := cr.Refresh(); err != nil {
+		// Fail-safe: keep the current config, do NOT record, do NOT restart — retry next cycle.
+		cr.logf("[WARN] config-refresh: re-pull failed: %v — keeping current config, will retry next report", err)
+		return
+	}
+	// Record BEFORE restarting so the freshly-started process sees the version as applied and does not
+	// loop. (The restart is delayed, so the record persists first.)
+	if err := cr.Record(ackVersion); err != nil {
+		cr.logf("[WARN] config-refresh: applied config but failed to record version=%d: %v — skipping restart to avoid a loop", ackVersion, err)
+		return
+	}
+	cr.logf("[INFO] config-refresh: applied config_version=%d — self-restarting to load it", ackVersion)
+	if cr.Restart != nil {
+		cr.Restart()
+	}
+}
+
+func (cr *ConfigRefresher) logf(format string, args ...interface{}) {
+	if cr.Logger != nil {
+		cr.Logger.Printf(format, args...)
+	}
+}

controller/internal/report/config_refresh_test.go

@@ -0,0 +1,129 @@
+package report
+
+import (
+	"errors"
+	"io"
+	"log"
+	"testing"
+)
+
+// recorder collects the side effects a Reconcile would cause, so each case asserts exactly what
+// happened (pull/record/restart) without a real hub, filesystem, or process exit.
+type recorder struct {
+	applied    int
+	recorded   []int
+	refreshes  int
+	refreshErr error
+	restarts   int
+	recordErr  error
+}
+
+func (r *recorder) refresher() *ConfigRefresher {
+	return &ConfigRefresher{
+		Applied: func() int { return r.applied },
+		Record: func(v int) error {
+			if r.recordErr != nil {
+				return r.recordErr
+			}
+			r.recorded = append(r.recorded, v)
+			r.applied = v
+			return nil
+		},
+		Refresh: func() error {
+			r.refreshes++
+			return r.refreshErr
+		},
+		Restart: func() { r.restarts++ },
+		Logger:  log.New(io.Discard, "", 0),
+	}
+}
+
+// A version change re-pulls, records the new version, then restarts (record BEFORE restart).
+func TestReconcile_VersionChange_RefreshRecordRestart(t *testing.T) {
+	r := &recorder{applied: 1}
+	r.refresher().Reconcile(2)
+
+	if r.refreshes != 1 {
+		t.Errorf("refreshes = %d, want 1", r.refreshes)
+	}
+	if len(r.recorded) != 1 || r.recorded[0] != 2 {
+		t.Errorf("recorded = %v, want [2]", r.recorded)
+	}
+	if r.restarts != 1 {
+		t.Errorf("restarts = %d, want 1", r.restarts)
+	}
+}
+
+// RED-PROOF for the no-restart-storm guard: when the ACK version == the applied version, Reconcile
+// must do nothing — no refresh, no restart. (Drop the `ackVersion == applied` guard in Reconcile and
+// this test fails: it would refresh + restart on every report.)
+func TestReconcile_SameVersion_NoOp(t *testing.T) {
+	r := &recorder{applied: 5}
+	r.refresher().Reconcile(5)
+
+	if r.refreshes != 0 {
+		t.Errorf("refreshes = %d, want 0 (unchanged version must not re-pull)", r.refreshes)
+	}
+	if r.restarts != 0 {
+		t.Errorf("restarts = %d, want 0 (unchanged version must NOT restart — restart storm)", r.restarts)
+	}
+	if len(r.recorded) != 0 {
+		t.Errorf("recorded = %v, want [] (nothing to record)", r.recorded)
+	}
+}
+
+// First-ever ACK (nothing recorded yet): record the baseline WITHOUT restarting or re-pulling — the
+// box already came up on the first-boot pull.
+func TestReconcile_FirstRun_RecordsBaselineNoRestart(t *testing.T) {
+	r := &recorder{applied: 0}
+	r.refresher().Reconcile(3)
+
+	if r.refreshes != 0 {
+		t.Errorf("refreshes = %d, want 0 (baseline must not re-pull)", r.refreshes)
+	}
+	if r.restarts != 0 {
+		t.Errorf("restarts = %d, want 0 (baseline must not restart)", r.restarts)
+	}
+	if len(r.recorded) != 1 || r.recorded[0] != 3 {
+		t.Errorf("recorded = %v, want [3] (baseline recorded)", r.recorded)
+	}
+}
+
+// A failed pull keeps the current config: do NOT record, do NOT restart (retried next cycle).
+func TestReconcile_FailedPull_NoRecordNoRestart(t *testing.T) {
+	r := &recorder{applied: 1, refreshErr: errors.New("hub unreachable")}
+	r.refresher().Reconcile(2)
+
+	if r.refreshes != 1 {
+		t.Errorf("refreshes = %d, want 1 (attempted)", r.refreshes)
+	}
+	if r.restarts != 0 {
+		t.Errorf("restarts = %d, want 0 (failed pull must not restart)", r.restarts)
+	}
+	if len(r.recorded) != 0 {
+		t.Errorf("recorded = %v, want [] (failed pull must not record — version stays so it retries)", r.recorded)
+	}
+}
+
+// ackVersion == 0 (old hub / report-only customer) is a no-op.
+func TestReconcile_ZeroVersion_NoOp(t *testing.T) {
+	r := &recorder{applied: 4}
+	r.refresher().Reconcile(0)
+	if r.refreshes != 0 || r.restarts != 0 || len(r.recorded) != 0 {
+		t.Errorf("zero version should be a no-op; got refreshes=%d restarts=%d recorded=%v", r.refreshes, r.restarts, r.recorded)
+	}
+}
+
+// If recording the applied version fails after a successful pull, skip the restart (avoid a loop:
+// a restart without a recorded version would re-pull + restart forever).
+func TestReconcile_RecordFails_SkipsRestart(t *testing.T) {
+	r := &recorder{applied: 1, recordErr: errors.New("disk full")}
+	r.refresher().Reconcile(2)
+
+	if r.refreshes != 1 {
+		t.Errorf("refreshes = %d, want 1", r.refreshes)
+	}
+	if r.restarts != 0 {
+		t.Errorf("restarts = %d, want 0 (must not restart if the version couldn't be recorded)", r.restarts)
+	}
+}

controller/internal/report/pusher.go

@@ -32,6 +32,11 @@ type PushResponse struct {
 	// "update to latest" button — never the auto-target).
 	MinControllerVersion string `json:"min_controller_version"`
 	LatestVersion        string `json:"latest_version"`
+	// ConfigVersion is the hub's per-customer config counter (v0.26.0). On a change vs. the
+	// last-applied version, the controller re-pulls controller.yaml + self-restarts (pull-based config
+	// delivery — the hub never connects into the box). 0 = the hub didn't advertise it (old hub, or a
+	// report-only customer with no config row) → the controller does nothing.
+	ConfigVersion int `json:"config_version"`
 }
 
 // Pusher sends reports to the central hub.

controller/internal/settings/settings.go

@@ -53,6 +53,11 @@ type Settings struct {
 	HubVerifiedAt string `json:"hub_verified_at,omitempty"` // RFC3339
 	HubLastCheck  string `json:"hub_last_check,omitempty"`  // RFC3339
 
+	// AppliedConfigVersion is the hub config_version this controller has last pulled + applied
+	// (v0.26.0 pull-based config-refresh). 0 = none recorded yet → the first report ACK records the
+	// baseline without restarting. A change vs. the ACK triggers a re-pull + self-restart.
+	AppliedConfigVersion int `json:"applied_config_version,omitempty"`
+
 	// Recovery credentials (saved from setup wizard input)
 	RetrievalPassword string `json:"retrieval_password,omitempty"`
 
@@ -1110,6 +1115,22 @@ func (s *Settings) SetHubVerified(verified bool, at time.Time) error {
 	return s.save()
 }
 
+// GetAppliedConfigVersion returns the last-applied hub config_version (0 = none recorded yet).
+func (s *Settings) GetAppliedConfigVersion() int {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.AppliedConfigVersion
+}
+
+// SetAppliedConfigVersion persists the config_version this controller has pulled + applied. Recorded
+// BEFORE a config-refresh self-restart so the restarted process sees it applied and does not loop.
+func (s *Settings) SetAppliedConfigVersion(v int) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.AppliedConfigVersion = v
+	return s.save()
+}
+
 // SetHubLastCheck updates the last Hub check timestamp without changing verification status.
 // GetLastGuestBootID returns the persisted last-seen guest boot-id ("" if never recorded).
 func (s *Settings) GetLastGuestBootID() string {