diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 6a8d173..206c464 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,5 +1,45 @@ # Felhom Hub — Changelog +## v0.58.0 — Direction-2 immediate-sync: the hub→box "sync now" wait channel (2026-07-16) + +Implements option (b) of `documentation/audits/SPIKE-immediate-sync-transport-2026-07-16.md`: an +operator action on the hub now reaches the box in **seconds** instead of on the next ~15-min report +cycle. The box holds a hanging authenticated `GET /api/v1/wait` over the existing outbound ingress; +the hub completes it the instant any operator intent lands for that customer. The box then fires its +ordinary out-of-cycle report — the ACK delivers config/escrow/claim/floor through the UNCHANGED +machinery. The 15-min cycle stays the reconciliation backbone; every wait failure degrades to it. +Pairs with controller v0.140.0 (the long-poll client). Ground truth #1 holds: the box pulls even the +wake-up; the hub never connects inbound and no state ever rides the wait response. + +- **`internal/intent` — the in-memory operator-intent notifier.** A per-customer generation counter + with a waiter registry: `Bump(customerID)` advances the generation and wakes every registered + waiter (a burst coalesces into ONE completion carrying the LATEST generation — a counter, not a + per-bump queue); `Wait(ctx, customerID, lastSeen, maxHold)` returns the instant the generation + differs from `lastSeen`, on ctx-cancel, on `maxHold`, or on `Close`. A pre-register gen-check + closes the bump-before-connect race (a bump is never lost). In-memory BY DESIGN — a hub restart + resets generations; the box compares with `!=`, so a restart costs exactly one harmless full-state + report, never a storm. No persistence, no store schema. Red-proofs: counter-vs-queue (return the + as-of-register snapshot → `TestWait_CoalescesBurstToLatestGen` fails) and the race-closer (drop the + pre-register check → `TestWait_RaceCloser_BumpBeforeWaitNotLost` fails); both run-fail-reverted. +- **`GET /api/v1/wait` (api/wait.go).** Authed via `checkAuthCustomer`; per-customer only (a global + operator key → 400; the customer is resolved from the key, no `customer_id` parameter is accepted, + so A can never observe B). Holds up to **240 s**, writing a **25 s heartbeat newline** while it + waits. nginx's `proxy_read_timeout` is measured BETWEEN upstream reads, so the heartbeat keeps the + default 60 s from ever firing — **no ingress annotation / manifest timeout change is needed** (the + transport spike measured that ceiling; §13 proves the heartbeat defeats it live). The response is + contentless — a single `{"gen":N}` line. The connection's write deadline is lifted per-request via + `http.NewResponseController().SetWriteDeadline` (the global `http.Server.WriteTimeout` of 60 s is + deliberately untouched). +- **Intent bumps (web).** Every operator-intent handler bumps the customer's generation AFTER its + successful store write (fire-after-commit, never on an error path), via nil-safe `s.bumpIntent`: + config create/update/delete, claim resend, offsite re-issue (UI + the re-enroll seam), offsite + freeze/unfreeze, retrieval-password regen, block/unblock, per-customer floor, global floor (bumps + every config-managed customer), controller log-tail request, and the CONTROLLER log-bundle request + (the AGENT ring rides the heartbeat envelope — a separate plane, deliberately not bumped). +- **Wiring + shutdown.** One `intent.New()` in `main.go`, injected into both the web server and the + API handler; `intentHub.Close()` runs before `server.Shutdown` so held waits complete instantly + instead of eating the 15 s grace window. nil-safe throughout (an unset hub → wait 503, bumps no-op). + ## v0.57.0 — reinstall-of-existing-customer arc: claim continuity, offsite re-issue, escrow honesty (2026-07-16) Closes the N100 physical-run findings F2/F3 and the correctness edge behind F4→2.3 diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 10c9861..3e04f82 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -18,6 +18,7 @@ import ( "gitea.dooplex.hu/admin/felhom-hub/internal/claim" "gitea.dooplex.hu/admin/felhom-hub/internal/gitea" "gitea.dooplex.hu/admin/felhom-hub/internal/hetznerapi" + "gitea.dooplex.hu/admin/felhom-hub/internal/intent" "gitea.dooplex.hu/admin/felhom-hub/internal/mailrelay" "gitea.dooplex.hu/admin/felhom-hub/internal/offsite" "gitea.dooplex.hu/admin/felhom-hub/internal/pbsdrheal" @@ -308,6 +309,15 @@ func main() { // unconfigured or the customer has no offsite tier. apiHandler.SetOffsiteReissuer(webServer.ReissueOffsiteForCustomer) + // Direction-2 immediate-sync (v0.58.0): one in-memory operator-intent notifier, shared by the + // web handlers (which Bump it after every intent write) and the API handler (which long-polls it + // at GET /api/v1/wait). In-memory BY DESIGN — a restart resets generations to zero; the box + // compares with != , so a restart costs exactly one harmless full-state report. Closed before + // server.Shutdown (below) so held waits complete instantly instead of eating the grace window. + intentHub := intent.New() + apiHandler.SetIntentHub(intentHub) + webServer.SetIntentHub(intentHub) + // Build HTTP mux mux := http.NewServeMux() @@ -515,6 +525,9 @@ func main() { logger.Printf("[INFO] Received signal %v, shutting down...", sig) cancel() + // Complete every held long-poll before Shutdown so they don't consume the grace window. + intentHub.Close() + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 15*time.Second) defer shutdownCancel() diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index d2cdd05..7782fd8 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -17,6 +17,7 @@ import ( "gitea.dooplex.hu/admin/felhom-hub/internal/assets" "gitea.dooplex.hu/admin/felhom-hub/internal/claim" "gitea.dooplex.hu/admin/felhom-hub/internal/configgen" + "gitea.dooplex.hu/admin/felhom-hub/internal/intent" "gitea.dooplex.hu/admin/felhom-hub/internal/mailrelay" "gitea.dooplex.hu/admin/felhom-hub/internal/notify" "gitea.dooplex.hu/admin/felhom-hub/internal/store" @@ -72,6 +73,11 @@ type Handler struct { // as the manual "Re-issue offsite credentials" button, so escrow invalidation + events ride // along). nil = no auto re-issue; a no-op when offsite isn't provisioned/enabled for the customer. offsiteReissuer func(ctx context.Context, customerID string) error + + // intentHub (v0.58.0, Direction-2 immediate-sync) is the in-memory per-customer generation + // notifier that GET /api/v1/wait long-polls against. nil = wait endpoint returns 503 (the box + // falls back to the 15-min cycle). Shared with the web server, whose intent handlers Bump it. + intentHub *intent.Hub } // SetClaimEngine wires the customer-claim code engine (nil-safe everywhere it is used). @@ -95,6 +101,12 @@ func (h *Handler) SetLatestVersionProvider(p LatestVersionProvider) { h.latestVersion = p } +// SetIntentHub wires the operator-intent notifier for GET /api/v1/wait (v0.58.0; nil-safe — an +// unset hub makes the wait endpoint return 503). +func (h *Handler) SetIntentHub(hub *intent.Hub) { + h.intentHub = hub +} + // New creates a new API handler. func New(store *store.Store, apiKey, resendAPIKey, fromEmail string, templateProvider ConfigTemplateProvider, logger *log.Logger) *Handler { return &Handler{ @@ -180,6 +192,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPost && path == "/report": h.handleReport(w, r) + // Direction-2 immediate-sync (v0.58.0): the box long-polls here; the hub completes it on any + // operator-intent bump for the box's customer, then the box fires its ordinary report. + case r.Method == http.MethodGet && path == "/wait": + h.handleWait(w, r) case r.Method == http.MethodPost && path == "/host-report": h.handleHostReport(w, r) case r.Method == http.MethodPost && path == "/host-enroll": diff --git a/hub/internal/api/wait.go b/hub/internal/api/wait.go new file mode 100644 index 0000000..64e5a04 --- /dev/null +++ b/hub/internal/api/wait.go @@ -0,0 +1,105 @@ +package api + +import ( + "fmt" + "net/http" + "strconv" + "time" +) + +// waitMaxHold / waitHeartbeat are effectively constants — vars only so tests can shrink them. +var ( + // waitMaxHold is how long the hub holds a wait before completing it with the (unchanged) + // generation. Kept well under the ingress/NAT idle ceilings the transport spike measured + // (SPIKE-immediate-sync-transport-2026-07-16.md): the nginx 60 s proxy_read_timeout is defeated + // by the heartbeat below, and the operator NAT held an idle hold > 600 s. 240 s → a box + // reconnects roughly every 4 min in steady state (negligible fleet load). + waitMaxHold = 240 * time.Second + // waitHeartbeat is the spacing of keepalive newline bytes written while holding. nginx's + // proxy_read_timeout is measured BETWEEN successive reads from the upstream, so a byte every + // 25 s keeps the default 60 s from ever firing — no ingress annotation / manifest change is + // needed (§13 proves this live; the annotation is the documented fallback if it doesn't hold). + // 25 s mirrors the WG PersistentKeepalive cadence and doubles as NAT keepalive. + waitHeartbeat = 25 * time.Second +) + +// handleWait is the Direction-2 immediate-sync long-poll (option b of the transport spike). A box +// holds this authenticated GET carrying its last-seen generation (?gen=N); the hub completes it the +// instant any operator intent lands for the box's customer (intent.Hub.Bump from the web handlers), +// or after waitMaxHold, whichever comes first. The response is CONTENTLESS — a single {"gen":N} +// line preceded by heartbeat newlines. The box reacts by firing its ordinary out-of-cycle report; +// the report ACK then delivers config/escrow/claim/floor through the UNCHANGED machinery. The hub +// never connects inbound and never rides state on this channel (ground truth #1). +func (h *Handler) handleWait(w http.ResponseWriter, r *http.Request) { + customerID, isGlobal, ok := h.checkAuthCustomer(r) + if !ok { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + // Per-customer only: the key RESOLVES the customer and no customer_id parameter is accepted, + // so customer A can never observe B's generation. A global operator key has no single customer + // to wait on → 400 (the controller always holds a per-customer key). + if isGlobal { + http.Error(w, "wait is per-customer (present a customer key)", http.StatusBadRequest) + return + } + if h.intentHub == nil { + http.Error(w, "wait channel unavailable", http.StatusServiceUnavailable) + return + } + + var lastSeen uint64 + if s := r.URL.Query().Get("gen"); s != "" { + if v, err := strconv.ParseUint(s, 10, 64); err == nil { + lastSeen = v + } + } + + // Lift THIS connection's write deadline past the hold. The global http.Server.WriteTimeout is + // 60 s (main.go — deliberately untouched); http.NewResponseController overrides it per-connection + // (Go 1.20+). Without this the held response is cut at 60 s regardless of the ingress. + rc := http.NewResponseController(w) + _ = rc.SetWriteDeadline(time.Now().Add(waitMaxHold + 30*time.Second)) + + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("X-Accel-Buffering", "no") // ask nginx not to buffer the streamed hold + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + if err := rc.Flush(); err != nil { + return // client already gone + } + + deadline := time.Now().Add(waitMaxHold) + for { + remaining := time.Until(deadline) + if remaining <= 0 { + // Hold elapsed with no change: complete with the current generation (Scenario B). + h.writeWaitGen(w, rc, h.intentHub.Generation(customerID)) + return + } + sub := remaining + if sub > waitHeartbeat { + sub = waitHeartbeat + } + gen := h.intentHub.Wait(r.Context(), customerID, lastSeen, sub) + if gen != lastSeen { + h.writeWaitGen(w, rc, gen) // intent moved → complete immediately + return + } + // No change within this heartbeat window: emit a keepalive byte so the ingress read-timeout + // never fires. A write/flush failure means the client is gone — return quietly (no log + // noise, no goroutine leak; the intent.Wait above already deregistered). + if _, err := w.Write([]byte("\n")); err != nil { + return + } + if err := rc.Flush(); err != nil { + return + } + } +} + +// writeWaitGen emits the single contentless completion line and flushes. +func (h *Handler) writeWaitGen(w http.ResponseWriter, rc *http.ResponseController, gen uint64) { + fmt.Fprintf(w, "{\"gen\":%d}\n", gen) + _ = rc.Flush() +} diff --git a/hub/internal/api/wait_test.go b/hub/internal/api/wait_test.go new file mode 100644 index 0000000..fd98a53 --- /dev/null +++ b/hub/internal/api/wait_test.go @@ -0,0 +1,183 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/intent" + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// shrinkWaitTiming shrinks the hold/heartbeat for the duration of a test (restored on cleanup) so +// long-poll tests run in milliseconds instead of the 240 s production hold. +func shrinkWaitTiming(t *testing.T, maxHold, heartbeat time.Duration) { + t.Helper() + om, oh := waitMaxHold, waitHeartbeat + waitMaxHold, waitHeartbeat = maxHold, heartbeat + t.Cleanup(func() { waitMaxHold, waitHeartbeat = om, oh }) +} + +// waitWith wires an intent hub onto a test handler and returns both. +func newWaitHandler(t *testing.T) (*Handler, *store.Store, *intent.Hub) { + t.Helper() + h, st, _ := newTestHandler(t) + hub := intent.New() + h.SetIntentHub(hub) + return h, st, hub +} + +// doWaitAsync runs GET /wait?gen= with the given bearer in a goroutine and returns a channel +// yielding the recorder once the handler returns. +func doWaitAsync(h *Handler, bearer string, gen string) <-chan *httptest.ResponseRecorder { + out := make(chan *httptest.ResponseRecorder, 1) + go func() { + req := httptest.NewRequest(http.MethodGet, "/api/v1/wait?gen="+gen, nil) + if bearer != "" { + req.Header.Set("Authorization", "Bearer "+bearer) + } + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + out <- rr + }() + return out +} + +func recvRR(t *testing.T, ch <-chan *httptest.ResponseRecorder, d time.Duration) *httptest.ResponseRecorder { + t.Helper() + select { + case rr := <-ch: + return rr + case <-time.After(d): + t.Fatalf("wait handler did not return within %s", d) + return nil + } +} + +func TestWait_Unauthorized(t *testing.T) { + h, _, _ := newWaitHandler(t) + shrinkWaitTiming(t, 50*time.Millisecond, 20*time.Millisecond) + rr := do(h, http.MethodGet, "/wait", "", "") + if rr.Code != http.StatusUnauthorized { + t.Fatalf("no bearer: status = %d, want 401", rr.Code) + } + rr = do(h, http.MethodGet, "/wait", "WRONGKEY", "") + if rr.Code != http.StatusUnauthorized { + t.Fatalf("wrong key: status = %d, want 401", rr.Code) + } +} + +func TestWait_GlobalKeyRejected(t *testing.T) { + h, _, _ := newWaitHandler(t) + shrinkWaitTiming(t, 50*time.Millisecond, 20*time.Millisecond) + rr := do(h, http.MethodGet, "/wait", globalKey, "") + if rr.Code != http.StatusBadRequest { + t.Fatalf("global key: status = %d, want 400 (wait is per-customer)", rr.Code) + } +} + +func TestWait_ServiceUnavailableWithoutHub(t *testing.T) { + h, st, _ := newTestHandler(t) // no SetIntentHub + if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c", RetrievalPassword: "pw", APIKey: "CKEY", ConfigJSON: "{}"}); err != nil { + t.Fatalf("SaveCustomerConfig: %v", err) + } + shrinkWaitTiming(t, 50*time.Millisecond, 20*time.Millisecond) + rr := do(h, http.MethodGet, "/wait", "CKEY", "") + if rr.Code != http.StatusServiceUnavailable { + t.Fatalf("no hub: status = %d, want 503", rr.Code) + } +} + +func TestWait_CompletesOnBumpWithNewGen(t *testing.T) { + h, st, hub := newWaitHandler(t) + if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c", RetrievalPassword: "pw", APIKey: "CKEY", ConfigJSON: "{}"}); err != nil { + t.Fatalf("SaveCustomerConfig: %v", err) + } + shrinkWaitTiming(t, 2*time.Second, 500*time.Millisecond) + + ch := doWaitAsync(h, "CKEY", "0") + // Let the handler register, then bump. + time.Sleep(30 * time.Millisecond) + hub.Bump("c") + + rr := recvRR(t, ch, time.Second) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rr.Code) + } + if got := strings.TrimSpace(rr.Body.String()); !strings.HasSuffix(got, `{"gen":1}`) { + t.Fatalf("body = %q, want to end with {\"gen\":1}", got) + } +} + +// TestWait_TimesOutWithHeartbeats proves Scenario B: with no bump, the hold completes with the +// unchanged generation, and heartbeat newline bytes were emitted while holding (the bytes that +// keep the nginx read-timeout from firing). +func TestWait_TimesOutWithHeartbeats(t *testing.T) { + h, st, _ := newWaitHandler(t) + if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c", RetrievalPassword: "pw", APIKey: "CKEY", ConfigJSON: "{}"}); err != nil { + t.Fatalf("SaveCustomerConfig: %v", err) + } + shrinkWaitTiming(t, 120*time.Millisecond, 25*time.Millisecond) + + start := time.Now() + rr := do(h, http.MethodGet, "/wait", "CKEY", "") + elapsed := time.Since(start) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rr.Code) + } + if elapsed < 100*time.Millisecond { + t.Fatalf("returned before the hold elapsed (%s) — did it not hold?", elapsed) + } + body := rr.Body.String() + if !strings.Contains(body, "\n") || !strings.HasPrefix(body, "\n") { + t.Fatalf("expected leading heartbeat newline(s); body = %q", body) + } + if !strings.HasSuffix(strings.TrimSpace(body), `{"gen":0}`) { + t.Fatalf("expected final {\"gen\":0} on timeout; body = %q", body) + } +} + +// TestWait_RaceCloserReturnsImmediately proves a bump that landed before the wait connected is not +// lost: the box passes its last-seen gen, the hub sees the generation already advanced and returns +// at once. +func TestWait_RaceCloserReturnsImmediately(t *testing.T) { + h, st, hub := newWaitHandler(t) + if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c", RetrievalPassword: "pw", APIKey: "CKEY", ConfigJSON: "{}"}); err != nil { + t.Fatalf("SaveCustomerConfig: %v", err) + } + shrinkWaitTiming(t, 5*time.Second, 1*time.Second) // long hold; must NOT be hit + hub.Bump("c") // gen -> 1 before the box connects + + start := time.Now() + rr := do(h, http.MethodGet, "/wait", "CKEY", "0") // last-seen 0 != 1 + if el := time.Since(start); el > 500*time.Millisecond { + t.Fatalf("race-closer should return immediately, took %s", el) + } + if got := strings.TrimSpace(rr.Body.String()); !strings.HasSuffix(got, `{"gen":1}`) { + t.Fatalf("body = %q, want {\"gen\":1}", got) + } +} + +// TestWait_CustomerIsolation proves customer A's key can never observe B's generation: A holds a +// wait, B is bumped, A must NOT complete (it times out on its own generation). +func TestWait_CustomerIsolation(t *testing.T) { + h, st, hub := newWaitHandler(t) + for _, id := range []string{"a", "b"} { + if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: id, RetrievalPassword: "pw", APIKey: "KEY-" + id, ConfigJSON: "{}"}); err != nil { + t.Fatalf("SaveCustomerConfig %s: %v", id, err) + } + } + shrinkWaitTiming(t, 150*time.Millisecond, 40*time.Millisecond) + + ch := doWaitAsync(h, "KEY-a", "0") // A waits + time.Sleep(30 * time.Millisecond) + hub.Bump("b") // B's intent moves — must not wake A + + rr := recvRR(t, ch, time.Second) + if got := strings.TrimSpace(rr.Body.String()); !strings.HasSuffix(got, `{"gen":0}`) { + t.Fatalf("A should time out on its own gen 0 (isolation); body = %q", got) + } +} diff --git a/hub/internal/intent/hub.go b/hub/internal/intent/hub.go new file mode 100644 index 0000000..04635da --- /dev/null +++ b/hub/internal/intent/hub.go @@ -0,0 +1,143 @@ +// Package intent is the hub's in-memory operator-intent notifier — the server side of the +// Direction-2 immediate-sync arc (SPIKE-immediate-sync-transport-2026-07-16.md, option b). +// +// Every operator action that changes a customer's desired state (config save/delete, claim +// resend, offsite re-issue/freeze, floor change, block/unblock, log-pull request) Bumps that +// customer's generation. A box holds a hanging GET /api/v1/wait against the hub carrying its +// last-seen generation; Wait returns the instant the generation moves (or on timeout/shutdown). +// The box then fires its ordinary out-of-cycle report — the ACK delivers everything through the +// UNCHANGED report machinery. The wait signal is CONTENTLESS: it carries a generation number +// and nothing else (no config, no escrow, no claim). +// +// The counter is IN-MEMORY BY DESIGN — a restart resets all generations to zero, which the box +// treats as "a different generation" (it compares with != , not >), firing exactly one harmless +// full-state report before settling. No persistence, no store schema (Part 2.1 / §5). +// +// Semantics mirror the wgsync reconciler / controller report.Trigger: a per-customer counter, +// coalescing (N bumps during one hold → one completion carrying the LATEST generation), never a +// per-bump queue. Every Wait exit path deregisters its channel, so there is no waiter leak. +package intent + +import ( + "context" + "sync" + "time" +) + +// Hub is the in-memory per-customer generation counter + waiter registry. The zero value is not +// usable; construct with New. All methods are safe for concurrent use. +type Hub struct { + mu sync.Mutex + gen map[string]uint64 // customerID -> current generation (0 = never bumped) + waiters map[string][]chan struct{} // customerID -> registered wake channels (buffered, cap 1) + closed chan struct{} // closed by Close() to wake every waiter + isClosed bool +} + +// New builds an empty intent hub. +func New() *Hub { + return &Hub{ + gen: make(map[string]uint64), + waiters: make(map[string][]chan struct{}), + closed: make(chan struct{}), + } +} + +// Bump advances the customer's generation and wakes every currently-registered waiter for that +// customer, then clears them (each waiter re-reads the LATEST generation under the lock on wake, +// so a burst of bumps collapses into one completion carrying the final value — a counter, not a +// queue). Safe from any goroutine; never blocks. An empty customerID is a no-op, as is any bump +// after Close. +func (h *Hub) Bump(customerID string) { + if customerID == "" { + return + } + h.mu.Lock() + defer h.mu.Unlock() + if h.isClosed { + return + } + h.gen[customerID]++ + for _, ch := range h.waiters[customerID] { + // Non-blocking send into a cap-1 channel: the waiter reads the latest gen under the lock, + // so a single wake is all it needs; a second bump before the waiter runs is harmless. + select { + case ch <- struct{}{}: + default: + } + } + delete(h.waiters, customerID) +} + +// Wait blocks until the customer's generation differs from lastSeen, ctx is cancelled, maxHold +// elapses, or Close is called — whichever first — and returns the customer's CURRENT generation +// to hand back to the caller. If the generation already differs from lastSeen at entry (a bump +// landed before this wait registered), it returns immediately: the race gap between a bump and a +// (re)connect is closed by this pre-register check, so a bump is never lost. +func (h *Hub) Wait(ctx context.Context, customerID string, lastSeen uint64, maxHold time.Duration) uint64 { + h.mu.Lock() + if h.isClosed { + cur := h.gen[customerID] + h.mu.Unlock() + return cur + } + if cur := h.gen[customerID]; cur != lastSeen { + h.mu.Unlock() + return cur // race-closer: intent already moved; no need to hold + } + ch := make(chan struct{}, 1) + h.waiters[customerID] = append(h.waiters[customerID], ch) + h.mu.Unlock() + + timer := time.NewTimer(maxHold) + defer timer.Stop() + + select { + case <-ch: + case <-ctx.Done(): + case <-timer.C: + case <-h.closed: + } + return h.finish(customerID, ch) +} + +// finish deregisters ch (if a Bump/Close hasn't already cleared it) and returns the customer's +// current — i.e. latest — generation. Called on every Wait exit path, so waiters never leak. +func (h *Hub) finish(customerID string, ch chan struct{}) uint64 { + h.mu.Lock() + defer h.mu.Unlock() + ws := h.waiters[customerID] + for i, c := range ws { + if c == ch { + h.waiters[customerID] = append(ws[:i], ws[i+1:]...) + break + } + } + if len(h.waiters[customerID]) == 0 { + delete(h.waiters, customerID) + } + return h.gen[customerID] +} + +// Close wakes every registered waiter (each returns its customer's current generation) and makes +// all subsequent Wait calls return immediately. Idempotent. Wired into the hub's graceful +// shutdown BEFORE server.Shutdown so held waits complete instantly instead of eating the grace +// window. +func (h *Hub) Close() { + h.mu.Lock() + defer h.mu.Unlock() + if h.isClosed { + return + } + h.isClosed = true + close(h.closed) + h.waiters = make(map[string][]chan struct{}) +} + +// Generation returns the customer's current generation (0 if never bumped). Test/observability +// helper — the wait path uses the value returned by Wait directly. +func (h *Hub) Generation(customerID string) uint64 { + h.mu.Lock() + defer h.mu.Unlock() + return h.gen[customerID] +} diff --git a/hub/internal/intent/hub_test.go b/hub/internal/intent/hub_test.go new file mode 100644 index 0000000..3972be3 --- /dev/null +++ b/hub/internal/intent/hub_test.go @@ -0,0 +1,246 @@ +package intent + +import ( + "context" + "sync" + "testing" + "time" +) + +// --- white-box helpers (same package) --- + +func (h *Hub) waiterCount(customerID string) int { + h.mu.Lock() + defer h.mu.Unlock() + return len(h.waiters[customerID]) +} + +func (h *Hub) totalWaiters() int { + h.mu.Lock() + defer h.mu.Unlock() + n := 0 + for _, ws := range h.waiters { + n += len(ws) + } + return n +} + +// waitUntilRegistered blocks (briefly) until n waiters are registered for customerID, so tests +// don't race the goroutine that calls Wait. Fails the test on timeout. +func waitUntilRegistered(t *testing.T, h *Hub, customerID string, n int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if h.waiterCount(customerID) >= n { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("timed out waiting for %d waiter(s) on %q (have %d)", n, customerID, h.waiterCount(customerID)) +} + +// asyncWait runs Wait in a goroutine and returns a channel that yields its result. +func asyncWait(h *Hub, ctx context.Context, customerID string, lastSeen uint64, maxHold time.Duration) <-chan uint64 { + out := make(chan uint64, 1) + go func() { out <- h.Wait(ctx, customerID, lastSeen, maxHold) }() + return out +} + +func recvWithin(t *testing.T, ch <-chan uint64, d time.Duration) uint64 { + t.Helper() + select { + case g := <-ch: + return g + case <-time.After(d): + t.Fatalf("Wait did not return within %s", d) + return 0 + } +} + +// --- tests --- + +func TestBump_WakesRegisteredWaiter(t *testing.T) { + h := New() + ch := asyncWait(h, context.Background(), "c1", 0, time.Second) + waitUntilRegistered(t, h, "c1", 1) + h.Bump("c1") + if g := recvWithin(t, ch, time.Second); g != 1 { + t.Fatalf("expected gen 1 after one bump, got %d", g) + } + if h.totalWaiters() != 0 { + t.Fatalf("waiter leaked after bump: %d", h.totalWaiters()) + } +} + +// TestWait_CoalescesBurstToLatestGen is the load-bearing property (§11 Group A) with its +// RED-PROOF anchor: N rapid bumps during one hold produce ONE completion carrying the LATEST +// generation (a counter, not a per-bump queue). +// +// RED-PROOF: change Bump to complete one waiter per bump / re-register (a queue), or make the +// completion carry the FIRST bump's value instead of finish() re-reading the latest — this +// assertion (gen == N) then fails (it sees 1). Revert to restore green. +func TestWait_CoalescesBurstToLatestGen(t *testing.T) { + h := New() + ch := asyncWait(h, context.Background(), "c1", 0, 2*time.Second) + waitUntilRegistered(t, h, "c1", 1) + + const n = 5 + for i := 0; i < n; i++ { + h.Bump("c1") + } + + if g := recvWithin(t, ch, time.Second); g != n { + t.Fatalf("expected coalesced completion with latest gen %d, got %d", n, g) + } + if h.totalWaiters() != 0 { + t.Fatalf("waiter leaked after burst: %d", h.totalWaiters()) + } + if got := h.Generation("c1"); got != n { + t.Fatalf("generation counter should be %d, got %d", n, got) + } +} + +// TestWait_RaceCloser_BumpBeforeWaitNotLost is the second load-bearing property with its +// RED-PROOF anchor: a bump that lands BEFORE the wait registers must not be lost — the wait +// returns immediately with the advanced generation instead of blocking to the timeout. +// +// RED-PROOF: remove the `cur != lastSeen` pre-register check in Wait — the wait then blocks the +// full maxHold and this sub-100ms assertion fails. Revert to restore green. +func TestWait_RaceCloser_BumpBeforeWaitNotLost(t *testing.T) { + h := New() + h.Bump("c1") // gen -> 1, no waiter registered yet + + start := time.Now() + g := h.Wait(context.Background(), "c1", 0, 2*time.Second) // lastSeen 0 != 1 → immediate + elapsed := time.Since(start) + + if g != 1 { + t.Fatalf("expected immediate gen 1, got %d", g) + } + if elapsed > 100*time.Millisecond { + t.Fatalf("race-closer should return immediately, took %s", elapsed) + } + if h.totalWaiters() != 0 { + t.Fatalf("waiter leaked on race-closer path: %d", h.totalWaiters()) + } +} + +func TestWait_TimeoutReturnsCurrentGenNoLeak(t *testing.T) { + h := New() + // Fresh customer, no bumps: times out returning 0. + start := time.Now() + if g := h.Wait(context.Background(), "c1", 0, 40*time.Millisecond); g != 0 { + t.Fatalf("expected 0 on timeout, got %d", g) + } + if el := time.Since(start); el < 30*time.Millisecond { + t.Fatalf("returned before maxHold elapsed (%s)", el) + } + if h.totalWaiters() != 0 { + t.Fatalf("waiter leaked after timeout: %d", h.totalWaiters()) + } + + // Existing generation: a same-gen wait times out returning that (unchanged) gen. + h.Bump("c2") + h.Bump("c2") // gen -> 2 + if g := h.Wait(context.Background(), "c2", 2, 40*time.Millisecond); g != 2 { + t.Fatalf("expected unchanged gen 2 on timeout, got %d", g) + } + if h.totalWaiters() != 0 { + t.Fatalf("waiter leaked after same-gen timeout: %d", h.totalWaiters()) + } +} + +func TestWait_CtxCancelReturnsPromptly(t *testing.T) { + h := New() + ctx, cancel := context.WithCancel(context.Background()) + ch := asyncWait(h, ctx, "c1", 0, 10*time.Second) + waitUntilRegistered(t, h, "c1", 1) + + cancel() + if g := recvWithin(t, ch, time.Second); g != 0 { + t.Fatalf("expected 0 on ctx cancel, got %d", g) + } + if h.totalWaiters() != 0 { + t.Fatalf("waiter leaked after ctx cancel: %d", h.totalWaiters()) + } +} + +func TestClose_WakesAllWaitersWithCurrentGen(t *testing.T) { + h := New() + h.Bump("c1") // gen c1 -> 1 + ch1 := asyncWait(h, context.Background(), "c1", 1, 10*time.Second) + ch2 := asyncWait(h, context.Background(), "c2", 0, 10*time.Second) + waitUntilRegistered(t, h, "c1", 1) + waitUntilRegistered(t, h, "c2", 1) + + h.Close() + + if g := recvWithin(t, ch1, time.Second); g != 1 { + t.Fatalf("c1 should return current gen 1 on close, got %d", g) + } + if g := recvWithin(t, ch2, time.Second); g != 0 { + t.Fatalf("c2 should return current gen 0 on close, got %d", g) + } + if h.totalWaiters() != 0 { + t.Fatalf("waiter leaked after close: %d", h.totalWaiters()) + } + // Post-close waits return immediately. + start := time.Now() + _ = h.Wait(context.Background(), "c3", 0, 10*time.Second) + if el := time.Since(start); el > 100*time.Millisecond { + t.Fatalf("post-close Wait should return immediately, took %s", el) + } + // Post-close bumps are no-ops (must not panic on the closed channel). + h.Bump("c1") +} + +func TestConcurrentWaitsSameCustomerBothComplete(t *testing.T) { + h := New() + ch1 := asyncWait(h, context.Background(), "c1", 0, 2*time.Second) + ch2 := asyncWait(h, context.Background(), "c1", 0, 2*time.Second) + waitUntilRegistered(t, h, "c1", 2) + + h.Bump("c1") + + if g := recvWithin(t, ch1, time.Second); g != 1 { + t.Fatalf("waiter 1 expected gen 1, got %d", g) + } + if g := recvWithin(t, ch2, time.Second); g != 1 { + t.Fatalf("waiter 2 expected gen 1, got %d", g) + } + if h.totalWaiters() != 0 { + t.Fatalf("waiter leaked: %d", h.totalWaiters()) + } +} + +// TestBump_EmptyCustomerIsNoop guards the global-floor path (which iterates customers) and any +// stray empty id. +func TestBump_EmptyCustomerIsNoop(t *testing.T) { + h := New() + h.Bump("") + if h.Generation("") != 0 { + t.Fatalf("empty-customer bump must be a no-op") + } +} + +// TestRaceUnderStress runs Wait/Bump concurrently to shake out data races (run with -race). +func TestRaceUnderStress(t *testing.T) { + h := New() + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + _ = h.Wait(ctx, "c1", 0, 80*time.Millisecond) + }() + } + for i := 0; i < 50; i++ { + h.Bump("c1") + } + wg.Wait() + if h.totalWaiters() != 0 { + t.Fatalf("waiter leaked under stress: %d", h.totalWaiters()) + } +} diff --git a/hub/internal/web/configs.go b/hub/internal/web/configs.go index 7188628..7e58049 100644 --- a/hub/internal/web/configs.go +++ b/hub/internal/web/configs.go @@ -561,6 +561,7 @@ func (s *Server) handleConfigCreate(w http.ResponseWriter, r *http.Request) { } s.logger.Printf("[INFO] Customer config created: %s", customerID) + s.bumpIntent(customerID) // Direction-2: wake a long-polling box in seconds http.Redirect(w, r, "/customers/"+customerID+"?flash=created", http.StatusSeeOther) } @@ -629,6 +630,7 @@ func (s *Server) handleConfigUpdate(w http.ResponseWriter, r *http.Request, cust } s.logger.Printf("[INFO] Customer config updated: %s", customerID) + s.bumpIntent(customerID) // Direction-2: wake a long-polling box in seconds http.Redirect(w, r, "/customers/"+customerID+"?flash=updated#tab=edit", http.StatusSeeOther) } @@ -652,6 +654,7 @@ func (s *Server) handleClaimResend(w http.ResponseWriter, r *http.Request, custo return } s.logger.Printf("[INFO] claim code re-sent for %s (operator resend; generation rotated)", customerID) + s.bumpIntent(customerID) // Direction-2: the fresh claim hash rides the next report ACK http.Redirect(w, r, "/customers/"+customerID+"?flash=claim-resent#tab=setup", http.StatusSeeOther) } @@ -696,6 +699,7 @@ func (s *Server) handleOffsiteReissue(w http.ResponseWriter, r *http.Request, cu return } s.logger.Printf("[INFO] offsite credentials re-issued for %s (fresh one-time password stored; ConfigVersion bumped)", customerID) + s.bumpIntent(customerID) // Direction-2: wake the stuck box to re-pull + re-run the bridge http.Redirect(w, r, "/customers/"+customerID+"?flash=offsite_reissued#tab=edit", http.StatusSeeOther) } @@ -735,6 +739,7 @@ func (s *Server) ReissueOffsiteForCustomer(ctx context.Context, customerID strin return fmt.Errorf("offsite re-issue: config bump: %w", err) } s.logger.Printf("[INFO] offsite credentials re-issued for %s on re-enroll (fresh one-time password; ConfigVersion bumped)", customerID) + s.bumpIntent(customerID) // Direction-2: the fresh box's first wait wakes on this return nil } @@ -771,6 +776,7 @@ func (s *Server) handleOffsiteFreeze(w http.ResponseWriter, r *http.Request, cus return } s.logger.Printf("[INFO] offsite frozen=%v (readonly) for %s (operator action)", frozen, customerID) + s.bumpIntent(customerID) // Direction-2: reflect the freeze state change to the box promptly flash := "offsite_frozen" if !frozen { flash = "offsite_unfrozen" @@ -787,6 +793,7 @@ func (s *Server) handleConfigDelete(w http.ResponseWriter, r *http.Request, cust } s.logger.Printf("[INFO] Customer config deleted: %s", customerID) + s.bumpIntent(customerID) // Direction-2: wake any still-holding wait so it completes promptly http.Redirect(w, r, "/configs?flash=deleted", http.StatusSeeOther) } @@ -832,6 +839,7 @@ func (s *Server) handleConfigRegenPassword(w http.ResponseWriter, r *http.Reques } s.logger.Printf("[INFO] Retrieval password regenerated for %s", customerID) + s.bumpIntent(customerID) // Direction-2: nudge the box promptly after a credential change http.Redirect(w, r, "/customers/"+customerID+"?flash=password_regenerated#tab=setup", http.StatusSeeOther) } @@ -848,6 +856,7 @@ func (s *Server) handleBlockCustomer(w http.ResponseWriter, r *http.Request, cus return } s.logger.Printf("[INFO] Customer blocked: %s", customerID) + s.bumpIntent(customerID) // Direction-2: deliver the blocked flag to the box in seconds http.Redirect(w, r, "/customers/"+customerID+"?flash=blocked#tab=edit", http.StatusSeeOther) } @@ -864,6 +873,7 @@ func (s *Server) handleUnblockCustomer(w http.ResponseWriter, r *http.Request, c return } s.logger.Printf("[INFO] Customer unblocked: %s", customerID) + s.bumpIntent(customerID) // Direction-2: clear the blocked flag on the box promptly http.Redirect(w, r, "/customers/"+customerID+"?flash=unblocked#tab=edit", http.StatusSeeOther) } @@ -927,6 +937,13 @@ func (s *Server) handleSetGlobalFloor(w http.ResponseWriter, r *http.Request) { return } s.logger.Printf("[INFO] Global controller-version floor set to %q", v) + // Direction-2: the global floor affects every config-managed customer — wake each long-polling + // box so the new floor lands in seconds (nil-safe; a customer with no held wait just advances). + if configs, cerr := s.store.ListCustomerConfigs(); cerr == nil { + for _, c := range configs { + s.bumpIntent(c.CustomerID) + } + } http.Redirect(w, r, "/configuration?flash=floor_set", http.StatusSeeOther) } @@ -1029,6 +1046,7 @@ func (s *Server) handleSetCustomerFloor(w http.ResponseWriter, r *http.Request, return } s.logger.Printf("[INFO] Customer %s controller-version floor override set to %q", customerID, v) + s.bumpIntent(customerID) // Direction-2: deliver the new floor to the box in seconds http.Redirect(w, r, "/customers/"+customerID+"?flash=floor_set", http.StatusSeeOther) } diff --git a/hub/internal/web/logbundle.go b/hub/internal/web/logbundle.go index f37b7a4..3d548eb 100644 --- a/hub/internal/web/logbundle.go +++ b/hub/internal/web/logbundle.go @@ -61,6 +61,12 @@ func (s *Server) handleRequestLogBundle(w http.ResponseWriter, r *http.Request, return } s.logger.Printf("[INFO] %s log bundle requested for host %s — the box delivers on its next cycle", component, hostID) + // Direction-2: only the CONTROLLER ring rides the report ACK (the wait channel wakes the + // controller). The AGENT ring rides the heartbeat envelope — a separate plane this task does not + // touch — so it is deliberately NOT bumped here. + if component == store.LogBundleComponentController { + s.bumpIntent(host.CustomerID) + } http.Redirect(w, r, "/hosts/"+hostID, http.StatusSeeOther) } diff --git a/hub/internal/web/logtail.go b/hub/internal/web/logtail.go index 44c8b5a..f9a2f9e 100644 --- a/hub/internal/web/logtail.go +++ b/hub/internal/web/logtail.go @@ -32,6 +32,7 @@ func (s *Server) handleRequestLogTail(w http.ResponseWriter, r *http.Request, cu s.logger.Printf("[WARN] SaveEvent log_tail_requested %s/%s: %v", customerID, app, err) } s.logger.Printf("[INFO] Log tail requested for %s/%s — controller delivers on its next report cycle", customerID, app) + s.bumpIntent(customerID) // Direction-2: pull the tail in seconds, not on the next cycle http.Redirect(w, r, "/customers/"+customerID+"?flash=log_tail_requested", http.StatusSeeOther) } diff --git a/hub/internal/web/server.go b/hub/internal/web/server.go index f384029..973b580 100644 --- a/hub/internal/web/server.go +++ b/hub/internal/web/server.go @@ -18,6 +18,7 @@ import ( "gitea.dooplex.hu/admin/felhom-hub/internal/assets" "gitea.dooplex.hu/admin/felhom-hub/internal/claim" "gitea.dooplex.hu/admin/felhom-hub/internal/gitea" + "gitea.dooplex.hu/admin/felhom-hub/internal/intent" "gitea.dooplex.hu/admin/felhom-hub/internal/offsite" "gitea.dooplex.hu/admin/felhom-hub/internal/semver" "gitea.dooplex.hu/admin/felhom-hub/internal/store" @@ -66,6 +67,11 @@ type Server struct { offsite *offsite.Provisioner // optional; enables Hetzner offsite provisioning (SLICE 1) tenantsync tenancyProvisioner // optional; enables PBS DR tier provisioning (web/pbsdr.go) claimEngine *claim.Engine // optional; enables the customer-claim resend button (v0.50.0) + // intentHub (v0.58.0, Direction-2 immediate-sync) is Bumped by every operator-intent handler + // (config save/delete, claim resend, offsite re-issue/freeze, floor, block/unblock, log pull) + // so a box long-polling GET /api/v1/wait wakes in seconds. Shared with the API handler. nil = + // no immediacy (bumps are no-ops; the 15-min cycle still reconciles). + intentHub *intent.Hub sessions map[string]*hubSession sessionsMu sync.RWMutex @@ -170,6 +176,19 @@ func (s *Server) SetOffsiteProvisioner(p *offsite.Provisioner) { s.offsite = p } // SetClaimEngine wires the customer-claim code engine for the Setup-tab resend button (v0.50.0). func (s *Server) SetClaimEngine(e *claim.Engine) { s.claimEngine = e } +// SetIntentHub wires the operator-intent notifier (v0.58.0). Every intent handler bumps it via +// s.bumpIntent; nil-safe (bumps become no-ops). +func (s *Server) SetIntentHub(hub *intent.Hub) { s.intentHub = hub } + +// bumpIntent advances the customer's wait generation so a box long-polling GET /api/v1/wait wakes +// immediately. nil-safe. Call AFTER the successful store write (mirror the report.Trigger +// fire-after-commit rule — never on an error path). +func (s *Server) bumpIntent(customerID string) { + if s.intentHub != nil { + s.intentHub.Bump(customerID) + } +} + // SetGiteaClient enables the Day-0 artifact version dropdowns (optional). Without it the artifact form // degrades to manual text entry. func (s *Server) SetGiteaClient(c *gitea.Client) {