hub v0.46.0: observability pass — per-box log pulls, bundle custody, 72h TTL + secret gate

log_bundle_requests + log_bundles store (gzip, newest-3, 72h TTL purged on the 60s sweep); SaveLogBundle secret gate fail-closed (blocked flag row, no payload; REDACTED/checksums pass). Report ACK gains controller_log_requested + ingests controller_log_tail; heartbeat envelope gains log_tail_requested + ingests log_tail (consume-once on arrival; pre-0.83 agents stay visibly pending). Host detail Diagnostics section: request buttons (controller/agent), state rows with honest latency hints, View/Download endpoint. Red-proofs: gate disabled and clear-on-arrival removed both FAIL their tests. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-11 16:57:47 +02:00
parent 1ee5559772
commit e35b1ae0e6
13 changed files with 898 additions and 4 deletions
@@ -331,6 +331,29 @@ func (h *Handler) handleReport(w http.ResponseWriter, r *http.Request) {
 		}
 	}

+	// Controller self-log tail (v0.46.0) — the controller's OWN debug ring, shipped on the
+	// cycle after the ACK's controller_log_requested. SaveLogBundle runs the secret gate
+	// (a hit stores a BLOCKED flag row, nothing else) and clears the pending request
+	// (consume-once). Backward-compatible: old controllers never send this.
+	var selfTailPayload struct {
+		ControllerLogTail *struct {
+			CollectedAt time.Time `json:"collected_at"`
+			Lines       []string  `json:"lines"`
+		} `json:"controller_log_tail"`
+	}
+	if err := json.Unmarshal(body, &selfTailPayload); err == nil && selfTailPayload.ControllerLogTail != nil {
+		lt := selfTailPayload.ControllerLogTail
+		blocked, berr := h.store.SaveLogBundle(payload.CustomerID, store.LogBundleComponentController, lt.CollectedAt, lt.Lines)
+		switch {
+		case berr != nil:
+			h.logger.Printf("[WARN] Failed to save controller log bundle for %s: %v", payload.CustomerID, berr)
+		case blocked:
+			h.logger.Printf("[WARN] controller log bundle for %s BLOCKED: possible secret in log content — nothing stored", payload.CustomerID)
+		default:
+			h.logger.Printf("[INFO] controller log bundle received for %s (%d lines)", payload.CustomerID, len(lt.Lines))
+		}
+	}
+
 	// DR recipe — persist the controller's secret-free customer/apps half (preserving any host half).
 	// Backward-compatible (old controllers won't have this field); a failure must not drop the report.
 	var drPayload struct {
@@ -376,6 +399,12 @@ func (h *Handler) handleReport(w http.ResponseWriter, r *http.Request) {
 		resp["log_tail_requests"] = apps
 	}

+	// v0.46.0 — pending CONTROLLER self-log pull (same additive ACK-flag pattern): the
+	// controller ships controller_log_tail on its NEXT report; omitted when nothing pending.
+	if pending, err := h.store.PendingLogBundleRequest(payload.CustomerID, store.LogBundleComponentController); err == nil && pending {
+		resp["controller_log_requested"] = true
+	}
+
 	// Phase 2 managed updates: advertise the effective controller-version FLOOR (per-customer override
 	// else global default) and the latest available version. The controller compares its current
 	// version against the floor and auto-updates when below it (latest stays the customer's opt-in
@@ -447,6 +476,12 @@ type hostReportPayload struct {
 	// DR recipe — the agent's storage/guest/PBS half (secret-free). RawMessage = stored verbatim,
 	// ignore-unknown (forward-compat). Persisted to dr_recipe, assembled with the controller half.
 	DRRecipe json.RawMessage `json:"dr_recipe"`
+	// LogTail (v0.46.0) — the agent's on-demand debug-ring tail, present only on the
+	// heartbeat right after the envelope's log_tail_requested (agent ≥ 0.83.0).
+	LogTail *struct {
+		CollectedAt time.Time `json:"collected_at"`
+		Lines       []string  `json:"lines"`
+	} `json:"log_tail"`
 }

 // drRecipeVersionOnly extracts just recipe_version from a half's JSON (ignore-unknown). 0 if absent.
@@ -680,6 +715,21 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
 	h.logger.Printf("[INFO] host-report from %s (%d guests, %d storage targets, %d backups, %d restore-tests, %d pbs-snapshots, %d bytes)",
 		hostID, len(rep.Guests), len(rep.StorageTargets), len(rep.Backups), len(rep.RestoreTests), len(rep.PBSSnapshots), len(body))

+	// Agent log tail (v0.46.0) — the debug-ring bundle a prior envelope requested.
+	// SaveLogBundle runs the secret gate + clears the pending request (consume-once).
+	// A failure must NOT drop the heartbeat; just warn.
+	if rep.LogTail != nil {
+		blocked, berr := h.store.SaveLogBundle(hostID, store.LogBundleComponentAgent, rep.LogTail.CollectedAt, rep.LogTail.Lines)
+		switch {
+		case berr != nil:
+			h.logger.Printf("[WARN] Failed to save agent log bundle for %s: %v", hostID, berr)
+		case blocked:
+			h.logger.Printf("[WARN] agent log bundle for %s BLOCKED: possible secret in log content — nothing stored", hostID)
+		default:
+			h.logger.Printf("[INFO] agent log bundle received for %s (%d lines)", hostID, len(rep.LogTail.Lines))
+		}
+	}
+
 	// DR recipe — persist the agent's secret-free storage/guest/PBS half (preserving any app half).
 	// A failure here must NOT drop the heartbeat (the report already saved); just warn.
 	if len(rep.DRRecipe) > 0 && custID != "" {
@@ -717,6 +767,11 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
 		"desired_generation":    desiredGen,
 		"has_signed_ops":        hasSignedOps,
 	}
+	// v0.46.0 — pending agent log pull: the NEXT heartbeat carries log_tail (agent ≥
+	// 0.83.0; older agents ignore the flag and the request stays visibly pending).
+	if pending, err := h.store.PendingLogBundleRequest(hostID, store.LogBundleComponentAgent); err == nil && pending {
+		resp["log_tail_requested"] = true
+	}
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(resp)
@@ -0,0 +1,119 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// v0.46.0 (API half, controller channel) — the full pull round-trip on /report:
+//  1. baseline ACK omits controller_log_requested
+//  2. operator requests the controller bundle → the ACK advertises it
+//  3. the next report carries controller_log_tail → stored + request cleared
+//  4. the FOLLOWING ACK omits the flag again (consume-once)
+func TestReportACK_ControllerLogRoundTrip(t *testing.T) {
+	h, st, _ := newTestHandler(t)
+	reportBody := `{"customer_id":"cust-a"}`
+
+	rr := do(h, http.MethodPost, "/report", globalKey, reportBody)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("report POST = %d (%s)", rr.Code, rr.Body.String())
+	}
+	var ack map[string]json.RawMessage
+	json.Unmarshal(rr.Body.Bytes(), &ack)
+	if _, present := ack["controller_log_requested"]; present {
+		t.Fatalf("baseline ACK must omit controller_log_requested: %s", rr.Body.String())
+	}
+
+	if err := st.RequestLogBundle("cust-a", store.LogBundleComponentController); err != nil {
+		t.Fatal(err)
+	}
+	rr = do(h, http.MethodPost, "/report", globalKey, reportBody)
+	ack = map[string]json.RawMessage{}
+	json.Unmarshal(rr.Body.Bytes(), &ack)
+	if string(ack["controller_log_requested"]) != "true" {
+		t.Fatalf("ACK controller_log_requested = %s, want true", ack["controller_log_requested"])
+	}
+
+	tailReport := `{"customer_id":"cust-a","controller_log_tail":{"collected_at":"2026-07-11T10:00:00Z","lines":["[DEBUG] phase agent_add -> verifying","[INFO] operator log pull served (controller ring)"]}}`
+	rr = do(h, http.MethodPost, "/report", globalKey, tailReport)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("tail report POST = %d", rr.Code)
+	}
+	bundles, err := st.GetLogBundles("cust-a")
+	if err != nil || len(bundles) != 1 || bundles[0].Component != store.LogBundleComponentController {
+		t.Fatalf("stored bundles = %+v err=%v, want 1 controller bundle", bundles, err)
+	}
+	_, lines, _ := st.GetLogBundleContent(bundles[0].ID, "cust-a")
+	if len(lines) != 2 || lines[0] != "[DEBUG] phase agent_add -> verifying" {
+		t.Fatalf("bundle lines not stored verbatim: %#v", lines)
+	}
+
+	rr = do(h, http.MethodPost, "/report", globalKey, reportBody)
+	ack = map[string]json.RawMessage{}
+	json.Unmarshal(rr.Body.Bytes(), &ack)
+	if _, present := ack["controller_log_requested"]; present {
+		t.Fatalf("request survived fulfillment — the controller would ship every cycle: %s", rr.Body.String())
+	}
+}
+
+// v0.46.0 (API half, agent channel) — the heartbeat round-trip on /host-report,
+// PLUS S6: a pre-0.83 agent that never ships log_tail leaves the request visibly
+// PENDING across heartbeats (documented, harmless — the envelope just keeps
+// advertising). Companion red-proof: clear the request at advertise time instead of
+// arrival → the stays-pending assertion fails.
+func TestHostReportEnvelope_AgentLogRoundTripAndOldAgentPending(t *testing.T) {
+	h, st, _ := newTestHandler(t)
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
+
+	// Baseline: no flag.
+	rr := do(h, http.MethodPost, "/host-report", "HKEY", validReportBody("h1"))
+	var env map[string]json.RawMessage
+	json.Unmarshal(rr.Body.Bytes(), &env)
+	if _, present := env["log_tail_requested"]; present {
+		t.Fatalf("baseline envelope must omit log_tail_requested: %s", rr.Body.String())
+	}
+
+	// Request → advertised.
+	if err := st.RequestLogBundle("h1", store.LogBundleComponentAgent); err != nil {
+		t.Fatal(err)
+	}
+	rr = do(h, http.MethodPost, "/host-report", "HKEY", validReportBody("h1"))
+	env = map[string]json.RawMessage{}
+	json.Unmarshal(rr.Body.Bytes(), &env)
+	if string(env["log_tail_requested"]) != "true" {
+		t.Fatalf("envelope log_tail_requested = %s, want true", env["log_tail_requested"])
+	}
+
+	// S6: an old agent ignores the flag — after N plain heartbeats the request is STILL pending.
+	for i := 0; i < 3; i++ {
+		do(h, http.MethodPost, "/host-report", "HKEY", validReportBody("h1"))
+	}
+	if pending, _ := st.PendingLogBundleRequest("h1", store.LogBundleComponentAgent); !pending {
+		t.Fatal("pre-0.83 request vanished without fulfillment — it must stay visibly pending")
+	}
+
+	// A 0.83 agent ships the tail → stored + cleared.
+	tailBody := `{"host_id":"h1","host":{},"guests":[],"storage_targets":[],"backups":[],"cloudflared":{},"audit_tail":[],` +
+		`"log_tail":{"collected_at":"2026-07-11T10:00:00Z","lines":["2026-07-11T10:00:00Z [INFO] operator log pull served"]}}`
+	rr = do(h, http.MethodPost, "/host-report", "HKEY", tailBody)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("tail heartbeat = %d (%s)", rr.Code, rr.Body.String())
+	}
+	bundles, err := st.GetLogBundles("h1")
+	if err != nil || len(bundles) != 1 || bundles[0].Component != store.LogBundleComponentAgent {
+		t.Fatalf("agent bundle = %+v err=%v", bundles, err)
+	}
+	if pending, _ := st.PendingLogBundleRequest("h1", store.LogBundleComponentAgent); pending {
+		t.Error("request survived fulfillment (consume-once broken)")
+	}
+	// The following envelope no longer advertises.
+	rr = do(h, http.MethodPost, "/host-report", "HKEY", validReportBody("h1"))
+	env = map[string]json.RawMessage{}
+	json.Unmarshal(rr.Body.Bytes(), &env)
+	if _, present := env["log_tail_requested"]; present {
+		t.Fatalf("envelope still advertises after fulfillment: %s", rr.Body.String())
+	}
+}