v0.83.0: observability pass — always-DEBUG capture ring + GET /debug/logs + heartbeat log-pull + gap-fill sweep

Capture layer: applog.New returns (logger, Ring) — slog fan-out, stderr at the
configured level, ~1000-entry ring fixed at LevelDebug (remote diagnostics
without a config flip). GET /debug/logs (token-authed, ?raw=1) + request-level
DEBUG middleware. Heartbeat log-pull mirrors the report logtail pattern:
envelope log_tail_requested -> next heartbeat carries log_tail (128KB cap,
consume-once, failed-push retry proven). Gap-fill sweep over netverify/
netstorage/netmount/signedjobs/selfupdate/disks/controller-swap/desired/loop.
Red-proofs: ring-at-emit-level FAILs capture test; drain removed FAILs
consume-once; dropped phase line FAILs the S7 log-sequence smoke.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
2026-07-11 16:24:07 +02:00
parent 461eaf42c1
commit cb692f8788
20 changed files with 902 additions and 29 deletions
+2
View File
@@ -241,6 +241,7 @@ func (c *ControllerSwapper) Swap(ctx context.Context, vmid int, target string) *
c.saveState(st) // crash-safety: previous recorded BEFORE any mutation
// The controller pre-pulled the image; refuse to swap to an absent image (would brick the guest).
c.logger.Debug("controller-swap: pre-pull verify", "vmid", vmid, "target", target, "previous", prev)
if !c.imagePresent(ctx, vmid, target) {
st.State = "failed"
st.Error = "target image not present in guest (controller did not pre-pull it)"
@@ -270,6 +271,7 @@ func (c *ControllerSwapper) Swap(ctx context.Context, vmid int, target string) *
c.logger.Info("controller-swap: new controller healthy", "vmid", vmid, "target", target)
return st
}
c.logger.Warn("controller-swap: health verdict negative — rolling back", "vmid", vmid, "target", target)
return c.rollback(ctx, st, "new controller did not become healthy within timeout")
}
+64
View File
@@ -0,0 +1,64 @@
package localapi
import (
"net/http"
"strconv"
"time"
applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
)
// GET /debug/logs (v0.83.0 observability) — the agent's always-DEBUG capture ring,
// served over the same token-authed, self-scoped local API as every sibling route.
// This is what makes the agent's side of a flow (e.g. a NAS verify) visible from the
// controller's Debug page without journald access or a config flip. ?raw=1 mirrors
// the controller viewer's plain-text variant. Log lines carry keys never values
// (logging conventions), so the ring is safe to serve to the guest's operator view.
// handleDebugLogs serves the ring as JSON entries ({entries, total}) or plain text.
func (s *Server) handleDebugLogs(w http.ResponseWriter, r *http.Request, vmid int) {
if s.logRing == nil {
writeErr(w, http.StatusServiceUnavailable, "debug log ring not configured on this agent")
return
}
limit := 0 // 0 = everything held (ring-bounded)
if v := r.URL.Query().Get("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= applog.DefaultRingSize {
limit = n
}
}
if r.URL.Query().Get("raw") == "1" {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
for _, line := range s.logRing.Lines(0) {
w.Write([]byte(line))
w.Write([]byte("\n"))
}
return
}
entries, total := s.logRing.Entries(limit)
writeOK(w, map[string]any{"vmid": vmid, "entries": entries, "total": total})
}
// statusRecorder captures the wrapped handler's status for the request log line.
type statusRecorder struct {
http.ResponseWriter
status int
}
func (sr *statusRecorder) WriteHeader(code int) {
sr.status = code
sr.ResponseWriter.WriteHeader(code)
}
// logRequests is the request-level DEBUG middleware: method, path, status, duration
// — never bodies (bodies can carry secrets; the ring must not).
func (s *Server) logRequests(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
next.ServeHTTP(rec, r)
s.logger.Debug("local-api: request",
"method", r.Method, "path", r.URL.Path,
"status", rec.status, "duration_ms", time.Since(start).Milliseconds())
})
}
+130
View File
@@ -0,0 +1,130 @@
package localapi
import (
"bytes"
"encoding/json"
"io"
"log/slog"
"strings"
"testing"
applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
)
// newDebugLogServer builds a minimal server with the debug ring wired and one line
// of each level captured (emit level info — the capture-at-info posture).
func newDebugLogServer(t *testing.T) (*Server, *applog.Ring) {
t.Helper()
logger, ring := applog.NewWithWriter(&bytes.Buffer{}, "info", 50)
logger.Debug("netverify: flow detail", "step", "probe")
logger.Info("netmount: ensured network mount", "name", "nas")
srv, err := NewServer(Options{
ListenAddr: "127.0.0.1:0",
Guests: &fakeGuests{},
Backups: &fakeBackups{},
Store: &fakeStore{},
Storage: fakeStorage{},
Tokens: staticTokens{"A": 8200},
LogRing: ring,
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
})
if err != nil {
t.Fatalf("new server: %v", err)
}
return srv, ring
}
// GET /debug/logs serves the ring's entries — INCLUDING the DEBUG line captured at
// emit level info (the endpoint exists so the controller's agent tab can show flow
// detail without a config flip). Companion red-proof: unwire LogRing → 503.
func TestDebugLogs_ServesRingEntriesIncludingDebug(t *testing.T) {
srv, _ := newDebugLogServer(t)
w := do(t, srv.Handler(), "GET", "/debug/logs", "A", "")
if w.Code != 200 {
t.Fatalf("status = %d body=%s", w.Code, w.Body.String())
}
var resp struct {
OK bool `json:"ok"`
Data struct {
Entries []applog.Entry `json:"entries"`
Total int `json:"total"`
} `json:"data"`
}
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode: %v", err)
}
if !resp.OK || resp.Data.Total != 2 || len(resp.Data.Entries) != 2 {
t.Fatalf("resp = %+v, want ok with 2 entries", resp)
}
if resp.Data.Entries[0].Level != "DEBUG" || !strings.Contains(resp.Data.Entries[0].Message, "flow detail") {
t.Errorf("entry 0 = %+v, want the captured DEBUG line", resp.Data.Entries[0])
}
}
// ?raw=1 serves plain text lines (the controller viewer's raw variant).
func TestDebugLogs_RawVariant(t *testing.T) {
srv, _ := newDebugLogServer(t)
w := do(t, srv.Handler(), "GET", "/debug/logs?raw=1", "A", "")
if w.Code != 200 || !strings.HasPrefix(w.Header().Get("Content-Type"), "text/plain") {
t.Fatalf("status=%d content-type=%q", w.Code, w.Header().Get("Content-Type"))
}
body := w.Body.String()
if !strings.Contains(body, "[DEBUG]") || !strings.Contains(body, "ensured network mount") {
t.Errorf("raw body missing lines:\n%s", body)
}
}
// The route is auth-gated like every sibling (no token → 401), and reports "not
// configured" (503) when the ring is not wired — never a panic, never an empty 200.
func TestDebugLogs_AuthAndUnconfigured(t *testing.T) {
srv, _ := newDebugLogServer(t)
if w := do(t, srv.Handler(), "GET", "/debug/logs", "", ""); w.Code != 401 {
t.Errorf("unauthed status = %d, want 401", w.Code)
}
bare, err := NewServer(Options{
ListenAddr: "127.0.0.1:0",
Guests: &fakeGuests{},
Backups: &fakeBackups{},
Store: &fakeStore{},
Storage: fakeStorage{},
Tokens: staticTokens{"A": 8200},
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
})
if err != nil {
t.Fatalf("new server: %v", err)
}
if w := do(t, bare.Handler(), "GET", "/debug/logs", "A", ""); w.Code != 503 {
t.Errorf("unconfigured status = %d, want 503", w.Code)
}
}
// The request middleware logs method/path/status/duration at DEBUG into the wired
// logger — proven through a ring-backed server logger (the line must land in the ring).
func TestRequestLogging_DebugLineInRing(t *testing.T) {
logger, ring := applog.NewWithWriter(&bytes.Buffer{}, "info", 50)
srv, err := NewServer(Options{
ListenAddr: "127.0.0.1:0",
Guests: &fakeGuests{},
Backups: &fakeBackups{},
Store: &fakeStore{},
Storage: fakeStorage{},
Tokens: staticTokens{"A": 8200},
LogRing: ring,
Logger: logger,
})
if err != nil {
t.Fatalf("new server: %v", err)
}
do(t, srv.Handler(), "GET", "/storage", "A", "")
entries, _ := ring.Entries(0)
found := false
for _, e := range entries {
if e.Level == "DEBUG" && strings.Contains(e.Message, "local-api: request") &&
strings.Contains(e.Message, "path=/storage") {
found = true
}
}
if !found {
t.Errorf("no request DEBUG line in the ring; entries=%+v", entries)
}
}
+9 -1
View File
@@ -319,6 +319,7 @@ func (s *Server) handleDiskAssign(w http.ResponseWriter, r *http.Request, vmid i
writeErr(w, http.StatusBadRequest, "assign failed: "+err.Error())
return
}
s.logger.Info("local-api: disk assigned (host mount ensured)", "vmid", vmid, "where", req.Where)
writeOK(w, map[string]any{"vmid": vmid, "assigned": req.Where})
}
@@ -372,6 +373,8 @@ func (s *Server) handleDiskEject(w http.ResponseWriter, r *http.Request, vmid in
return
}
}
s.logger.Info("local-api: drive ejected (bind detached, raw mount kept)",
"vmid", vmid, "where", req.Where, "dependent_guests", len(dependents))
writeOK(w, map[string]any{"vmid": vmid, "ejected": req.Where, "dependent_guests": dependents})
}
@@ -440,6 +443,8 @@ func (s *Server) handleDiskDecommission(w http.ResponseWriter, r *http.Request,
// the RAW /mnt/<name> host mount — that would orphan the drive (a non-removable SATA drive doesn't get
// re-plugged), so a one-click re-enroll (H3) could not re-bind it. The soft decommission marker blocks
// scheduling; physical removal is the separate "remove from system" action. NEVER format/mkfs here.
s.logger.Info("local-api: drive decommissioned (logical retire, data untouched)",
"vmid", vmid, "where", req.Where, "durable_id", id, "dependent_guests", len(dependents))
writeOK(w, map[string]any{"vmid": vmid, "decommissioned": req.Where, "dependent_guests": dependents})
}
@@ -920,7 +925,10 @@ func (s *Server) ReassertGuestBinds(ctx context.Context) {
}
}
}
for vmid, ids := range s.guestBinds.Guests() {
guests := s.guestBinds.Guests()
s.logger.Debug("reconcile: guest-bind re-assert pass",
"guests", len(guests), "resolved_mounts", len(mountByDurable))
for vmid, ids := range guests {
for _, id := range ids {
// Intent-aware (B2, load-bearing): NEVER bind a drive that is not currently `enrolled` — an
// ejected or decommissioned drive must not auto-rebind, even if still host-mounted. A nil
+8 -2
View File
@@ -127,12 +127,15 @@ func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request, vmi
if !s.netReachable(spec.Protocol, spec.Server) {
s.releaseNetVerify(job)
s.logger.Warn("local-api: network mount refused — endpoint not reachable",
"name", spec.Name, "server", spec.Server, "proto", spec.Protocol)
"name", spec.Name, "server", spec.Server, "proto", spec.Protocol,
"code", storage.NetVerifyUnreachable)
writeStatus(w, http.StatusBadGateway, false,
map[string]any{"code": storage.NetVerifyUnreachable},
"NAS endpoint not reachable")
return
}
s.logger.Debug("local-api: NAS endpoint pre-probe passed",
"name", spec.Name, "server", spec.Server, "proto", spec.Protocol)
// SMB: stage the credentials out-of-band (0600). NFS needs none (server squash).
if spec.Protocol == storage.ProtocolSMB {
@@ -144,6 +147,7 @@ func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request, vmi
return
}
spec.CredsRef = credsPath
s.logger.Debug("local-api: SMB credentials staged", "name", spec.Name, "path", credsPath) // path only, never content
}
if err := s.netStorage.EnsureNetworkMount(r.Context(), spec); err != nil {
@@ -251,7 +255,9 @@ func (s *Server) writeSMBCreds(name, username, password string) (string, error)
// removeSMBCreds deletes a share's creds file (best-effort; absent is fine).
func (s *Server) removeSMBCreds(name string) {
_ = os.Remove(s.smbCredsPath(name))
if err := os.Remove(s.smbCredsPath(name)); err == nil {
s.logger.Debug("local-api: SMB credentials file removed", "name", name, "path", s.smbCredsPath(name))
}
}
// smbCredsPath computes a share's creds file path (pure — used by validation BEFORE the file exists).
@@ -0,0 +1,67 @@
package localapi
import (
"bytes"
"net/http"
"strings"
"testing"
applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
)
// S7 sweep smoke (agent half): a full fake NAS add at emit level INFO must leave the
// EXPECTED LOG SEQUENCE in the debug ring — this is the test that encodes "an operator
// can reconstruct the NAS flow from the debug view". Companion red-proof: remove any
// one of the asserted phase lines (e.g. the /proc/mounts verdict Debug) → its marker
// is absent → FAIL naming the missing phase.
func TestNetAdd_LogSequenceReconstructsFlow(t *testing.T) {
logger, ring := applog.NewWithWriter(&bytes.Buffer{}, "info", 200)
n := &fakeNetOps{}
srv := newVerifyServer(t, n, t.TempDir(), verifySeams{
mounted: func(string) bool { return true }, // §8: in /proc/mounts ⇒ verified
})
srv.logger = logger // ring-backed capture layer under the whole flow
body := `{"name":"vids","protocol":"smb","server":"nas","export":"vids","mapped_uid":1000,"mapped_gid":1000,"username":"u","password":"p"}`
if w := do(t, srv.Handler(), "POST", "/netstorage/add", "A", body); w.Code != http.StatusOK {
t.Fatalf("add: got %d (%s)", w.Code, w.Body.String())
}
final := pollVerify(t, srv.Handler())
if final["phase"] != netVerifyPhaseDone {
t.Fatalf("phase = %v, want done", final["phase"])
}
lines := ring.Lines(0)
joined := strings.Join(lines, "\n")
// The phase markers, in order (each must exist; each must come after the previous).
sequence := []string{
"NAS endpoint pre-probe passed",
"SMB credentials staged",
"network mount installed",
"netverify: job started",
"netverify: /proc/mounts verdict",
"netverify: mount verified",
}
pos := -1
for _, marker := range sequence {
idx := indexOfLine(lines, marker, pos+1)
if idx < 0 {
t.Fatalf("phase line %q missing (or out of order) — flow not reconstructable.\nring:\n%s", marker, joined)
}
pos = idx
}
// The secret never appears in any line (creds are path-only).
if strings.Contains(joined, "password") || strings.Contains(joined, `"p"`) {
t.Errorf("a credential-looking token leaked into the log ring:\n%s", joined)
}
}
// indexOfLine finds the first line at or after `from` containing marker; -1 if none.
func indexOfLine(lines []string, marker string, from int) int {
for i := from; i < len(lines); i++ {
if strings.Contains(lines[i], marker) {
return i
}
}
return -1
}
+21 -4
View File
@@ -112,8 +112,12 @@ func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob)
done := make(chan struct{})
go func() {
defer close(done)
start := time.Now()
ctx, cancel := context.WithTimeout(base, netVerifyDeadline)
defer cancel()
s.logger.Debug("netverify: job started",
"job_id", job.JobID, "name", spec.Name, "where", spec.Where(),
"proto", spec.Protocol, "deadline_s", int(netVerifyDeadline.Seconds()))
// Trigger a real mount: a directory read through the enabled automount mounts the share
// (spike Q1, ~1 s on a healthy LAN). The read may block until systemd resolves the mount
@@ -123,16 +127,24 @@ func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob)
go func() { trigger <- s.netTrigger(spec.Where()) }()
timedOut := false
select {
case <-trigger:
case terr := <-trigger:
// The read error is NOT the verdict (§8) — logged for the flow trace only.
s.logger.Debug("netverify: automount trigger returned",
"name", spec.Name, "read_err", fmt.Sprint(terr))
case <-ctx.Done():
timedOut = true
s.logger.Debug("netverify: deadline fired before the trigger resolved", "name", spec.Name)
}
// §8 truth table: /proc/mounts is the ONLY success judge. A trigger read error on a mounted
// share (EACCES on a 0700 export) is a GOOD mount — the controller's uid-1000 probe decides
// writability, not the agent user's readability.
if s.netMounted(spec.Where()) {
s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where())
mounted := s.netMounted(spec.Where())
s.logger.Debug("netverify: /proc/mounts verdict",
"name", spec.Name, "where", spec.Where(), "mounted", mounted, "timed_out", timedOut)
if mounted {
s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where(),
"duration_ms", time.Since(start).Milliseconds())
s.finishNetVerify(job, netVerifyPhaseDone, "", "")
return
}
@@ -142,7 +154,8 @@ func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob)
code, detail := s.classifyNetFailure(base, spec, timedOut)
s.rollbackNetMount(base, spec)
s.logger.Warn("netverify: verify failed — install rolled back",
"name", spec.Name, "code", code, "timed_out", timedOut)
"name", spec.Name, "code", code, "timed_out", timedOut,
"duration_ms", time.Since(start).Milliseconds())
s.finishNetVerify(job, netVerifyPhaseFailed, code, detail)
}()
return done
@@ -168,7 +181,9 @@ func (s *Server) classifyNetFailure(base context.Context, spec storage.NetworkMo
"unit", unit, "err", jerr)
return fallback, "journal unavailable — add the felhom-agent user to the systemd-journal group (usermod -aG systemd-journal felhom-agent)"
}
s.logger.Debug("netverify: journal tail read for classification", "unit", unit, "bytes", len(tail))
code, hint := storage.ClassifyNetVerifyFailure(tail, s.netReachable(spec.Protocol, spec.Server))
s.logger.Debug("netverify: failure classified", "name", spec.Name, "code", code, "timed_out", timedOut)
if code == storage.NetVerifyMountFailed && timedOut {
code = storage.NetVerifyTimeout
}
@@ -189,6 +204,8 @@ func (s *Server) rollbackNetMount(base context.Context, spec storage.NetworkMoun
if err := s.netStorage.RemoveNetworkMount(ctx, spec.Name); err != nil {
s.logger.Error("netverify: rollback RemoveNetworkMount failed (manual cleanup may be needed)",
"name", spec.Name, "err", err)
} else {
s.logger.Info("netverify: failed install rolled back (unit pair removed)", "name", spec.Name)
}
if spec.Protocol == storage.ProtocolSMB {
s.removeSMBCreds(spec.Name)
+14 -3
View File
@@ -16,6 +16,7 @@ import (
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
)
@@ -133,7 +134,10 @@ type Options struct {
// Supports() compares this against a per-feature MinAgent table instead of route-probing
// (v0.82.0; the probe stays as the fallback for header-less agents). Optional.
AgentVersion string
Logger *slog.Logger
// LogRing is the agent's always-DEBUG capture ring (v0.83.0 observability), served by
// GET /debug/logs. OPTIONAL — when nil the endpoint reports "not configured".
LogRing *applog.Ring
Logger *slog.Logger
}
// defaultBackupCadence is the fallback /backup/due window when none is configured.
@@ -191,6 +195,7 @@ type Server struct {
hostMetrics HostMetricsProvider // slice 9 (optional)
hostID string // slice 10B: for the data-bearing-format pending-op hint
agentVersion string // v0.82.0: the X-Felhom-Agent-Version response header value
logRing *applog.Ring // v0.83.0: GET /debug/logs source (optional)
// reresolveWipe performs the [AGENT-001] anti-retarget re-resolution before an
// inline customer-confirmed wipe (durable id → current device, re-derive+match,
@@ -281,6 +286,7 @@ func NewServer(o Options) (*Server, error) {
hostMetrics: o.HostMetrics,
hostID: o.HostID,
agentVersion: o.AgentVersion,
logRing: o.LogRing,
jobs: map[int]*backupJob{},
swapInFlight: map[int]bool{},
}
@@ -344,16 +350,21 @@ func (s *Server) Handler() http.Handler {
// fork-4 hygiene: wipe the staged secret once escrowed (controller calls this on confirm). Idempotent.
mux.HandleFunc("DELETE /escrow/stage-secret", s.withGuest(s.handleWipeStagedEscrowSecret))
// v0.83.0 observability: the agent's always-DEBUG capture ring, for the controller's
// Debug page agent tab (same auth/self-scoping wrap as every sibling route).
mux.HandleFunc("GET /debug/logs", s.withGuest(s.handleDebugLogs))
// v0.82.0 version channel: EVERY response (any route, any status — including auth failures)
// carries X-Felhom-Agent-Version, so the controller learns the agent version passively from its
// ordinary traffic and can capability-gate by comparison instead of route-probing. Header-less
// (pre-0.82) agents keep working — the controller falls back to the probe.
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// v0.83.0: wrapped in the request-level DEBUG log middleware (method/path/status/duration).
return s.logRequests(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if s.agentVersion != "" {
w.Header().Set("X-Felhom-Agent-Version", s.agentVersion)
}
mux.ServeHTTP(w, r)
})
}))
}
// Run binds the bridge socket, serves TLS, and shuts down gracefully on ctx cancellation. It