v0.83.0: observability pass — always-DEBUG capture ring + GET /debug/logs + heartbeat log-pull + gap-fill sweep

Capture layer: applog.New returns (logger, Ring) — slog fan-out, stderr at the
configured level, ~1000-entry ring fixed at LevelDebug (remote diagnostics
without a config flip). GET /debug/logs (token-authed, ?raw=1) + request-level
DEBUG middleware. Heartbeat log-pull mirrors the report logtail pattern:
envelope log_tail_requested -> next heartbeat carries log_tail (128KB cap,
consume-once, failed-push retry proven). Gap-fill sweep over netverify/
netstorage/netmount/signedjobs/selfupdate/disks/controller-swap/desired/loop.
Red-proofs: ring-at-emit-level FAILs capture test; drain removed FAILs
consume-once; dropped phase line FAILs the S7 log-sequence smoke.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
2026-07-11 16:24:07 +02:00
parent 461eaf42c1
commit cb692f8788
20 changed files with 902 additions and 29 deletions
+21 -4
View File
@@ -112,8 +112,12 @@ func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob)
done := make(chan struct{})
go func() {
defer close(done)
start := time.Now()
ctx, cancel := context.WithTimeout(base, netVerifyDeadline)
defer cancel()
s.logger.Debug("netverify: job started",
"job_id", job.JobID, "name", spec.Name, "where", spec.Where(),
"proto", spec.Protocol, "deadline_s", int(netVerifyDeadline.Seconds()))
// Trigger a real mount: a directory read through the enabled automount mounts the share
// (spike Q1, ~1 s on a healthy LAN). The read may block until systemd resolves the mount
@@ -123,16 +127,24 @@ func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob)
go func() { trigger <- s.netTrigger(spec.Where()) }()
timedOut := false
select {
case <-trigger:
case terr := <-trigger:
// The read error is NOT the verdict (§8) — logged for the flow trace only.
s.logger.Debug("netverify: automount trigger returned",
"name", spec.Name, "read_err", fmt.Sprint(terr))
case <-ctx.Done():
timedOut = true
s.logger.Debug("netverify: deadline fired before the trigger resolved", "name", spec.Name)
}
// §8 truth table: /proc/mounts is the ONLY success judge. A trigger read error on a mounted
// share (EACCES on a 0700 export) is a GOOD mount — the controller's uid-1000 probe decides
// writability, not the agent user's readability.
if s.netMounted(spec.Where()) {
s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where())
mounted := s.netMounted(spec.Where())
s.logger.Debug("netverify: /proc/mounts verdict",
"name", spec.Name, "where", spec.Where(), "mounted", mounted, "timed_out", timedOut)
if mounted {
s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where(),
"duration_ms", time.Since(start).Milliseconds())
s.finishNetVerify(job, netVerifyPhaseDone, "", "")
return
}
@@ -142,7 +154,8 @@ func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob)
code, detail := s.classifyNetFailure(base, spec, timedOut)
s.rollbackNetMount(base, spec)
s.logger.Warn("netverify: verify failed — install rolled back",
"name", spec.Name, "code", code, "timed_out", timedOut)
"name", spec.Name, "code", code, "timed_out", timedOut,
"duration_ms", time.Since(start).Milliseconds())
s.finishNetVerify(job, netVerifyPhaseFailed, code, detail)
}()
return done
@@ -168,7 +181,9 @@ func (s *Server) classifyNetFailure(base context.Context, spec storage.NetworkMo
"unit", unit, "err", jerr)
return fallback, "journal unavailable — add the felhom-agent user to the systemd-journal group (usermod -aG systemd-journal felhom-agent)"
}
s.logger.Debug("netverify: journal tail read for classification", "unit", unit, "bytes", len(tail))
code, hint := storage.ClassifyNetVerifyFailure(tail, s.netReachable(spec.Protocol, spec.Server))
s.logger.Debug("netverify: failure classified", "name", spec.Name, "code", code, "timed_out", timedOut)
if code == storage.NetVerifyMountFailed && timedOut {
code = storage.NetVerifyTimeout
}
@@ -189,6 +204,8 @@ func (s *Server) rollbackNetMount(base context.Context, spec storage.NetworkMoun
if err := s.netStorage.RemoveNetworkMount(ctx, spec.Name); err != nil {
s.logger.Error("netverify: rollback RemoveNetworkMount failed (manual cleanup may be needed)",
"name", spec.Name, "err", err)
} else {
s.logger.Info("netverify: failed install rolled back (unit pair removed)", "name", spec.Name)
}
if spec.Protocol == storage.ProtocolSMB {
s.removeSMBCreds(spec.Name)