v0.83.0: observability pass — always-DEBUG capture ring + GET /debug/logs + heartbeat log-pull + gap-fill sweep
Capture layer: applog.New returns (logger, Ring) — slog fan-out, stderr at the configured level, ~1000-entry ring fixed at LevelDebug (remote diagnostics without a config flip). GET /debug/logs (token-authed, ?raw=1) + request-level DEBUG middleware. Heartbeat log-pull mirrors the report logtail pattern: envelope log_tail_requested -> next heartbeat carries log_tail (128KB cap, consume-once, failed-push retry proven). Gap-fill sweep over netverify/ netstorage/netmount/signedjobs/selfupdate/disks/controller-swap/desired/loop. Red-proofs: ring-at-emit-level FAILs capture test; drain removed FAILs consume-once; dropped phase line FAILs the S7 log-sequence smoke. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -112,8 +112,12 @@ func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
start := time.Now()
|
||||
ctx, cancel := context.WithTimeout(base, netVerifyDeadline)
|
||||
defer cancel()
|
||||
s.logger.Debug("netverify: job started",
|
||||
"job_id", job.JobID, "name", spec.Name, "where", spec.Where(),
|
||||
"proto", spec.Protocol, "deadline_s", int(netVerifyDeadline.Seconds()))
|
||||
|
||||
// Trigger a real mount: a directory read through the enabled automount mounts the share
|
||||
// (spike Q1, ~1 s on a healthy LAN). The read may block until systemd resolves the mount
|
||||
@@ -123,16 +127,24 @@ func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob)
|
||||
go func() { trigger <- s.netTrigger(spec.Where()) }()
|
||||
timedOut := false
|
||||
select {
|
||||
case <-trigger:
|
||||
case terr := <-trigger:
|
||||
// The read error is NOT the verdict (§8) — logged for the flow trace only.
|
||||
s.logger.Debug("netverify: automount trigger returned",
|
||||
"name", spec.Name, "read_err", fmt.Sprint(terr))
|
||||
case <-ctx.Done():
|
||||
timedOut = true
|
||||
s.logger.Debug("netverify: deadline fired before the trigger resolved", "name", spec.Name)
|
||||
}
|
||||
|
||||
// §8 truth table: /proc/mounts is the ONLY success judge. A trigger read error on a mounted
|
||||
// share (EACCES on a 0700 export) is a GOOD mount — the controller's uid-1000 probe decides
|
||||
// writability, not the agent user's readability.
|
||||
if s.netMounted(spec.Where()) {
|
||||
s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where())
|
||||
mounted := s.netMounted(spec.Where())
|
||||
s.logger.Debug("netverify: /proc/mounts verdict",
|
||||
"name", spec.Name, "where", spec.Where(), "mounted", mounted, "timed_out", timedOut)
|
||||
if mounted {
|
||||
s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where(),
|
||||
"duration_ms", time.Since(start).Milliseconds())
|
||||
s.finishNetVerify(job, netVerifyPhaseDone, "", "")
|
||||
return
|
||||
}
|
||||
@@ -142,7 +154,8 @@ func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob)
|
||||
code, detail := s.classifyNetFailure(base, spec, timedOut)
|
||||
s.rollbackNetMount(base, spec)
|
||||
s.logger.Warn("netverify: verify failed — install rolled back",
|
||||
"name", spec.Name, "code", code, "timed_out", timedOut)
|
||||
"name", spec.Name, "code", code, "timed_out", timedOut,
|
||||
"duration_ms", time.Since(start).Milliseconds())
|
||||
s.finishNetVerify(job, netVerifyPhaseFailed, code, detail)
|
||||
}()
|
||||
return done
|
||||
@@ -168,7 +181,9 @@ func (s *Server) classifyNetFailure(base context.Context, spec storage.NetworkMo
|
||||
"unit", unit, "err", jerr)
|
||||
return fallback, "journal unavailable — add the felhom-agent user to the systemd-journal group (usermod -aG systemd-journal felhom-agent)"
|
||||
}
|
||||
s.logger.Debug("netverify: journal tail read for classification", "unit", unit, "bytes", len(tail))
|
||||
code, hint := storage.ClassifyNetVerifyFailure(tail, s.netReachable(spec.Protocol, spec.Server))
|
||||
s.logger.Debug("netverify: failure classified", "name", spec.Name, "code", code, "timed_out", timedOut)
|
||||
if code == storage.NetVerifyMountFailed && timedOut {
|
||||
code = storage.NetVerifyTimeout
|
||||
}
|
||||
@@ -189,6 +204,8 @@ func (s *Server) rollbackNetMount(base context.Context, spec storage.NetworkMoun
|
||||
if err := s.netStorage.RemoveNetworkMount(ctx, spec.Name); err != nil {
|
||||
s.logger.Error("netverify: rollback RemoveNetworkMount failed (manual cleanup may be needed)",
|
||||
"name", spec.Name, "err", err)
|
||||
} else {
|
||||
s.logger.Info("netverify: failed install rolled back (unit pair removed)", "name", spec.Name)
|
||||
}
|
||||
if spec.Protocol == storage.ProtocolSMB {
|
||||
s.removeSMBCreds(spec.Name)
|
||||
|
||||
Reference in New Issue
Block a user