v0.83.0: observability pass — always-DEBUG capture ring + GET /debug/logs + heartbeat log-pull + gap-fill sweep
Capture layer: applog.New returns (logger, Ring) — slog fan-out, stderr at the configured level, ~1000-entry ring fixed at LevelDebug (remote diagnostics without a config flip). GET /debug/logs (token-authed, ?raw=1) + request-level DEBUG middleware. Heartbeat log-pull mirrors the report logtail pattern: envelope log_tail_requested -> next heartbeat carries log_tail (128KB cap, consume-once, failed-push retry proven). Gap-fill sweep over netverify/ netstorage/netmount/signedjobs/selfupdate/disks/controller-swap/desired/loop. Red-proofs: ring-at-emit-level FAILs capture test; drain removed FAILs consume-once; dropped phase line FAILs the S7 log-sequence smoke. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
+38
-1
@@ -57,8 +57,20 @@ type Loop struct {
|
||||
logger *slog.Logger
|
||||
trigger <-chan struct{} // optional: an out-of-band report request (storage watchdog)
|
||||
observer EnvelopeObserver // optional: the slice-10A desired-state sync hook
|
||||
|
||||
// Heartbeat log-pull (v0.83.0): logTailSource yields the debug ring's formatted
|
||||
// lines newest-kept within a byte budget (applog.Ring.Lines). logTailPending is
|
||||
// armed by an envelope's log_tail_requested and drained onto the NEXT report —
|
||||
// the report-channel logtail.go consume-once shape: a failed push leaves the
|
||||
// hub's request pending, so the next successful envelope re-arms it (fail-safe
|
||||
// retry, no duplicate shipping). Loop state is single-goroutine (cycle only).
|
||||
logTailSource func(maxBytes int) []string
|
||||
logTailPending bool
|
||||
}
|
||||
|
||||
// logTailMaxBytes caps the heartbeat log tail (newest lines kept).
|
||||
const logTailMaxBytes = 128 * 1024
|
||||
|
||||
// NewLoop builds the loop. interval is the starting cadence (the hub may override it
|
||||
// per-cycle via the control envelope).
|
||||
func NewLoop(collector collectorIface, client reporter, interval time.Duration, logger *slog.Logger) *Loop {
|
||||
@@ -79,6 +91,10 @@ func (l *Loop) SetTrigger(ch <-chan struct{}) { l.trigger = ch }
|
||||
// desired-state when the generation advances. Optional — unset is a clean no-op.
|
||||
func (l *Loop) SetEnvelopeObserver(o EnvelopeObserver) { l.observer = o }
|
||||
|
||||
// SetLogTailSource wires the debug ring for the heartbeat log-pull (v0.83.0).
|
||||
// Optional — unset means an envelope's log_tail_requested is ignored.
|
||||
func (l *Loop) SetLogTailSource(src func(maxBytes int) []string) { l.logTailSource = src }
|
||||
|
||||
// Run reports immediately, then on each tick, until ctx is cancelled (then nil).
|
||||
func (l *Loop) Run(ctx context.Context) error {
|
||||
interval := l.interval
|
||||
@@ -116,19 +132,40 @@ func (l *Loop) Run(ctx context.Context) error {
|
||||
// cycle runs one collect→report→adopt. It never returns an error: failures are
|
||||
// logged and the current interval is kept, so the loop keeps running.
|
||||
func (l *Loop) cycle(ctx context.Context, current time.Duration) time.Duration {
|
||||
start := time.Now()
|
||||
report, err := l.collector.Collect(ctx)
|
||||
if err != nil {
|
||||
l.logger.Warn("hub: collect failed; skipping this cycle's report", "err", err)
|
||||
return current
|
||||
}
|
||||
// Fulfill a pending log-pull: attach the ring tail to THIS report and clear the
|
||||
// local pending flag (consume-once). On a failed push the hub's request is still
|
||||
// pending and the next envelope re-arms it — logtail.go's fail-safe retry shape.
|
||||
// The explicit nil first makes this robust to a collector reusing its report struct.
|
||||
report.LogTail = nil
|
||||
if l.logTailPending && l.logTailSource != nil {
|
||||
report.LogTail = &LogTail{
|
||||
CollectedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Lines: l.logTailSource(logTailMaxBytes),
|
||||
}
|
||||
}
|
||||
l.logTailPending = false
|
||||
env, err := l.client.Report(ctx, report)
|
||||
if err != nil {
|
||||
l.logger.Warn("hub: report failed; keeping current interval", "err", err)
|
||||
return current
|
||||
}
|
||||
if report.LogTail != nil {
|
||||
// Transparency: the pull is visible in the box's own log (and thus in the ring).
|
||||
l.logger.Info("operator log pull served", "component", "agent", "lines", len(report.LogTail.Lines))
|
||||
}
|
||||
l.logger.Debug("hub: report sent",
|
||||
"guests", len(report.Guests),
|
||||
"guests", len(report.Guests), "duration_ms", time.Since(start).Milliseconds(),
|
||||
"blocked", env.Blocked, "desired_generation", env.DesiredGeneration, "has_signed_ops", env.HasSignedOps)
|
||||
if env.LogTailRequested {
|
||||
l.logger.Debug("hub: log tail requested — shipping on the next heartbeat")
|
||||
l.logTailPending = true
|
||||
}
|
||||
|
||||
// Slice 10A: hand the envelope to the desired-state sync hook (fetch desired-state on a
|
||||
// generation advance). Done off the report's critical path semantics — a sync/fetch failure
|
||||
|
||||
Reference in New Issue
Block a user