v0.83.0: observability pass — always-DEBUG capture ring + GET /debug/logs + heartbeat log-pull + gap-fill sweep
Capture layer: applog.New returns (logger, Ring) — slog fan-out, stderr at the configured level, ~1000-entry ring fixed at LevelDebug (remote diagnostics without a config flip). GET /debug/logs (token-authed, ?raw=1) + request-level DEBUG middleware. Heartbeat log-pull mirrors the report logtail pattern: envelope log_tail_requested -> next heartbeat carries log_tail (128KB cap, consume-once, failed-push retry proven). Gap-fill sweep over netverify/ netstorage/netmount/signedjobs/selfupdate/disks/controller-swap/desired/loop. Red-proofs: ring-at-emit-level FAILs capture test; drain removed FAILs consume-once; dropped phase line FAILs the S7 log-sequence smoke. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
+38
-1
@@ -57,8 +57,20 @@ type Loop struct {
|
||||
logger *slog.Logger
|
||||
trigger <-chan struct{} // optional: an out-of-band report request (storage watchdog)
|
||||
observer EnvelopeObserver // optional: the slice-10A desired-state sync hook
|
||||
|
||||
// Heartbeat log-pull (v0.83.0): logTailSource yields the debug ring's formatted
|
||||
// lines newest-kept within a byte budget (applog.Ring.Lines). logTailPending is
|
||||
// armed by an envelope's log_tail_requested and drained onto the NEXT report —
|
||||
// the report-channel logtail.go consume-once shape: a failed push leaves the
|
||||
// hub's request pending, so the next successful envelope re-arms it (fail-safe
|
||||
// retry, no duplicate shipping). Loop state is single-goroutine (cycle only).
|
||||
logTailSource func(maxBytes int) []string
|
||||
logTailPending bool
|
||||
}
|
||||
|
||||
// logTailMaxBytes caps the heartbeat log tail (newest lines kept).
|
||||
const logTailMaxBytes = 128 * 1024
|
||||
|
||||
// NewLoop builds the loop. interval is the starting cadence (the hub may override it
|
||||
// per-cycle via the control envelope).
|
||||
func NewLoop(collector collectorIface, client reporter, interval time.Duration, logger *slog.Logger) *Loop {
|
||||
@@ -79,6 +91,10 @@ func (l *Loop) SetTrigger(ch <-chan struct{}) { l.trigger = ch }
|
||||
// desired-state when the generation advances. Optional — unset is a clean no-op.
|
||||
func (l *Loop) SetEnvelopeObserver(o EnvelopeObserver) { l.observer = o }
|
||||
|
||||
// SetLogTailSource wires the debug ring for the heartbeat log-pull (v0.83.0).
|
||||
// Optional — unset means an envelope's log_tail_requested is ignored.
|
||||
func (l *Loop) SetLogTailSource(src func(maxBytes int) []string) { l.logTailSource = src }
|
||||
|
||||
// Run reports immediately, then on each tick, until ctx is cancelled (then nil).
|
||||
func (l *Loop) Run(ctx context.Context) error {
|
||||
interval := l.interval
|
||||
@@ -116,19 +132,40 @@ func (l *Loop) Run(ctx context.Context) error {
|
||||
// cycle runs one collect→report→adopt. It never returns an error: failures are
|
||||
// logged and the current interval is kept, so the loop keeps running.
|
||||
func (l *Loop) cycle(ctx context.Context, current time.Duration) time.Duration {
|
||||
start := time.Now()
|
||||
report, err := l.collector.Collect(ctx)
|
||||
if err != nil {
|
||||
l.logger.Warn("hub: collect failed; skipping this cycle's report", "err", err)
|
||||
return current
|
||||
}
|
||||
// Fulfill a pending log-pull: attach the ring tail to THIS report and clear the
|
||||
// local pending flag (consume-once). On a failed push the hub's request is still
|
||||
// pending and the next envelope re-arms it — logtail.go's fail-safe retry shape.
|
||||
// The explicit nil first makes this robust to a collector reusing its report struct.
|
||||
report.LogTail = nil
|
||||
if l.logTailPending && l.logTailSource != nil {
|
||||
report.LogTail = &LogTail{
|
||||
CollectedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Lines: l.logTailSource(logTailMaxBytes),
|
||||
}
|
||||
}
|
||||
l.logTailPending = false
|
||||
env, err := l.client.Report(ctx, report)
|
||||
if err != nil {
|
||||
l.logger.Warn("hub: report failed; keeping current interval", "err", err)
|
||||
return current
|
||||
}
|
||||
if report.LogTail != nil {
|
||||
// Transparency: the pull is visible in the box's own log (and thus in the ring).
|
||||
l.logger.Info("operator log pull served", "component", "agent", "lines", len(report.LogTail.Lines))
|
||||
}
|
||||
l.logger.Debug("hub: report sent",
|
||||
"guests", len(report.Guests),
|
||||
"guests", len(report.Guests), "duration_ms", time.Since(start).Milliseconds(),
|
||||
"blocked", env.Blocked, "desired_generation", env.DesiredGeneration, "has_signed_ops", env.HasSignedOps)
|
||||
if env.LogTailRequested {
|
||||
l.logger.Debug("hub: log tail requested — shipping on the next heartbeat")
|
||||
l.logTailPending = true
|
||||
}
|
||||
|
||||
// Slice 10A: hand the envelope to the desired-state sync hook (fetch desired-state on a
|
||||
// generation advance). Done off the report's critical path semantics — a sync/fetch failure
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// recordingReporter records each pushed report and serves a scripted per-call
|
||||
// (envelope, error) sequence — the S2 heartbeat log-pull harness.
|
||||
type recordingReporter struct {
|
||||
reports []*HostReport
|
||||
script []struct {
|
||||
env *ControlEnvelope
|
||||
err error
|
||||
}
|
||||
}
|
||||
|
||||
func (r *recordingReporter) Report(_ context.Context, rep *HostReport) (*ControlEnvelope, error) {
|
||||
// Copy the LogTail pointer state at push time (the loop reuses collector reports).
|
||||
cp := *rep
|
||||
r.reports = append(r.reports, &cp)
|
||||
i := len(r.reports) - 1
|
||||
if i < len(r.script) {
|
||||
return r.script[i].env, r.script[i].err
|
||||
}
|
||||
return &ControlEnvelope{}, nil
|
||||
}
|
||||
|
||||
func tailLoop(rep *recordingReporter) *Loop {
|
||||
var cn int32
|
||||
l := NewLoop(&fakeCollector{report: &HostReport{}, n: &cn}, rep, time.Hour, quietLogger())
|
||||
l.SetLogTailSource(func(maxBytes int) []string { return []string{"line-a", "line-b"} })
|
||||
return l
|
||||
}
|
||||
|
||||
// S2 (agent half): an envelope's log_tail_requested arms the pull; the NEXT report
|
||||
// carries log_tail; the one after (request cleared hub-side) carries nothing —
|
||||
// consume-once. Companion red-proof: drop the `l.logTailPending = false` drain →
|
||||
// report 3 also carries a tail → the last assertion fails.
|
||||
func TestLoop_LogTailRequestedShipsOnNextReportOnce(t *testing.T) {
|
||||
rep := &recordingReporter{script: []struct {
|
||||
env *ControlEnvelope
|
||||
err error
|
||||
}{
|
||||
{env: &ControlEnvelope{LogTailRequested: true}},
|
||||
{env: &ControlEnvelope{}}, // the tail arrived — hub cleared the request
|
||||
{env: &ControlEnvelope{}},
|
||||
}}
|
||||
l := tailLoop(rep)
|
||||
ctx := context.Background()
|
||||
l.cycle(ctx, time.Hour)
|
||||
l.cycle(ctx, time.Hour)
|
||||
l.cycle(ctx, time.Hour)
|
||||
|
||||
if len(rep.reports) != 3 {
|
||||
t.Fatalf("reports = %d, want 3", len(rep.reports))
|
||||
}
|
||||
if rep.reports[0].LogTail != nil {
|
||||
t.Errorf("report 1 must not carry a tail (the request only arrived in its envelope)")
|
||||
}
|
||||
got := rep.reports[1].LogTail
|
||||
if got == nil || len(got.Lines) != 2 || got.Lines[0] != "line-a" || got.CollectedAt == "" {
|
||||
t.Fatalf("report 2 log_tail = %+v, want the 2 ring lines + collected_at", got)
|
||||
}
|
||||
if rep.reports[2].LogTail != nil {
|
||||
t.Errorf("report 3 carries a tail again — consume-once broken: %+v", rep.reports[2].LogTail)
|
||||
}
|
||||
}
|
||||
|
||||
// S2 companion (fail-safe retry): the push CARRYING the tail fails → the local pending
|
||||
// is spent, but the hub's request is still pending, so the next envelope re-arms it and
|
||||
// the following report fulfills. Asserts the retry ships the tail exactly once more.
|
||||
func TestLoop_FailedTailPushIsReArmedByNextEnvelope(t *testing.T) {
|
||||
rep := &recordingReporter{script: []struct {
|
||||
env *ControlEnvelope
|
||||
err error
|
||||
}{
|
||||
{env: &ControlEnvelope{LogTailRequested: true}}, // arm
|
||||
{err: errors.New("hub 5xx")}, // the carrying push FAILS
|
||||
{env: &ControlEnvelope{LogTailRequested: true}}, // hub still pending → re-arm
|
||||
{env: &ControlEnvelope{}}, // fulfilled
|
||||
}}
|
||||
l := tailLoop(rep)
|
||||
ctx := context.Background()
|
||||
for i := 0; i < 4; i++ {
|
||||
l.cycle(ctx, time.Hour)
|
||||
}
|
||||
if len(rep.reports) != 4 {
|
||||
t.Fatalf("reports = %d, want 4", len(rep.reports))
|
||||
}
|
||||
if rep.reports[1].LogTail == nil {
|
||||
t.Errorf("report 2 (the failed push) should have carried the tail")
|
||||
}
|
||||
if rep.reports[2].LogTail != nil {
|
||||
t.Errorf("report 3 must not carry a tail (pending was spent; envelope re-arms only after it)")
|
||||
}
|
||||
if rep.reports[3].LogTail == nil {
|
||||
t.Errorf("report 4 must fulfill the re-armed request — retry lost")
|
||||
}
|
||||
}
|
||||
|
||||
// No source wired → the request is ignored (clean no-op, no panic).
|
||||
func TestLoop_LogTailRequestIgnoredWithoutSource(t *testing.T) {
|
||||
rep := &recordingReporter{script: []struct {
|
||||
env *ControlEnvelope
|
||||
err error
|
||||
}{
|
||||
{env: &ControlEnvelope{LogTailRequested: true}},
|
||||
{env: &ControlEnvelope{}},
|
||||
}}
|
||||
var cn int32
|
||||
l := NewLoop(&fakeCollector{report: &HostReport{}, n: &cn}, rep, time.Hour, quietLogger())
|
||||
ctx := context.Background()
|
||||
l.cycle(ctx, time.Hour)
|
||||
l.cycle(ctx, time.Hour)
|
||||
if rep.reports[1].LogTail != nil {
|
||||
t.Errorf("tail shipped with no source wired: %+v", rep.reports[1].LogTail)
|
||||
}
|
||||
}
|
||||
@@ -85,6 +85,15 @@ type HostReport struct {
|
||||
// Carries NO secret.
|
||||
PBSDR *PBSDRStatus `json:"pbs_dr,omitempty"`
|
||||
|
||||
// LogTail is the agent's on-demand debug-ring tail (v0.83.0 observability) — the agent
|
||||
// mirror of the controller's report log_tails channel. Present ONLY on the heartbeat
|
||||
// right after the control envelope requested it (log_tail_requested); consume-once on
|
||||
// both ends (the hub clears its pending request on arrival). Newest lines kept, byte-
|
||||
// capped loop-side. Carries log lines only — the logging conventions forbid secrets in
|
||||
// any log line, and the hub's bundle gate re-checks before storing. `omitempty`: absent
|
||||
// in the steady state, so the cross-repo host-report golden stays byte-stable.
|
||||
LogTail *LogTail `json:"log_tail,omitempty"`
|
||||
|
||||
// OOB is the operator-access health stanza (TASK H1). It answers the operator's question — "can I
|
||||
// get into this box right now, and if not, why" — from the hub: felhom-sshd up + on which port,
|
||||
// locally reachable, the tunnel handshake age (the OOB path rides wg-felhom), whether the operator
|
||||
@@ -368,6 +377,16 @@ type ControlEnvelope struct {
|
||||
Blocked bool `json:"blocked"` // reserved — ignored
|
||||
DesiredGeneration int64 `json:"desired_generation"` // slice 10A: the cached-vs-current change signal
|
||||
HasSignedOps bool `json:"has_signed_ops"` // slice 10A: signed-jobs queue non-empty (exec 10B)
|
||||
// LogTailRequested (v0.83.0) — the operator wants this agent's debug-ring tail; the
|
||||
// NEXT heartbeat carries it in log_tail (the report-channel log_tail_requests mirror).
|
||||
// Absent/false on an old hub → nothing happens.
|
||||
LogTailRequested bool `json:"log_tail_requested"`
|
||||
}
|
||||
|
||||
// LogTail is the heartbeat's on-demand agent log tail (see HostReport.LogTail).
|
||||
type LogTail struct {
|
||||
CollectedAt string `json:"collected_at"` // RFC3339
|
||||
Lines []string `json:"lines"`
|
||||
}
|
||||
|
||||
// DesiredStateResponse is GET /hosts/{host_id}/desired-state (slice 10A — the "Down" channel's
|
||||
|
||||
Reference in New Issue
Block a user