v0.83.0: observability pass — always-DEBUG capture ring + GET /debug/logs + heartbeat log-pull + gap-fill sweep

Capture layer: applog.New returns (logger, Ring) — slog fan-out, stderr at the
configured level, ~1000-entry ring fixed at LevelDebug (remote diagnostics
without a config flip). GET /debug/logs (token-authed, ?raw=1) + request-level
DEBUG middleware. Heartbeat log-pull mirrors the report logtail pattern:
envelope log_tail_requested -> next heartbeat carries log_tail (128KB cap,
consume-once, failed-push retry proven). Gap-fill sweep over netverify/
netstorage/netmount/signedjobs/selfupdate/disks/controller-swap/desired/loop.
Red-proofs: ring-at-emit-level FAILs capture test; drain removed FAILs
consume-once; dropped phase line FAILs the S7 log-sequence smoke.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

internal/hub/loop_logtail_test.go

@@ -0,0 +1,121 @@
+package hub
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+// recordingReporter records each pushed report and serves a scripted per-call
+// (envelope, error) sequence — the S2 heartbeat log-pull harness.
+type recordingReporter struct {
+	reports []*HostReport
+	script  []struct {
+		env *ControlEnvelope
+		err error
+	}
+}
+
+func (r *recordingReporter) Report(_ context.Context, rep *HostReport) (*ControlEnvelope, error) {
+	// Copy the LogTail pointer state at push time (the loop reuses collector reports).
+	cp := *rep
+	r.reports = append(r.reports, &cp)
+	i := len(r.reports) - 1
+	if i < len(r.script) {
+		return r.script[i].env, r.script[i].err
+	}
+	return &ControlEnvelope{}, nil
+}
+
+func tailLoop(rep *recordingReporter) *Loop {
+	var cn int32
+	l := NewLoop(&fakeCollector{report: &HostReport{}, n: &cn}, rep, time.Hour, quietLogger())
+	l.SetLogTailSource(func(maxBytes int) []string { return []string{"line-a", "line-b"} })
+	return l
+}
+
+// S2 (agent half): an envelope's log_tail_requested arms the pull; the NEXT report
+// carries log_tail; the one after (request cleared hub-side) carries nothing —
+// consume-once. Companion red-proof: drop the `l.logTailPending = false` drain →
+// report 3 also carries a tail → the last assertion fails.
+func TestLoop_LogTailRequestedShipsOnNextReportOnce(t *testing.T) {
+	rep := &recordingReporter{script: []struct {
+		env *ControlEnvelope
+		err error
+	}{
+		{env: &ControlEnvelope{LogTailRequested: true}},
+		{env: &ControlEnvelope{}}, // the tail arrived — hub cleared the request
+		{env: &ControlEnvelope{}},
+	}}
+	l := tailLoop(rep)
+	ctx := context.Background()
+	l.cycle(ctx, time.Hour)
+	l.cycle(ctx, time.Hour)
+	l.cycle(ctx, time.Hour)
+
+	if len(rep.reports) != 3 {
+		t.Fatalf("reports = %d, want 3", len(rep.reports))
+	}
+	if rep.reports[0].LogTail != nil {
+		t.Errorf("report 1 must not carry a tail (the request only arrived in its envelope)")
+	}
+	got := rep.reports[1].LogTail
+	if got == nil || len(got.Lines) != 2 || got.Lines[0] != "line-a" || got.CollectedAt == "" {
+		t.Fatalf("report 2 log_tail = %+v, want the 2 ring lines + collected_at", got)
+	}
+	if rep.reports[2].LogTail != nil {
+		t.Errorf("report 3 carries a tail again — consume-once broken: %+v", rep.reports[2].LogTail)
+	}
+}
+
+// S2 companion (fail-safe retry): the push CARRYING the tail fails → the local pending
+// is spent, but the hub's request is still pending, so the next envelope re-arms it and
+// the following report fulfills. Asserts the retry ships the tail exactly once more.
+func TestLoop_FailedTailPushIsReArmedByNextEnvelope(t *testing.T) {
+	rep := &recordingReporter{script: []struct {
+		env *ControlEnvelope
+		err error
+	}{
+		{env: &ControlEnvelope{LogTailRequested: true}},          // arm
+		{err: errors.New("hub 5xx")},                             // the carrying push FAILS
+		{env: &ControlEnvelope{LogTailRequested: true}},          // hub still pending → re-arm
+		{env: &ControlEnvelope{}},                                // fulfilled
+	}}
+	l := tailLoop(rep)
+	ctx := context.Background()
+	for i := 0; i < 4; i++ {
+		l.cycle(ctx, time.Hour)
+	}
+	if len(rep.reports) != 4 {
+		t.Fatalf("reports = %d, want 4", len(rep.reports))
+	}
+	if rep.reports[1].LogTail == nil {
+		t.Errorf("report 2 (the failed push) should have carried the tail")
+	}
+	if rep.reports[2].LogTail != nil {
+		t.Errorf("report 3 must not carry a tail (pending was spent; envelope re-arms only after it)")
+	}
+	if rep.reports[3].LogTail == nil {
+		t.Errorf("report 4 must fulfill the re-armed request — retry lost")
+	}
+}
+
+// No source wired → the request is ignored (clean no-op, no panic).
+func TestLoop_LogTailRequestIgnoredWithoutSource(t *testing.T) {
+	rep := &recordingReporter{script: []struct {
+		env *ControlEnvelope
+		err error
+	}{
+		{env: &ControlEnvelope{LogTailRequested: true}},
+		{env: &ControlEnvelope{}},
+	}}
+	var cn int32
+	l := NewLoop(&fakeCollector{report: &HostReport{}, n: &cn}, rep, time.Hour, quietLogger())
+	ctx := context.Background()
+	l.cycle(ctx, time.Hour)
+	l.cycle(ctx, time.Hour)
+	if rep.reports[1].LogTail != nil {
+		t.Errorf("tail shipped with no source wired: %+v", rep.reports[1].LogTail)
+	}
+}