v0.111.0: remote app-log diagnostics — error context capture (±5 lines, capped+redacted) + on-demand log tails via ACK pull pattern

- metrics: LogIssue.Context on first-occurrence errors (≤11 lines, ≤400 chars/line, warns carry none); RedactLine sanitizer (password/token/api-key/bearer/64-hex) applied to everything shipped; FetchContainerLogTail
- report: 16KB per-report context budget (lowest-count issues dropped first); log_tail_requests ACK flag → next report ships log_tails (200 lines, ≤64KB/app head-truncated, ordered, redacted); consume-once drain
- tests: synthetic-window context capture, caps, redaction, budget order, consume-once, fetch-error skip

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

controller/internal/report/builder.go

@@ -155,6 +155,19 @@ func BuildReport(
 	// App telemetry (metrics + log scan)
 	r.AppTelemetry = buildAppTelemetrySection(stackMgr, metricsStore, logger)
 
+	// On-demand log tails (v0.111.0): drain the ACK-requested apps and collect their tails.
+	// Stacks go through the existing compose-logs plumbing; the controller's own container
+	// through the scanner's docker-logs plumbing. Consume-once: drained here, re-armed by
+	// the next ACK if the push fails.
+	if tailApps := drainPendingLogTails(); len(tailApps) > 0 {
+		r.LogTails = buildLogTailsSection(tailApps, func(app string, lines int) (string, error) {
+			if app == controllerContainerName {
+				return metrics.FetchContainerLogTail(app, lines)
+			}
+			return stackMgr.GetLogs(app, lines)
+		}, logger)
+	}
+
 	// Geo-restriction status — ALWAYS present (even when never configured) so the hub
 	// always renders the section. A nil pointer (omitempty) made the hub hide the whole
 	// section for a never-configured controller; a present-but-disabled report renders

controller/internal/report/logtail.go

@@ -0,0 +1,109 @@
+package report
+
+import (
+	"log"
+	"strings"
+	"sync"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
+)
+
+// On-demand log tail (pull-based, same ACK-flag pattern as escrow/config-refresh):
+// the hub stores a pending request per (customer, app); the report ACK advertises the
+// requested app names in log_tail_requests; the NEXT report carries the collected tails
+// in log_tails. The hub clears the pending request when a tail arrives (consume-once) —
+// the controller listens to no one, it only pushes on its own cycle.
+
+const (
+	logTailLines        = 200
+	logTailMaxLineChars = 400
+	logTailMaxBytes     = 64 * 1024 // per app; head-truncated (newest lines kept)
+)
+
+// LogTail is one app's on-demand ordered log tail riding the report.
+type LogTail struct {
+	App         string    `json:"app"`
+	CollectedAt time.Time `json:"collected_at"`
+	Lines       []string  `json:"lines"`
+}
+
+var (
+	pendingTailsMu sync.Mutex
+	pendingTails   []string
+)
+
+// SetPendingLogTails records the hub-requested apps from the latest ACK. The hub is
+// the source of truth: an ACK without requests clears any stale local pending set.
+func SetPendingLogTails(apps []string) {
+	pendingTailsMu.Lock()
+	defer pendingTailsMu.Unlock()
+	pendingTails = append([]string(nil), apps...)
+}
+
+// drainPendingLogTails takes and CLEARS the pending set (consume-once controller-side).
+// If the subsequent push fails, the hub's request is still pending and the next ACK
+// re-arms it — fail-safe retry, no duplicate shipping.
+func drainPendingLogTails() []string {
+	pendingTailsMu.Lock()
+	defer pendingTailsMu.Unlock()
+	apps := pendingTails
+	pendingTails = nil
+	return apps
+}
+
+// buildLogTailsSection collects the requested tails via fetch (seam: stacks.GetLogs or
+// metrics.FetchContainerLogTail). Lines stay ordered as emitted; each line is truncated
+// to 400 chars and redacted; per-app total is capped at 64KB keeping the NEWEST lines.
+// A failed fetch is skipped (logged) — the hub request stays pending and retries next cycle.
+func buildLogTailsSection(apps []string, fetch func(app string, lines int) (string, error), logger *log.Logger) []LogTail {
+	var tails []LogTail
+	for _, app := range apps {
+		raw, err := fetch(app, logTailLines)
+		if err != nil {
+			if logger != nil {
+				logger.Printf("[WARN] [report] log-tail fetch for %s failed: %v", app, err)
+			}
+			continue
+		}
+		lines := capTailLines(strings.Split(raw, "\n"))
+		tails = append(tails, LogTail{App: app, CollectedAt: time.Now().UTC(), Lines: lines})
+		if logger != nil {
+			logger.Printf("[INFO] [report] log-tail collected for %s: %d lines", app, len(lines))
+		}
+	}
+	return tails
+}
+
+// capTailLines truncates each line to logTailMaxLineChars, redacts it, and enforces the
+// per-app byte budget by dropping from the HEAD (oldest) so the newest lines survive.
+func capTailLines(lines []string) []string {
+	// Drop a trailing empty split artifact.
+	for len(lines) > 0 && lines[len(lines)-1] == "" {
+		lines = lines[:len(lines)-1]
+	}
+	out := make([]string, 0, len(lines))
+	for _, l := range lines {
+		l = metrics.RedactLine(truncateTailLine(l, logTailMaxLineChars))
+		out = append(out, l)
+	}
+	// Head-truncate to the byte budget: walk from the end accumulating.
+	total := 0
+	start := len(out)
+	for i := len(out) - 1; i >= 0; i-- {
+		total += len(out[i]) + 1 // +1 for the newline it represents
+		if total > logTailMaxBytes {
+			break
+		}
+		start = i
+	}
+	return out[start:]
+}
+
+func truncateTailLine(s string, max int) string {
+	runes := []rune(s)
+	if len(runes) <= max {
+		return s
+	}
+	return string(runes[:max]) + "…"
+}

controller/internal/report/logtail_test.go

@@ -0,0 +1,115 @@
+package report
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+// Part D — the consume-once seam: an ACK arms the pending set, the FIRST build drains
+// it, the FOLLOWING build gets nothing. Red-proof: removing the clear in
+// drainPendingLogTails makes the second drain return the apps again → this fails.
+func TestPendingLogTails_ConsumeOnce(t *testing.T) {
+	SetPendingLogTails([]string{"gokapi", "cwa"})
+
+	first := drainPendingLogTails()
+	if len(first) != 2 || first[0] != "gokapi" || first[1] != "cwa" {
+		t.Fatalf("first drain = %v, want the two requested apps", first)
+	}
+	second := drainPendingLogTails()
+	if len(second) != 0 {
+		t.Fatalf("second drain = %v — tails would ship on EVERY report (consume-once broken)", second)
+	}
+}
+
+// An ACK without requests clears any stale local set (hub is the source of truth).
+func TestPendingLogTails_EmptyAckClears(t *testing.T) {
+	SetPendingLogTails([]string{"gokapi"})
+	SetPendingLogTails(nil)
+	if got := drainPendingLogTails(); len(got) != 0 {
+		t.Fatalf("stale pending set survived an empty ACK: %v", got)
+	}
+}
+
+func TestBuildLogTailsSection_OrderedAndRedacted(t *testing.T) {
+	fetch := func(app string, lines int) (string, error) {
+		if lines != 200 {
+			t.Fatalf("fetch asked for %d lines, want 200", lines)
+		}
+		return "line one\nline two with password=hunter2\nline three\n", nil
+	}
+	tails := buildLogTailsSection([]string{"gokapi"}, fetch, nil)
+	if len(tails) != 1 || tails[0].App != "gokapi" {
+		t.Fatalf("tails = %+v", tails)
+	}
+	got := tails[0].Lines
+	if len(got) != 3 {
+		t.Fatalf("lines = %d (trailing empty must be dropped): %#v", len(got), got)
+	}
+	if got[0] != "line one" || got[2] != "line three" {
+		t.Fatalf("order broken: %#v", got)
+	}
+	if strings.Contains(got[1], "hunter2") || !strings.Contains(got[1], "[REDACTED]") {
+		t.Fatalf("tail line not redacted: %q", got[1])
+	}
+	if tails[0].CollectedAt.IsZero() {
+		t.Fatalf("collected_at not stamped")
+	}
+}
+
+func TestBuildLogTailsSection_LineLengthCap(t *testing.T) {
+	fetch := func(string, int) (string, error) {
+		return strings.Repeat("y", 900) + "\nshort", nil
+	}
+	tails := buildLogTailsSection([]string{"a"}, fetch, nil)
+	first := tails[0].Lines[0]
+	if utf8.RuneCountInString(first) != 401 || !strings.HasSuffix(first, "…") {
+		t.Fatalf("line not capped at 400+ellipsis: runes=%d", utf8.RuneCountInString(first))
+	}
+}
+
+// 64KB per-app budget: HEAD-truncate — the newest lines must survive.
+func TestBuildLogTailsSection_ByteBudgetKeepsNewest(t *testing.T) {
+	var sb strings.Builder
+	for i := 0; i < 300; i++ {
+		sb.WriteString(fmt.Sprintf("%04d ", i))
+		sb.WriteString(strings.Repeat("p", 295))
+		sb.WriteString("\n")
+	}
+	fetch := func(string, int) (string, error) { return sb.String(), nil }
+	tails := buildLogTailsSection([]string{"a"}, fetch, nil)
+	lines := tails[0].Lines
+	if len(lines) >= 300 {
+		t.Fatalf("budget not enforced: %d lines kept", len(lines))
+	}
+	total := 0
+	for _, l := range lines {
+		total += len(l) + 1
+	}
+	if total > logTailMaxBytes {
+		t.Fatalf("kept %d bytes > %d budget", total, logTailMaxBytes)
+	}
+	if !strings.HasPrefix(lines[len(lines)-1], "0299") {
+		t.Fatalf("NEWEST line lost — head-truncation inverted; last kept = %q", lines[len(lines)-1][:5])
+	}
+	if strings.HasPrefix(lines[0], "0000") {
+		t.Fatalf("oldest line survived a budget overflow — nothing was dropped")
+	}
+}
+
+// A failed fetch skips that app but the others still ship (hub retries the failed one
+// next cycle since its pending request was never fulfilled).
+func TestBuildLogTailsSection_FetchErrorSkips(t *testing.T) {
+	fetch := func(app string, _ int) (string, error) {
+		if app == "broken" {
+			return "", errors.New("no such stack")
+		}
+		return "ok line\n", nil
+	}
+	tails := buildLogTailsSection([]string{"broken", "healthy"}, fetch, nil)
+	if len(tails) != 1 || tails[0].App != "healthy" {
+		t.Fatalf("tails = %+v, want only the healthy app", tails)
+	}
+}

controller/internal/report/pusher.go

@@ -40,6 +40,10 @@ type PushResponse struct {
 	// Escrow (SLICE 3) is the hub's escrow status for this customer — the input to the hub-verified
 	// auto-confirm (EscrowAutoConfirmer). nil = no escrow row on the hub (or an old hub) → stays pending.
 	Escrow *EscrowStatus `json:"escrow"`
+	// LogTailRequests (v0.111.0) — app names the operator wants a log tail for. Same
+	// pull-based ACK-flag pattern as escrow: the NEXT report ships the tails; the hub
+	// clears the pending request on receipt (consume-once). Absent/empty = nothing pending.
+	LogTailRequests []string `json:"log_tail_requests"`
 }
 
 // Pusher sends reports to the central hub.

controller/internal/report/telemetry.go

@@ -59,9 +59,52 @@ func buildAppTelemetrySection(stackMgr *stacks.Manager, metricsStore *metrics.Me
 	if result == nil {
 		result = []AppTelemetry{}
 	}
+
+	// 7. Enforce the per-report context budget (v0.111.0): drop context from the
+	// lowest-count issues first until under 16KB. Warns never carry context (scanner).
+	enforceContextBudget(result, contextBudgetBytes)
+
 	return result
 }
 
+// contextBudgetBytes is the HARD per-report total for all LogIssue.Context lines.
+const contextBudgetBytes = 16 * 1024
+
+// enforceContextBudget drops Context from the lowest-count issues (across all apps)
+// until the summed context bytes fit the budget. Mutates apps in place.
+func enforceContextBudget(apps []AppTelemetry, budget int) {
+	type ref struct {
+		app, issue, size, count int
+	}
+	var refs []ref
+	total := 0
+	for ai := range apps {
+		for ii := range apps[ai].Issues {
+			ctx := apps[ai].Issues[ii].Context
+			if len(ctx) == 0 {
+				continue
+			}
+			sz := 0
+			for _, l := range ctx {
+				sz += len(l)
+			}
+			refs = append(refs, ref{app: ai, issue: ii, size: sz, count: apps[ai].Issues[ii].Count})
+			total += sz
+		}
+	}
+	if total <= budget {
+		return
+	}
+	sort.Slice(refs, func(i, j int) bool { return refs[i].count < refs[j].count })
+	for _, r := range refs {
+		if total <= budget {
+			break
+		}
+		apps[r.app].Issues[r.issue].Context = nil
+		total -= r.size
+	}
+}
+
 // buildAppTelemetry aggregates container-level telemetry and log data into per-stack AppTelemetry entries.
 func buildAppTelemetry(allStacks []stacks.Stack, telemetry []metrics.ContainerTelemetry, logs []metrics.ContainerLogSummary) []AppTelemetry {
 	// Build lookup maps

controller/internal/report/telemetry_test.go

@@ -0,0 +1,48 @@
+package report
+
+import (
+	"strings"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
+)
+
+func appWithIssue(name string, count, ctxBytes int) AppTelemetry {
+	var ctx []string
+	if ctxBytes > 0 {
+		ctx = []string{strings.Repeat("c", ctxBytes)}
+	}
+	return AppTelemetry{
+		AppName: name,
+		Issues:  []metrics.LogIssue{{Severity: "error", Message: name + " boom", Count: count, Context: ctx}},
+	}
+}
+
+// Part B caps — budget overflow drops context from the LOWEST-count issues first,
+// across apps. Red-proof: inverting the sort (highest first) keeps the wrong survivor.
+func TestEnforceContextBudget_DropsLowestCountFirst(t *testing.T) {
+	apps := []AppTelemetry{
+		appWithIssue("rare", 2, 600),      // lowest count — first to lose context
+		appWithIssue("frequent", 50, 600), // must keep context
+		appWithIssue("medium", 10, 600),   // dropped second
+	}
+	enforceContextBudget(apps, 1000) // total 1800 → must shed 800 → drop rare, then medium
+
+	if apps[0].Issues[0].Context != nil {
+		t.Fatalf("lowest-count issue kept its context")
+	}
+	if apps[2].Issues[0].Context != nil {
+		t.Fatalf("second-lowest issue kept its context (still over budget after one drop)")
+	}
+	if len(apps[1].Issues[0].Context) == 0 {
+		t.Fatalf("highest-count issue LOST its context — wrong drop order")
+	}
+}
+
+func TestEnforceContextBudget_UnderBudgetUntouched(t *testing.T) {
+	apps := []AppTelemetry{appWithIssue("a", 1, 100), appWithIssue("b", 2, 100)}
+	enforceContextBudget(apps, 16*1024)
+	if len(apps[0].Issues[0].Context) == 0 || len(apps[1].Issues[0].Context) == 0 {
+		t.Fatalf("under-budget contexts must be untouched")
+	}
+}

controller/internal/report/types.go

@@ -34,6 +34,10 @@ type Report struct {
 	// alerts). Absent when no offbox target is enabled (and on pre-v0.109 controllers — the checker is
 	// nil-safe on both).
 	Offsite *backup.OffboxReportStatus `json:"offsite,omitempty"`
+
+	// LogTails (v0.111.0) — on-demand ordered log tails, present only on the report cycle
+	// right after the ACK requested them (log_tail_requests). Redacted + capped (logtail.go).
+	LogTails []LogTail `json:"log_tails,omitempty"`
 }
 
 // SystemReport holds host-level system info.