v0.111.0: remote app-log diagnostics — error context capture (±5 lines, capped+redacted) + on-demand log tails via ACK pull pattern

- metrics: LogIssue.Context on first-occurrence errors (≤11 lines, ≤400 chars/line, warns carry none); RedactLine sanitizer (password/token/api-key/bearer/64-hex) applied to everything shipped; FetchContainerLogTail
- report: 16KB per-report context budget (lowest-count issues dropped first); log_tail_requests ACK flag → next report ships log_tails (200 lines, ≤64KB/app head-truncated, ordered, redacted); consume-once drain
- tests: synthetic-window context capture, caps, redaction, budget order, consume-once, fetch-error skip

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
2026-07-10 15:43:29 +02:00
parent 332725a024
commit 544c42a618
12 changed files with 670 additions and 8 deletions
+4
View File
@@ -443,6 +443,10 @@ func main() {
// SLICE 3: run the escrow auto-confirm on the same ACK (after the config refresh decision — // SLICE 3: run the escrow auto-confirm on the same ACK (after the config refresh decision —
// a refresh-restart re-enters here anyway on the next cycle). // a refresh-restart re-enters here anyway on the next cycle).
escrowConfirmer.Reconcile(resp.Escrow) escrowConfirmer.Reconcile(resp.Escrow)
// v0.111.0: operator log-tail requests ride the same ACK (pull pattern — the hub
// never reaches in). The NEXT report cycle collects + ships the tails; the hub
// clears its pending request on receipt. An empty list clears any stale local set.
report.SetPendingLogTails(resp.LogTailRequests)
} }
// Wire hub push status into alert manager for dashboard alerts // Wire hub push status into alert manager for dashboard alerts
alertMgr.SetHubPushStatus(func() web.HubPushStatusData { alertMgr.SetHubPushStatus(func() web.HubPushStatusData {
+83 -8
View File
@@ -26,8 +26,18 @@ type LogIssue struct {
Message string `json:"message"` Message string `json:"message"`
Count int `json:"count"` Count int `json:"count"`
LastSeen time.Time `json:"last_seen"` LastSeen time.Time `json:"last_seen"`
// Context is up to ±5 raw log lines around the FIRST occurrence of an error-severity
// issue in this scrape window (≤11 lines, each ≤400 chars, redacted). Warn-severity
// issues never carry context — bounds the payload. Additive since v0.111.0.
Context []string `json:"context,omitempty"`
} }
const (
// contextRadius lines before + the error line + contextRadius after = ≤11 lines.
contextRadius = 5
contextMaxLineChars = 400
)
var ( var (
// Strip ANSI escape codes (color, bold, etc.) // Strip ANSI escape codes (color, bold, etc.)
reANSI = regexp.MustCompile(`\x1b\[[0-9;]*m`) reANSI = regexp.MustCompile(`\x1b\[[0-9;]*m`)
@@ -97,17 +107,28 @@ func scanOneContainer(name string, since time.Duration, logger *log.Logger) Cont
return summary return summary
} }
lines := strings.Split(string(output), "\n")
summary.ErrorCount, summary.WarnCount, summary.RecentIssues = analyzeLogLines(lines)
return summary
}
// analyzeLogLines classifies and deduplicates a scan window's raw log lines.
// Pure (no docker, no I/O) so it is unit-testable with synthetic windows.
// Error-severity issues get a ±5-line context captured on their FIRST occurrence
// only (never on repeats); warns carry no context.
func analyzeLogLines(lines []string) (errorCount, warnCount int, issues []LogIssue) {
// fingerprint → issue tracking // fingerprint → issue tracking
type issueEntry struct { type issueEntry struct {
severity string severity string
message string message string
count int count int
lastSeen time.Time lastSeen time.Time
context []string
} }
fingerprints := make(map[string]*issueEntry) fingerprints := make(map[string]*issueEntry)
lines := strings.Split(string(output), "\n") for i, line := range lines {
for _, line := range lines {
if !utf8.Valid([]byte(line)) { if !utf8.Valid([]byte(line)) {
continue continue
} }
@@ -124,9 +145,9 @@ func scanOneContainer(name string, since time.Duration, logger *log.Logger) Cont
} }
if severity == "error" { if severity == "error" {
summary.ErrorCount++ errorCount++
} else { } else {
summary.WarnCount++ warnCount++
} }
fp := fingerprint(line) fp := fingerprint(line)
@@ -138,23 +159,28 @@ func scanOneContainer(name string, since time.Duration, logger *log.Logger) Cont
if len(msg) > 200 { if len(msg) > 200 {
msg = msg[:200] msg = msg[:200]
} }
fingerprints[fp] = &issueEntry{ entry := &issueEntry{
severity: severity, severity: severity,
message: msg, message: msg,
count: 1, count: 1,
lastSeen: time.Now(), lastSeen: time.Now(),
} }
if severity == "error" {
entry.context = captureContext(lines, i)
}
fingerprints[fp] = entry
} }
} }
// Convert map to slice, sort by count DESC then lastSeen DESC, cap at 10 // Convert map to slice, sort by count DESC then lastSeen DESC, cap at 10
issues := make([]LogIssue, 0, len(fingerprints)) issues = make([]LogIssue, 0, len(fingerprints))
for _, e := range fingerprints { for _, e := range fingerprints {
issues = append(issues, LogIssue{ issues = append(issues, LogIssue{
Severity: e.severity, Severity: e.severity,
Message: e.message, Message: e.message,
Count: e.count, Count: e.count,
LastSeen: e.lastSeen, LastSeen: e.lastSeen,
Context: e.context,
}) })
} }
sort.Slice(issues, func(i, j int) bool { sort.Slice(issues, func(i, j int) bool {
@@ -166,9 +192,58 @@ func scanOneContainer(name string, since time.Duration, logger *log.Logger) Cont
if len(issues) > 10 { if len(issues) > 10 {
issues = issues[:10] issues = issues[:10]
} }
summary.RecentIssues = issues return errorCount, warnCount, issues
}
return summary // captureContext returns the ±contextRadius window around lines[i], ordered as emitted.
// Empty and invalid-UTF8 slots are dropped; each kept line is ANSI-stripped, truncated
// to contextMaxLineChars with "…", and redacted (RedactLine) — nothing raw ships.
func captureContext(lines []string, i int) []string {
lo := i - contextRadius
if lo < 0 {
lo = 0
}
hi := i + contextRadius
if hi > len(lines)-1 {
hi = len(lines) - 1
}
out := make([]string, 0, hi-lo+1)
for j := lo; j <= hi; j++ {
l := lines[j]
if l == "" || !utf8.Valid([]byte(l)) {
continue
}
l = reANSI.ReplaceAllString(l, "")
l = truncateRunes(l, contextMaxLineChars)
out = append(out, RedactLine(l))
}
return out
}
// truncateRunes rune-safely truncates s to max runes, appending "…" when cut.
func truncateRunes(s string, max int) string {
runes := []rune(s)
if len(runes) <= max {
return s
}
return string(runes[:max]) + "…"
}
// FetchContainerLogTail returns the last tailLines lines of one container's docker log,
// ordered as emitted (stdout+stderr merged). Same CLI plumbing as the scanner; used by
// the report's on-demand log-tail (operator request riding the ACK). Caller caps/redacts.
func FetchContainerLogTail(name string, tailLines int) (string, error) {
if tailLines <= 0 || tailLines > 1000 {
tailLines = 200
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "docker", "logs", fmt.Sprintf("--tail=%d", tailLines), name)
output, err := cmd.CombinedOutput()
if err != nil {
return "", fmt.Errorf("docker logs %s: %w", name, err)
}
return string(output), nil
} }
// cleanLine strips ANSI escape codes and timestamps from a log line. // cleanLine strips ANSI escape codes and timestamps from a log line.
@@ -0,0 +1,159 @@
package metrics
import (
"fmt"
"strings"
"testing"
"unicode/utf8"
)
func findIssue(t *testing.T, issues []LogIssue, substr string) LogIssue {
t.Helper()
for _, is := range issues {
if strings.Contains(is.Message, substr) {
return is
}
}
t.Fatalf("no issue containing %q in %+v", substr, issues)
return LogIssue{}
}
// Part B — a synthetic scrape with an error mid-stream: Context must be EXACTLY the
// ±5 window, ordered as emitted. Red-proof: dropping the captureContext call leaves
// Context empty → this fails.
func TestAnalyzeLogLines_ErrorContextWindow(t *testing.T) {
lines := []string{
"boot line zero",
"ctx minus five",
"ctx minus four",
"ctx minus three",
"ctx minus two",
"ctx minus one",
"ERROR: nfs mount lost", // index 6
"ctx plus one",
"ctx plus two",
"ctx plus three",
"ctx plus four",
"ctx plus five",
"outside the window",
}
errC, warnC, issues := analyzeLogLines(lines)
if errC != 1 || warnC != 0 {
t.Fatalf("counts: errors=%d warns=%d, want 1/0", errC, warnC)
}
is := findIssue(t, issues, "nfs mount lost")
want := []string{
"ctx minus five", "ctx minus four", "ctx minus three", "ctx minus two", "ctx minus one",
"ERROR: nfs mount lost",
"ctx plus one", "ctx plus two", "ctx plus three", "ctx plus four", "ctx plus five",
}
if len(is.Context) != len(want) {
t.Fatalf("context has %d lines, want %d: %#v", len(is.Context), len(want), is.Context)
}
for i := range want {
if is.Context[i] != want[i] {
t.Fatalf("context[%d] = %q, want %q (order must match emission)", i, is.Context[i], want[i])
}
}
}
// Repeat occurrence of the same fingerprint in one window: count aggregates but the
// context stays the FIRST occurrence's window — never re-captured on repeats.
func TestAnalyzeLogLines_RepeatKeepsFirstContext(t *testing.T) {
lines := []string{
"first neighborhood",
"ERROR: db connection refused",
"after the first",
"filler a", "filler b", "filler c", "filler d", "filler e", "filler f",
"second neighborhood",
"ERROR: db connection refused",
"after the second",
}
_, _, issues := analyzeLogLines(lines)
is := findIssue(t, issues, "db connection refused")
if is.Count != 2 {
t.Fatalf("count = %d, want 2 (same fingerprint must dedupe)", is.Count)
}
joined := strings.Join(is.Context, "\n")
if !strings.Contains(joined, "first neighborhood") {
t.Fatalf("context lost the FIRST occurrence window: %#v", is.Context)
}
if strings.Contains(joined, "second neighborhood") {
t.Fatalf("context re-captured on a repeat occurrence: %#v", is.Context)
}
}
// Warn-severity issues carry NO context (message-only) — bounds the payload.
func TestAnalyzeLogLines_WarnHasNoContext(t *testing.T) {
lines := []string{"before", "WARN: disk latency high", "after"}
_, warnC, issues := analyzeLogLines(lines)
if warnC != 1 {
t.Fatalf("warns = %d, want 1", warnC)
}
is := findIssue(t, issues, "disk latency high")
if len(is.Context) != 0 {
t.Fatalf("warn issue must not carry context, got %#v", is.Context)
}
}
// Caps: oversized context lines are truncated to 400 runes + "…" (rune-safe).
func TestAnalyzeLogLines_ContextLineTruncated(t *testing.T) {
long := strings.Repeat("x", 450)
lines := []string{long, "ERROR: it broke", "after"}
_, _, issues := analyzeLogLines(lines)
is := findIssue(t, issues, "it broke")
if len(is.Context) == 0 {
t.Fatalf("expected context")
}
got := is.Context[0]
if utf8.RuneCountInString(got) != 401 || !strings.HasSuffix(got, "…") {
t.Fatalf("oversized line not truncated to 400+ellipsis: len=%d suffix=%q",
utf8.RuneCountInString(got), got[len(got)-3:])
}
}
// Part E on the capture path: a secret in a NEIGHBOR line must ship redacted.
func TestAnalyzeLogLines_ContextRedacted(t *testing.T) {
lines := []string{
"connecting with password=hunter2 now",
"ERROR: auth failed",
"retrying",
}
_, _, issues := analyzeLogLines(lines)
is := findIssue(t, issues, "auth failed")
joined := strings.Join(is.Context, "\n")
if strings.Contains(joined, "hunter2") {
t.Fatalf("secret shipped in context: %#v", is.Context)
}
if !strings.Contains(joined, "password=[REDACTED]") {
t.Fatalf("expected redaction marker in context: %#v", is.Context)
}
}
// Error at the very start/end of the window: the ±5 clamps without panicking
// and still contains the error line itself.
func TestAnalyzeLogLines_WindowClamped(t *testing.T) {
lines := []string{"ERROR: first line broke", "after one"}
_, _, issues := analyzeLogLines(lines)
is := findIssue(t, issues, "first line broke")
if len(is.Context) != 2 {
t.Fatalf("clamped context = %#v, want the 2 available lines", is.Context)
}
}
// The 10-issue cap and count-DESC ordering still hold with contexts attached.
func TestAnalyzeLogLines_CapAndOrder(t *testing.T) {
var lines []string
for i := 0; i < 12; i++ {
lines = append(lines, fmt.Sprintf("ERROR: distinct failure mode %s", strings.Repeat("z", i+1)))
}
// make one of them dominant
lines = append(lines, "ERROR: distinct failure mode z", "ERROR: distinct failure mode z")
_, _, issues := analyzeLogLines(lines)
if len(issues) != 10 {
t.Fatalf("issues = %d, want capped at 10", len(issues))
}
if issues[0].Count != 3 {
t.Fatalf("top issue count = %d, want the dominant one (3) first", issues[0].Count)
}
}
+22
View File
@@ -0,0 +1,22 @@
package metrics
import "regexp"
// Redaction is controller-side and authoritative: every context line and log-tail line
// must pass through RedactLine before it leaves the box (rides the hub report).
var (
// key[=: ]value shapes — the value is masked. The optional "bearer " prefix inside the
// value group catches "Authorization: Bearer <token>" in one pass.
reSecretKV = regexp.MustCompile(`(?i)\b(password|passwd|secret|token|api[_-]?key|authorization|bearer)([=: ]\s*)((?:bearer\s+)?\S+)`)
// 64-hex string = restic repo password / key-material shape.
reHex64 = regexp.MustCompile(`\b[0-9a-fA-F]{64}\b`)
)
// RedactLine masks secret-shaped values in a log line. Applied to context lines and
// log-tail lines before shipping; deliberately narrow (support usefulness over
// aggression) but the named patterns are non-negotiable.
func RedactLine(s string) string {
s = reSecretKV.ReplaceAllString(s, "${1}${2}[REDACTED]")
s = reHex64.ReplaceAllString(s, "[REDACTED-HEX64]")
return s
}
@@ -0,0 +1,66 @@
package metrics
import (
"strings"
"testing"
)
// Part E — sanitization. Red-proof: gutting RedactLine (return s unchanged) must fail
// every case here with the secret visible in the failure message.
func TestRedactLine_SpecPatterns(t *testing.T) {
in := "password=hunter2 token: abc Bearer xyz"
got := RedactLine(in)
want := "password=[REDACTED] token: [REDACTED] Bearer [REDACTED]"
if got != want {
t.Fatalf("RedactLine(%q) = %q, want %q", in, got, want)
}
for _, secret := range []string{"hunter2", "abc", "xyz"} {
if strings.Contains(got, secret) {
t.Fatalf("secret %q shipped: %q", secret, got)
}
}
}
func TestRedactLine_Hex64(t *testing.T) {
hex64 := strings.Repeat("ab12", 16) // 64 hex chars — restic repo password shape
in := "repo unlock with " + hex64 + " done"
got := RedactLine(in)
if strings.Contains(got, hex64) {
t.Fatalf("64-hex secret shipped: %q", got)
}
if !strings.Contains(got, "[REDACTED-HEX64]") {
t.Fatalf("expected [REDACTED-HEX64] marker, got %q", got)
}
}
func TestRedactLine_Variants(t *testing.T) {
cases := []struct{ in, want string }{
{"authorization: Bearer eyJhbGciOi.payload.sig", "authorization: [REDACTED]"},
{"api_key=sk-live-123", "api_key=[REDACTED]"},
{"API-KEY: verysecret", "API-KEY: [REDACTED]"},
{"apikey=whatever", "apikey=[REDACTED]"},
{"PASSWD: root123", "PASSWD: [REDACTED]"},
{"client secret=s3cr3t", "client secret=[REDACTED]"},
}
for _, c := range cases {
if got := RedactLine(c.in); got != c.want {
t.Errorf("RedactLine(%q) = %q, want %q", c.in, got, c.want)
}
}
}
// Benign lines must pass through byte-identical — support usefulness over aggression.
func TestRedactLine_BenignUntouched(t *testing.T) {
cases := []string{
"connection refused to 10.0.0.5:5432",
"GET /api/keys 200 12ms", // "api/keys" is not "api_key"
"tokenizer initialized in 40ms", // "token" not followed by separator+value
"ERROR: NFS mount /mnt/media gone", // the CWA shape — must stay readable
"deadbeef", // short hex, not 64
}
for _, c := range cases {
if got := RedactLine(c); got != c {
t.Errorf("benign line mangled: %q -> %q", c, got)
}
}
}
+13
View File
@@ -155,6 +155,19 @@ func BuildReport(
// App telemetry (metrics + log scan) // App telemetry (metrics + log scan)
r.AppTelemetry = buildAppTelemetrySection(stackMgr, metricsStore, logger) r.AppTelemetry = buildAppTelemetrySection(stackMgr, metricsStore, logger)
// On-demand log tails (v0.111.0): drain the ACK-requested apps and collect their tails.
// Stacks go through the existing compose-logs plumbing; the controller's own container
// through the scanner's docker-logs plumbing. Consume-once: drained here, re-armed by
// the next ACK if the push fails.
if tailApps := drainPendingLogTails(); len(tailApps) > 0 {
r.LogTails = buildLogTailsSection(tailApps, func(app string, lines int) (string, error) {
if app == controllerContainerName {
return metrics.FetchContainerLogTail(app, lines)
}
return stackMgr.GetLogs(app, lines)
}, logger)
}
// Geo-restriction status — ALWAYS present (even when never configured) so the hub // Geo-restriction status — ALWAYS present (even when never configured) so the hub
// always renders the section. A nil pointer (omitempty) made the hub hide the whole // always renders the section. A nil pointer (omitempty) made the hub hide the whole
// section for a never-configured controller; a present-but-disabled report renders // section for a never-configured controller; a present-but-disabled report renders
+109
View File
@@ -0,0 +1,109 @@
package report
import (
"log"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
)
// On-demand log tail (pull-based, same ACK-flag pattern as escrow/config-refresh):
// the hub stores a pending request per (customer, app); the report ACK advertises the
// requested app names in log_tail_requests; the NEXT report carries the collected tails
// in log_tails. The hub clears the pending request when a tail arrives (consume-once) —
// the controller listens to no one, it only pushes on its own cycle.
const (
logTailLines = 200
logTailMaxLineChars = 400
logTailMaxBytes = 64 * 1024 // per app; head-truncated (newest lines kept)
)
// LogTail is one app's on-demand ordered log tail riding the report.
type LogTail struct {
App string `json:"app"`
CollectedAt time.Time `json:"collected_at"`
Lines []string `json:"lines"`
}
var (
pendingTailsMu sync.Mutex
pendingTails []string
)
// SetPendingLogTails records the hub-requested apps from the latest ACK. The hub is
// the source of truth: an ACK without requests clears any stale local pending set.
func SetPendingLogTails(apps []string) {
pendingTailsMu.Lock()
defer pendingTailsMu.Unlock()
pendingTails = append([]string(nil), apps...)
}
// drainPendingLogTails takes and CLEARS the pending set (consume-once controller-side).
// If the subsequent push fails, the hub's request is still pending and the next ACK
// re-arms it — fail-safe retry, no duplicate shipping.
func drainPendingLogTails() []string {
pendingTailsMu.Lock()
defer pendingTailsMu.Unlock()
apps := pendingTails
pendingTails = nil
return apps
}
// buildLogTailsSection collects the requested tails via fetch (seam: stacks.GetLogs or
// metrics.FetchContainerLogTail). Lines stay ordered as emitted; each line is truncated
// to 400 chars and redacted; per-app total is capped at 64KB keeping the NEWEST lines.
// A failed fetch is skipped (logged) — the hub request stays pending and retries next cycle.
func buildLogTailsSection(apps []string, fetch func(app string, lines int) (string, error), logger *log.Logger) []LogTail {
var tails []LogTail
for _, app := range apps {
raw, err := fetch(app, logTailLines)
if err != nil {
if logger != nil {
logger.Printf("[WARN] [report] log-tail fetch for %s failed: %v", app, err)
}
continue
}
lines := capTailLines(strings.Split(raw, "\n"))
tails = append(tails, LogTail{App: app, CollectedAt: time.Now().UTC(), Lines: lines})
if logger != nil {
logger.Printf("[INFO] [report] log-tail collected for %s: %d lines", app, len(lines))
}
}
return tails
}
// capTailLines truncates each line to logTailMaxLineChars, redacts it, and enforces the
// per-app byte budget by dropping from the HEAD (oldest) so the newest lines survive.
func capTailLines(lines []string) []string {
// Drop a trailing empty split artifact.
for len(lines) > 0 && lines[len(lines)-1] == "" {
lines = lines[:len(lines)-1]
}
out := make([]string, 0, len(lines))
for _, l := range lines {
l = metrics.RedactLine(truncateTailLine(l, logTailMaxLineChars))
out = append(out, l)
}
// Head-truncate to the byte budget: walk from the end accumulating.
total := 0
start := len(out)
for i := len(out) - 1; i >= 0; i-- {
total += len(out[i]) + 1 // +1 for the newline it represents
if total > logTailMaxBytes {
break
}
start = i
}
return out[start:]
}
func truncateTailLine(s string, max int) string {
runes := []rune(s)
if len(runes) <= max {
return s
}
return string(runes[:max]) + "…"
}
+115
View File
@@ -0,0 +1,115 @@
package report
import (
"errors"
"fmt"
"strings"
"testing"
"unicode/utf8"
)
// Part D — the consume-once seam: an ACK arms the pending set, the FIRST build drains
// it, the FOLLOWING build gets nothing. Red-proof: removing the clear in
// drainPendingLogTails makes the second drain return the apps again → this fails.
func TestPendingLogTails_ConsumeOnce(t *testing.T) {
SetPendingLogTails([]string{"gokapi", "cwa"})
first := drainPendingLogTails()
if len(first) != 2 || first[0] != "gokapi" || first[1] != "cwa" {
t.Fatalf("first drain = %v, want the two requested apps", first)
}
second := drainPendingLogTails()
if len(second) != 0 {
t.Fatalf("second drain = %v — tails would ship on EVERY report (consume-once broken)", second)
}
}
// An ACK without requests clears any stale local set (hub is the source of truth).
func TestPendingLogTails_EmptyAckClears(t *testing.T) {
SetPendingLogTails([]string{"gokapi"})
SetPendingLogTails(nil)
if got := drainPendingLogTails(); len(got) != 0 {
t.Fatalf("stale pending set survived an empty ACK: %v", got)
}
}
func TestBuildLogTailsSection_OrderedAndRedacted(t *testing.T) {
fetch := func(app string, lines int) (string, error) {
if lines != 200 {
t.Fatalf("fetch asked for %d lines, want 200", lines)
}
return "line one\nline two with password=hunter2\nline three\n", nil
}
tails := buildLogTailsSection([]string{"gokapi"}, fetch, nil)
if len(tails) != 1 || tails[0].App != "gokapi" {
t.Fatalf("tails = %+v", tails)
}
got := tails[0].Lines
if len(got) != 3 {
t.Fatalf("lines = %d (trailing empty must be dropped): %#v", len(got), got)
}
if got[0] != "line one" || got[2] != "line three" {
t.Fatalf("order broken: %#v", got)
}
if strings.Contains(got[1], "hunter2") || !strings.Contains(got[1], "[REDACTED]") {
t.Fatalf("tail line not redacted: %q", got[1])
}
if tails[0].CollectedAt.IsZero() {
t.Fatalf("collected_at not stamped")
}
}
func TestBuildLogTailsSection_LineLengthCap(t *testing.T) {
fetch := func(string, int) (string, error) {
return strings.Repeat("y", 900) + "\nshort", nil
}
tails := buildLogTailsSection([]string{"a"}, fetch, nil)
first := tails[0].Lines[0]
if utf8.RuneCountInString(first) != 401 || !strings.HasSuffix(first, "…") {
t.Fatalf("line not capped at 400+ellipsis: runes=%d", utf8.RuneCountInString(first))
}
}
// 64KB per-app budget: HEAD-truncate — the newest lines must survive.
func TestBuildLogTailsSection_ByteBudgetKeepsNewest(t *testing.T) {
var sb strings.Builder
for i := 0; i < 300; i++ {
sb.WriteString(fmt.Sprintf("%04d ", i))
sb.WriteString(strings.Repeat("p", 295))
sb.WriteString("\n")
}
fetch := func(string, int) (string, error) { return sb.String(), nil }
tails := buildLogTailsSection([]string{"a"}, fetch, nil)
lines := tails[0].Lines
if len(lines) >= 300 {
t.Fatalf("budget not enforced: %d lines kept", len(lines))
}
total := 0
for _, l := range lines {
total += len(l) + 1
}
if total > logTailMaxBytes {
t.Fatalf("kept %d bytes > %d budget", total, logTailMaxBytes)
}
if !strings.HasPrefix(lines[len(lines)-1], "0299") {
t.Fatalf("NEWEST line lost — head-truncation inverted; last kept = %q", lines[len(lines)-1][:5])
}
if strings.HasPrefix(lines[0], "0000") {
t.Fatalf("oldest line survived a budget overflow — nothing was dropped")
}
}
// A failed fetch skips that app but the others still ship (hub retries the failed one
// next cycle since its pending request was never fulfilled).
func TestBuildLogTailsSection_FetchErrorSkips(t *testing.T) {
fetch := func(app string, _ int) (string, error) {
if app == "broken" {
return "", errors.New("no such stack")
}
return "ok line\n", nil
}
tails := buildLogTailsSection([]string{"broken", "healthy"}, fetch, nil)
if len(tails) != 1 || tails[0].App != "healthy" {
t.Fatalf("tails = %+v, want only the healthy app", tails)
}
}
+4
View File
@@ -40,6 +40,10 @@ type PushResponse struct {
// Escrow (SLICE 3) is the hub's escrow status for this customer — the input to the hub-verified // Escrow (SLICE 3) is the hub's escrow status for this customer — the input to the hub-verified
// auto-confirm (EscrowAutoConfirmer). nil = no escrow row on the hub (or an old hub) → stays pending. // auto-confirm (EscrowAutoConfirmer). nil = no escrow row on the hub (or an old hub) → stays pending.
Escrow *EscrowStatus `json:"escrow"` Escrow *EscrowStatus `json:"escrow"`
// LogTailRequests (v0.111.0) — app names the operator wants a log tail for. Same
// pull-based ACK-flag pattern as escrow: the NEXT report ships the tails; the hub
// clears the pending request on receipt (consume-once). Absent/empty = nothing pending.
LogTailRequests []string `json:"log_tail_requests"`
} }
// Pusher sends reports to the central hub. // Pusher sends reports to the central hub.
+43
View File
@@ -59,9 +59,52 @@ func buildAppTelemetrySection(stackMgr *stacks.Manager, metricsStore *metrics.Me
if result == nil { if result == nil {
result = []AppTelemetry{} result = []AppTelemetry{}
} }
// 7. Enforce the per-report context budget (v0.111.0): drop context from the
// lowest-count issues first until under 16KB. Warns never carry context (scanner).
enforceContextBudget(result, contextBudgetBytes)
return result return result
} }
// contextBudgetBytes is the HARD per-report total for all LogIssue.Context lines.
const contextBudgetBytes = 16 * 1024
// enforceContextBudget drops Context from the lowest-count issues (across all apps)
// until the summed context bytes fit the budget. Mutates apps in place.
func enforceContextBudget(apps []AppTelemetry, budget int) {
type ref struct {
app, issue, size, count int
}
var refs []ref
total := 0
for ai := range apps {
for ii := range apps[ai].Issues {
ctx := apps[ai].Issues[ii].Context
if len(ctx) == 0 {
continue
}
sz := 0
for _, l := range ctx {
sz += len(l)
}
refs = append(refs, ref{app: ai, issue: ii, size: sz, count: apps[ai].Issues[ii].Count})
total += sz
}
}
if total <= budget {
return
}
sort.Slice(refs, func(i, j int) bool { return refs[i].count < refs[j].count })
for _, r := range refs {
if total <= budget {
break
}
apps[r.app].Issues[r.issue].Context = nil
total -= r.size
}
}
// buildAppTelemetry aggregates container-level telemetry and log data into per-stack AppTelemetry entries. // buildAppTelemetry aggregates container-level telemetry and log data into per-stack AppTelemetry entries.
func buildAppTelemetry(allStacks []stacks.Stack, telemetry []metrics.ContainerTelemetry, logs []metrics.ContainerLogSummary) []AppTelemetry { func buildAppTelemetry(allStacks []stacks.Stack, telemetry []metrics.ContainerTelemetry, logs []metrics.ContainerLogSummary) []AppTelemetry {
// Build lookup maps // Build lookup maps
@@ -0,0 +1,48 @@
package report
import (
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/metrics"
)
func appWithIssue(name string, count, ctxBytes int) AppTelemetry {
var ctx []string
if ctxBytes > 0 {
ctx = []string{strings.Repeat("c", ctxBytes)}
}
return AppTelemetry{
AppName: name,
Issues: []metrics.LogIssue{{Severity: "error", Message: name + " boom", Count: count, Context: ctx}},
}
}
// Part B caps — budget overflow drops context from the LOWEST-count issues first,
// across apps. Red-proof: inverting the sort (highest first) keeps the wrong survivor.
func TestEnforceContextBudget_DropsLowestCountFirst(t *testing.T) {
apps := []AppTelemetry{
appWithIssue("rare", 2, 600), // lowest count — first to lose context
appWithIssue("frequent", 50, 600), // must keep context
appWithIssue("medium", 10, 600), // dropped second
}
enforceContextBudget(apps, 1000) // total 1800 → must shed 800 → drop rare, then medium
if apps[0].Issues[0].Context != nil {
t.Fatalf("lowest-count issue kept its context")
}
if apps[2].Issues[0].Context != nil {
t.Fatalf("second-lowest issue kept its context (still over budget after one drop)")
}
if len(apps[1].Issues[0].Context) == 0 {
t.Fatalf("highest-count issue LOST its context — wrong drop order")
}
}
func TestEnforceContextBudget_UnderBudgetUntouched(t *testing.T) {
apps := []AppTelemetry{appWithIssue("a", 1, 100), appWithIssue("b", 2, 100)}
enforceContextBudget(apps, 16*1024)
if len(apps[0].Issues[0].Context) == 0 || len(apps[1].Issues[0].Context) == 0 {
t.Fatalf("under-budget contexts must be untouched")
}
}
+4
View File
@@ -34,6 +34,10 @@ type Report struct {
// alerts). Absent when no offbox target is enabled (and on pre-v0.109 controllers — the checker is // alerts). Absent when no offbox target is enabled (and on pre-v0.109 controllers — the checker is
// nil-safe on both). // nil-safe on both).
Offsite *backup.OffboxReportStatus `json:"offsite,omitempty"` Offsite *backup.OffboxReportStatus `json:"offsite,omitempty"`
// LogTails (v0.111.0) — on-demand ordered log tails, present only on the report cycle
// right after the ACK requested them (log_tail_requests). Redacted + capped (logtail.go).
LogTails []LogTail `json:"log_tails,omitempty"`
} }
// SystemReport holds host-level system info. // SystemReport holds host-level system info.