From 19f2c908fcf0e57d4a69cc1ad4091bf14945b45a Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 25 Feb 2026 16:01:53 +0100 Subject: [PATCH] =?UTF-8?q?telemetry:=20fix=20log=20deduplication=20?= =?UTF-8?q?=E2=80=94=20strip=20ANSI=20codes,=20tz=20offsets,=20mid-line=20?= =?UTF-8?q?timestamps=20(v0.30.6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- CHANGELOG.md | 10 ++++++++ controller/README.md | 2 +- controller/internal/metrics/logscanner.go | 28 ++++++++++++++--------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d221cbe..79b2db2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ ## Changelog +### v0.30.6 — Telemetry: Better Log Deduplication (2026-02-25) + +#### Fixed +- **ANSI escape code stripping** — Log scanner now strips ANSI color codes (e.g. `\x1b[35m`) before classifying and fingerprinting lines, preventing color codes from polluting error messages and breaking deduplication +- **Timezone offset in timestamps** — ISO timestamp regex now handles `+01:00`/`-0500` timezone offsets and optional trailing colons (fixes Vikunja-style log entries) +- **Mid-line timestamps** — Removed `^` anchor from both ISO and syslog timestamp regexes, so timestamps embedded after log-level keywords (e.g. `ERROR 2026-02-24T21:27:05`) are now stripped correctly + +#### Improved +- **`cleanLine()` helper** — Consolidated ANSI + timestamp stripping into a single reusable function used by both message display and fingerprint deduplication + ### v0.30.5 — Health Probe: Fast Initial Checking (2026-02-25) #### Improved diff --git a/controller/README.md b/controller/README.md index 832da1d..9db58cb 100644 --- a/controller/README.md +++ b/controller/README.md @@ -892,7 +892,7 @@ Each report push now includes per-app telemetry data: **Log scanning** (`logscanner.go`): - `ScanContainerLogs(containerNames, since, logger)` runs `docker logs --since=15m --tail=1000` sequentially on all non-protected deployed containers. - Classifies lines by keyword match (errors: `error`, `fatal`, `panic`, `crit`, `oom`, `killed`, `exception`, `traceback`; warnings: `warn`, `warning`) on the first 5 words (case-insensitive). -- Deduplicates via fingerprinting: strips timestamps, replaces 6+ digit numbers with ``, 8+ char hex with ``, UUIDs with ``. Groups identical fingerprints, keeps top 10 per container. +- Deduplicates via fingerprinting: strips ANSI escape codes, ISO timestamps (with timezone offsets), and syslog timestamps (including mid-line); replaces 6+ digit numbers with ``, 8+ char hex with ``, UUIDs with ``. Groups identical fingerprints, keeps top 10 per container. - Returns `[]ContainerLogSummary` with `ErrorCount`, `WarnCount`, `RecentIssues []LogIssue`. **Report integration** (`report/telemetry.go`): diff --git a/controller/internal/metrics/logscanner.go b/controller/internal/metrics/logscanner.go index 6976ec8..2512136 100644 --- a/controller/internal/metrics/logscanner.go +++ b/controller/internal/metrics/logscanner.go @@ -29,10 +29,12 @@ type LogIssue struct { } var ( - // Strip leading ISO timestamp: 2006-01-02T15:04:05 or 2006/01/02 15:04:05 etc. - reTimestamp = regexp.MustCompile(`^\d{4}[-/]\d{2}[-/]\d{2}[T ]\d{2}:\d{2}:\d{2}[.\d]*[Z ]?`) + // Strip ANSI escape codes (color, bold, etc.) + reANSI = regexp.MustCompile(`\x1b\[[0-9;]*m`) + // Strip ISO timestamp: 2006-01-02T15:04:05 or 2006/01/02 15:04:05, with optional tz offset + reTimestamp = regexp.MustCompile(`\d{4}[-/]\d{2}[-/]\d{2}[T ]\d{2}:\d{2}:\d{2}[.\d]*([+-]\d{2}:?\d{2})?[Z ]?:? ?`) // Strip syslog-style timestamp: Jan 2 15:04:05 - reSyslog = regexp.MustCompile(`^[A-Z][a-z]{2}\s+\d{1,2} \d{2}:\d{2}:\d{2} `) + reSyslog = regexp.MustCompile(`[A-Z][a-z]{2}\s+\d{1,2} \d{2}:\d{2}:\d{2} `) // Replace 6+ digit sequences with (avoids mangling 4-digit HTTP codes/ports) reNumbers = regexp.MustCompile(`\b\d{6,}\b`) // Replace 8+ char hex strings @@ -121,10 +123,7 @@ func scanOneContainer(name string, since time.Duration, logger *log.Logger) Cont e.count++ e.lastSeen = time.Now() } else { - // Use original line trimmed as message (strip timestamp) - msg := reTimestamp.ReplaceAllString(line, "") - msg = reSyslog.ReplaceAllString(msg, "") - msg = strings.TrimSpace(msg) + msg := cleanLine(line) if len(msg) > 200 { msg = msg[:200] } @@ -161,9 +160,18 @@ func scanOneContainer(name string, since time.Duration, logger *log.Logger) Cont return summary } +// cleanLine strips ANSI escape codes and timestamps from a log line. +func cleanLine(line string) string { + s := reANSI.ReplaceAllString(line, "") + s = reTimestamp.ReplaceAllString(s, "") + s = reSyslog.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} + // classifyLine returns "error", "warn", or "" based on first 5 words of the line. func classifyLine(line string) string { - lower := strings.ToLower(line) + cleaned := reANSI.ReplaceAllString(line, "") + lower := strings.ToLower(cleaned) words := strings.Fields(lower) if len(words) > 5 { words = words[:5] @@ -185,9 +193,7 @@ func classifyLine(line string) string { // fingerprint produces a deduplication key for a log line. func fingerprint(line string) string { - // Strip leading timestamp - s := reTimestamp.ReplaceAllString(line, "") - s = reSyslog.ReplaceAllString(s, "") + s := cleanLine(line) // Replace UUIDs before hex to avoid partial matches s = reUUID.ReplaceAllString(s, "") s = reHex.ReplaceAllString(s, "")