v0.83.0: observability pass — always-DEBUG capture ring + GET /debug/logs + heartbeat log-pull + gap-fill sweep

Capture layer: applog.New returns (logger, Ring) — slog fan-out, stderr at the
configured level, ~1000-entry ring fixed at LevelDebug (remote diagnostics
without a config flip). GET /debug/logs (token-authed, ?raw=1) + request-level
DEBUG middleware. Heartbeat log-pull mirrors the report logtail pattern:
envelope log_tail_requested -> next heartbeat carries log_tail (128KB cap,
consume-once, failed-push retry proven). Gap-fill sweep over netverify/
netstorage/netmount/signedjobs/selfupdate/disks/controller-swap/desired/loop.
Red-proofs: ring-at-emit-level FAILs capture test; drain removed FAILs
consume-once; dropped phase line FAILs the S7 log-sequence smoke.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
2026-07-11 16:24:07 +02:00
parent 461eaf42c1
commit cb692f8788
20 changed files with 902 additions and 29 deletions
+13 -5
View File
@@ -167,11 +167,13 @@ func main() {
}
cfg = config.Default()
}
logger := applog.New(cfg.LogLevel)
// logRing is the always-DEBUG capture ring (v0.83.0 observability): served by the
// local API's GET /debug/logs and the heartbeat log-pull. Selftests ignore it.
logger, logRing := applog.New(cfg.LogLevel)
switch selftest.mode {
case "":
os.Exit(runDaemon(cfg, logger))
os.Exit(runDaemon(cfg, logger, logRing))
case "read":
os.Exit(runSelftestRead(context.Background(), cfg, logger))
case "task":
@@ -369,7 +371,7 @@ func logCapabilities(statuses []capability.Status, logger *slog.Logger) {
// runDaemon is the default mode: collect a host-report and POST it to the hub on a
// loop. Requires both proxmox (to collect) and hub config.
func runDaemon(cfg config.Config, logger *slog.Logger) int {
func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int {
if err := cfg.Validate(); err != nil {
fmt.Fprintln(os.Stderr, "daemon: proxmox not configured:", err)
return 2
@@ -427,6 +429,11 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
collector.SetCapabilityProber(probeAll)
loop := hub.NewLoop(collector, client, time.Duration(hcfg.PollSeconds)*time.Second, logger)
interval := time.Duration(hcfg.PollSeconds) * time.Second
// Heartbeat log-pull (v0.83.0): when the control envelope requests it, the NEXT
// heartbeat carries the debug ring's tail (newest-kept, byte-capped in the loop).
if logRing != nil {
loop.SetLogTailSource(logRing.Lines)
}
// Desired-state provider (slice 10A): the hub-served target the reconcile engine converges
// toward. Starts EMPTY (generation 0) — reconcile is a live no-op until the hub serves intent,
@@ -659,7 +666,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec}, cfg.Hub.HostID, logger)
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logger, &localTokens)
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, driveKnown, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logRing, logger, &localTokens)
if localTokens != nil {
defer localTokens.Close()
}
@@ -948,7 +955,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
// leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the
// daemon — the host still reports/reconciles; only the controller channel is unavailable until
// fixed. The opened token store is returned via outTokens so the caller can Close it.
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, driveTargets storage.KnownTargets, hostOps *storage.SudoHostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logRing *applog.Ring, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
if !cfg.LocalAPI.Enabled() {
return nil
}
@@ -1019,6 +1026,7 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
// per-storage view to the customer's monitoring page (reuses the slice-4 collector).
HostMetrics: collector,
HostID: cfg.Hub.HostID, // slice 10B: anti-retarget host in the data-bearing-format pending-op
LogRing: logRing, // v0.83.0: GET /debug/logs — the always-DEBUG capture ring
Logger: logger,
})
if err != nil {