diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a21fbd..caeb9c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,26 @@ -## v0.70.0 — agent self-update (operator-signed, A/B slots, crash-loop auto-rollback) (2026-07-05) +## v0.71.0 — management-plane break-glass: privsep-dir watchdog + mgmt_plane health (TASK G1) (2026-07-05) + +Prerequisite for the felhom-sshd OOB feature (H1). Closes the lockout from +`felhom.eu/documentation/audits/SPIKE-felhom-sshd-2026-07-05.md` §8: a second sshd's +`RuntimeDirectory=sshd` removed the SHARED `/run/sshd` privsep dir and took the stock sshd on :22 down +too (sessions reset right after SSH2_MSG_KEXINIT) — a management lockout on a healthy box. Three +independent layers; this repo ships the host artifacts + the agent reporter (hub vault + surfacing are +the felhom.eu half). + +- **Host artifacts** (`configs/`, installed by felhom-host-install): `felhom-privsep.tmpfiles` + (`d /run/sshd 0755 root root -` — layer 1, boot-persistent, owned by no unit's lifecycle); + `felhom-mgmt-watchdog.sh` (layer 2 heal action — stat-first recreate `/run/sshd`, `reset-failed` + ssh ONLY when `failed`, write an RFC3339 heal-marker; NEVER restarts the stock sshd, NEVER touches a + healthy dir — shellcheck-clean); `.service` (oneshot) + `.timer` (~60s, `Persistent`). The healer is + **agent-INDEPENDENT** — it self-corrects with felhom-agent stopped (the whole point). **No unit + declares `RuntimeDirectory=`** (that IS the incident cause; the installer refuses any that does). +- **Go** (`internal/mgmtplane/`): a read-only `Reporter` (os.Stat `/run/sshd` + read the heal-marker + + a short TCP dial to sshd:22) producing the additive `mgmt_plane` heartbeat stanza + (`{privsep_dir_ok, sshd_reachable, healed_recently, privsep_healed_at}` — `omitempty`, the + SelfUpdatePending precedent, no hub-schema change). Wired always-on via + `Collector.SetMgmtPlaneReporter`. The hub raises a warning on a new `privsep_healed_at` so a + recurring clobber surfaces BEFORE a lockout (complements host_staleness). Touches host `/run` + the + stock sshd only — no guests. Closes the update asymmetry: the root-adjacent agent was updated by manual SSH binary-replace while the lower-stakes controller already auto-updates. Design provenance: diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index 0e1dbc7..ce4127b 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -36,6 +36,7 @@ import ( "gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver" "gitea.dooplex.hu/admin/felhom-agent/internal/localapi" applog "gitea.dooplex.hu/admin/felhom-agent/internal/log" + "gitea.dooplex.hu/admin/felhom-agent/internal/mgmtplane" "gitea.dooplex.hu/admin/felhom-agent/internal/pbs" "gitea.dooplex.hu/admin/felhom-agent/internal/provision" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" @@ -646,6 +647,12 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { }) collector.SetSelfUpdateReporter(selfUpdateMgr) // heartbeat pending-status field + // G1: management-plane health observer (break-glass visibility). Read-only + always wired — the + // dumb watchdog (configs/felhom-mgmt-watchdog.*) does the HEALING agent-independently; this only + // REPORTS /run/sshd presence + sshd reachability + any recent auto-heal so the hub can surface a + // recurring clobber before it locks the box out. Port 22 for G1 (H1 passes the felhom-sshd port). + collector.SetMgmtPlaneReporter(mgmtplane.NewReporter(mgmtplane.DefaultPrivsepDir, mgmtplane.DefaultHealMarker, mgmtplane.DefaultSshdPort)) + jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec}, cfg.Hub.HostID, logger) loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner)) diff --git a/configs/felhom-mgmt-watchdog.service b/configs/felhom-mgmt-watchdog.service new file mode 100644 index 0000000..8c95c8c --- /dev/null +++ b/configs/felhom-mgmt-watchdog.service @@ -0,0 +1,19 @@ +# felhom-mgmt-watchdog.service (TASK G1) — install as /etc/systemd/system/felhom-mgmt-watchdog.service. +# +# LAYER 2 (auto-heal without login): a DUMB oneshot that runs /usr/local/sbin/felhom-mgmt-watchdog, +# triggered every ~60s by felhom-mgmt-watchdog.timer. It recreates a missing /run/sshd and clears a +# start-limited stock sshd — so a management-plane lockout self-corrects in ≤1 tick with NOBODY logged +# in and WITHOUT the felhom-agent binary running (the agent only OBSERVES/reports; the heal must not +# depend on it — SPIKE-felhom-sshd-2026-07-05 finding #9 / TASK G1 trap 1). +# +# CRITICAL: this unit MUST NOT declare `RuntimeDirectory=` — that directive (with value `sshd`) is the +# incident it exists to fix. It writes only to /run (dir + heal marker) as root; no state dir needed. +[Unit] +Description=Felhom management-plane watchdog (privsep-dir auto-heal; agent-independent) +# Ordering only — the timer drives cadence; no hard dependency so it runs even in degraded boots. +After=systemd-tmpfiles-setup.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/felhom-mgmt-watchdog +# Deliberately NO RuntimeDirectory=, NO PrivateTmp, NO sandboxing that would hide /run/sshd from it. diff --git a/configs/felhom-mgmt-watchdog.sh b/configs/felhom-mgmt-watchdog.sh new file mode 100644 index 0000000..d681852 --- /dev/null +++ b/configs/felhom-mgmt-watchdog.sh @@ -0,0 +1,68 @@ +#!/bin/sh +# felhom-mgmt-watchdog — the DUMB, agent-INDEPENDENT management-plane healer (TASK G1). +# +# WHY THIS EXISTS: /run/sshd is OpenSSH's compiled-in privilege-separation directory, SHARED by +# every sshd on the host. If it goes missing, sshd's listener stays up but every new session RESETS +# right after SSH2_MSG_KEXINIT — a management-plane lockout on an otherwise-healthy box +# (SPIKE-felhom-sshd-2026-07-05 §8, caused live by a second unit's `RuntimeDirectory=sshd` cleanup). +# The tmpfiles.d entry (felhom-privsep.tmpfiles) PREVENTS the known cause; this script AUTO-HEALS any +# novel cause with ZERO login and ZERO dependency on the felhom-agent binary (so it self-corrects even +# when the agent is down — the whole point). It is run by felhom-mgmt-watchdog.timer every ~60s. +# +# CONTRACT (all four are load-bearing): +# 1. STAT-FIRST / IDEMPOTENT — a HEALTHY /run/sshd is NEVER touched (no mkdir/chmod/chown call), so +# steady state is zero mutation and the dir's mtime is preserved. Only a MISSING or +# wrong-mode/owner dir is corrected. +# 2. reset-failed the STOCK sshd ONLY when it is in the `failed` state (a start-limit lockout) — +# never otherwise, and NEVER `restart` it (mkdir alone restores it; PID stays, spike-proven). +# 3. HEAL MARKER — writes an RFC3339 UTC timestamp to $MARKER only on a real heal, so the agent can +# REPORT the condition to the hub (a recurring clobber must surface BEFORE it becomes a lockout). +# 4. NO `RuntimeDirectory=` anywhere in this feature (that IS the incident cause) — the unit that +# runs this script must not declare one either. +# +# Touches host /run + the stock sshd unit ONLY — no guests (pool-scoping is moot here). Fail-safe: +# any single step failing is logged and does not abort the others (`|| true`); the next tick retries. + +set -u + +PRIVSEP=/run/sshd +MARKER=/run/felhom-mgmt-watchdog.healed +SSHD_UNIT=ssh.service # Debian/PVE stock sshd unit name (sshd.service is an alias) +healed=0 + +log() { logger -t felhom-mgmt-watchdog "$*" 2>/dev/null || true; } + +# 1. Privsep dir — stat-first: create only when missing; correct mode/owner only when wrong. +if [ ! -d "$PRIVSEP" ]; then + if mkdir -p "$PRIVSEP" && chown root:root "$PRIVSEP" && chmod 0755 "$PRIVSEP"; then + healed=1 + log "recreated missing privsep dir $PRIVSEP (0755 root:root)" + else + log "ERROR: failed to recreate $PRIVSEP" + fi +else + mode=$(stat -c %a "$PRIVSEP" 2>/dev/null || echo "") + owner=$(stat -c %U:%G "$PRIVSEP" 2>/dev/null || echo "") + if [ "$mode" != "755" ]; then + chmod 0755 "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP mode ($mode -> 755)" + fi + if [ "$owner" != "root:root" ]; then + chown root:root "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP owner ($owner -> root:root)" + fi +fi + +# 2. Stock sshd — clear a start-limit lockout ONLY when the unit is genuinely `failed`. Never restart +# it (unnecessary — a recreated privsep dir is picked up by the next fork; the listener never died). +if systemctl is-failed --quiet "$SSHD_UNIT" 2>/dev/null; then + if systemctl reset-failed "$SSHD_UNIT" 2>/dev/null; then + healed=1 + log "reset-failed $SSHD_UNIT (was in the failed state)" + fi +fi + +# 3. Heal marker — only on a real heal, so the agent reports the condition (hub raises a warning). +if [ "$healed" = "1" ]; then + date -u +%Y-%m-%dT%H:%M:%SZ > "$MARKER" 2>/dev/null || true +fi + +exit 0 diff --git a/configs/felhom-mgmt-watchdog.timer b/configs/felhom-mgmt-watchdog.timer new file mode 100644 index 0000000..6325a43 --- /dev/null +++ b/configs/felhom-mgmt-watchdog.timer @@ -0,0 +1,18 @@ +# felhom-mgmt-watchdog.timer (TASK G1) — install as /etc/systemd/system/felhom-mgmt-watchdog.timer. +# +# Drives felhom-mgmt-watchdog.service on a ~60s cadence (the auto-heal tick). OnBootSec fires shortly +# after boot (belt-and-suspenders with the tmpfiles layer); OnUnitActiveSec=60s gives the ≤1-tick +# heal budget the G1 acceptance drill measures. Persistent=true runs a missed tick immediately after a +# resume/late boot. The oneshot is idempotent, so a fast cadence never churns a healthy host. +[Unit] +Description=Felhom management-plane watchdog timer (~60s privsep-dir auto-heal tick) + +[Timer] +OnBootSec=30s +OnUnitActiveSec=60s +AccuracySec=5s +Persistent=true +Unit=felhom-mgmt-watchdog.service + +[Install] +WantedBy=timers.target diff --git a/configs/felhom-privsep.tmpfiles b/configs/felhom-privsep.tmpfiles new file mode 100644 index 0000000..088cb2a --- /dev/null +++ b/configs/felhom-privsep.tmpfiles @@ -0,0 +1,15 @@ +# felhom privsep-dir persistence (TASK G1) — install as /etc/tmpfiles.d/felhom-privsep.conf. +# +# LAYER 1 (prevent): make OpenSSH's shared privilege-separation directory /run/sshd boot-persistent +# and owned by NO systemd unit's lifecycle. This closes the exact incident cause from +# SPIKE-felhom-sshd-2026-07-05 §8: a second sshd unit declaring `RuntimeDirectory=sshd` had systemd +# REMOVE the shared /run/sshd on that unit's stop/failure, taking the stock sshd on :22 down with it +# (sessions reset right after SSH2_MSG_KEXINIT). A tmpfiles.d entry recreates the dir at every boot +# independently of any unit, so no unit's RuntimeDirectory cleanup can be the sole owner. +# +# systemd-tmpfiles is idempotent: `systemd-tmpfiles --create` re-run is a no-op on an existing, +# correct dir (it only creates/fixes, never churns). Complemented at runtime by the +# felhom-mgmt-watchdog timer (layer 2), which re-heals a dir removed AFTER boot. +# +# Type d = create the directory if absent (leaves an existing one, only fixing mode/owner). +d /run/sshd 0755 root root - diff --git a/internal/hub/collect.go b/internal/hub/collect.go index c7ebb44..58a2ad4 100644 --- a/internal/hub/collect.go +++ b/internal/hub/collect.go @@ -69,6 +69,7 @@ type Collector struct { leafFP string // v0.48.0: served local-API leaf fp (static per process; "" when local API disabled) wg WireguardReporter // S3: offsite-tunnel status (nil → stanza omitted) selfUpdate SelfUpdateReporter // D1: agent self-update pending status (nil → false) + mgmtPlane MgmtPlaneReporter // G1: management-plane health (nil → stanza omitted) hostID string agentVersion string logger *slog.Logger @@ -140,6 +141,19 @@ func (c *Collector) SetSelfUpdateReporter(s SelfUpdateReporter) *Collector { return c } +// MgmtPlaneReporter is the G1 seam the mgmtplane observer plugs into (same consumer-side pattern — +// hub does not import mgmtplane). nil (feature not wired) → no mgmt_plane stanza on the report. +type MgmtPlaneReporter interface { + MgmtPlaneStatus(ctx context.Context) *MgmtPlaneStatus +} + +// SetMgmtPlaneReporter wires the management-plane health source (G1; nil-safe → stanza omitted). +// Returns the collector for chaining. +func (c *Collector) SetMgmtPlaneReporter(m MgmtPlaneReporter) *Collector { + c.mgmtPlane = m + return c +} + // Collect builds the report. Best-effort liveness: a failed NodeStatus is a hard // error (no useful report — the cycle skips the POST); a failed per-guest // GuestConfig degrades that guest to status="unknown" without spec but still sends; @@ -182,6 +196,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { if c.selfUpdate != nil { report.SelfUpdatePending, report.SelfUpdatePendingVersion = c.selfUpdate.SelfUpdatePending() } + // G1: management-plane health (nil reporter = feature not wired → stanza omitted). + if c.mgmtPlane != nil { + report.MgmtPlane = c.mgmtPlane.MgmtPlaneStatus(ctx) + } return report, nil } diff --git a/internal/hub/collect_mgmtplane_test.go b/internal/hub/collect_mgmtplane_test.go new file mode 100644 index 0000000..f617d2b --- /dev/null +++ b/internal/hub/collect_mgmtplane_test.go @@ -0,0 +1,41 @@ +package hub + +import ( + "context" + "testing" +) + +// fakeMgmtPlane is a MgmtPlaneReporter returning a fixed stanza (or nil). +type fakeMgmtPlane struct{ st *MgmtPlaneStatus } + +func (f fakeMgmtPlane) MgmtPlaneStatus(context.Context) *MgmtPlaneStatus { return f.st } + +func TestCollect_MgmtPlaneOmittedWhenReporterNil(t *testing.T) { + px := &fakePx{node: "n", ns: newTestNodeStatus()} + c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger()) + r, err := c.Collect(context.Background()) + if err != nil { + t.Fatalf("Collect: %v", err) + } + if r.MgmtPlane != nil { + t.Fatalf("no reporter wired → mgmt_plane must be omitted (nil), got %+v", r.MgmtPlane) + } +} + +func TestCollect_MgmtPlanePopulatedWhenWired(t *testing.T) { + px := &fakePx{node: "n", ns: newTestNodeStatus()} + c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger()) + c.SetMgmtPlaneReporter(fakeMgmtPlane{st: &MgmtPlaneStatus{ + PrivsepDirOK: true, SshdReachable: true, HealedRecently: true, PrivsepHealedAt: "2026-07-05T16:42:17Z", + }}) + r, err := c.Collect(context.Background()) + if err != nil { + t.Fatalf("Collect: %v", err) + } + if r.MgmtPlane == nil { + t.Fatal("wired reporter → mgmt_plane must be present") + } + if !r.MgmtPlane.HealedRecently || r.MgmtPlane.PrivsepHealedAt != "2026-07-05T16:42:17Z" { + t.Fatalf("mgmt_plane not carried through: %+v", r.MgmtPlane) + } +} diff --git a/internal/hub/report.go b/internal/hub/report.go index ca9a060..57393b1 100644 --- a/internal/hub/report.go +++ b/internal/hub/report.go @@ -66,6 +66,28 @@ type HostReport struct { // pending=false, the correct default. SelfUpdatePending bool `json:"selfupdate_pending,omitempty"` SelfUpdatePendingVersion string `json:"selfupdate_pending_version,omitempty"` + + // MgmtPlane is the management-plane health stanza (TASK G1). It reports whether OpenSSH's SHARED + // privilege-separation dir (/run/sshd) is present and whether the stock sshd listener answers, plus + // whether the agent-independent watchdog auto-healed a missing dir since boot (and when). This is + // the visibility half of the break-glass system: the dumb watchdog fixes /run/sshd with no login, + // and this stanza surfaces a RECURRING clobber to the operator BEFORE it becomes a lockout — + // complementing host_staleness (which only catches a box gone silent). `omitempty` (the + // SelfUpdatePending precedent): stored opaquely hub-side, so these additive fields need no + // hub-schema change and are absent when the reporter is not wired. + MgmtPlane *MgmtPlaneStatus `json:"mgmt_plane,omitempty"` +} + +// MgmtPlaneStatus is the per-heartbeat management-plane health (TASK G1). Carries no secret. +// HealedRecently is true while the watchdog's heal-marker exists (a privsep-dir heal happened this +// boot); PrivsepHealedAt is that marker's RFC3339 timestamp (absent when no heal has occurred). The +// hub raises a warning event on a PrivsepHealedAt it has not alerted on — a recurring auto-heal means +// a persistent clobber cause worth investigating before the box locks out. +type MgmtPlaneStatus struct { + PrivsepDirOK bool `json:"privsep_dir_ok"` // /run/sshd exists (the KEXINIT-reset detector) + SshdReachable bool `json:"sshd_reachable"` // the stock sshd listener accepts TCP + HealedRecently bool `json:"healed_recently"` // the watchdog heal-marker is present (this boot) + PrivsepHealedAt string `json:"privsep_healed_at,omitempty"` // marker timestamp; hub warns on a NEW value } // WireguardStatus is the per-heartbeat offsite-tunnel status (S3). LastHandshakeAgeS is nil when diff --git a/internal/mgmtplane/mgmtplane.go b/internal/mgmtplane/mgmtplane.go new file mode 100644 index 0000000..5143c7c --- /dev/null +++ b/internal/mgmtplane/mgmtplane.go @@ -0,0 +1,139 @@ +// Package mgmtplane is the agent-side OBSERVER for the management-plane break-glass system (TASK G1). +// +// It does NOT heal anything — the healing is done by the dumb, agent-independent +// felhom-mgmt-watchdog timer (configs/felhom-mgmt-watchdog.{sh,service,timer}), precisely so the +// heal works when the agent is down. This package only READS host state each heartbeat and produces +// the report stanza the hub surfaces: +// +// - /run/sshd present? — OpenSSH's SHARED privsep dir; its absence is the KEXINIT-reset +// lockout (SPIKE-felhom-sshd-2026-07-05 §8). +// - stock sshd listener answers? — a TCP connect to the sshd port (22 for G1; H1 passes the +// discovered felhom-sshd port later). +// - did the watchdog auto-heal? — the watchdog writes an RFC3339 marker to /run when it heals; +// its presence (+ timestamp) tells the hub a clobber recurred, so +// the operator learns of a recurring cause BEFORE a lockout. +// +// Read-only and dependency-light: os.Stat + os.ReadFile + a short net.Dial, no exec, no sudo. Safe to +// run every heartbeat. +package mgmtplane + +import ( + "context" + "net" + "os" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +const ( + // DefaultPrivsepDir is OpenSSH's compiled-in privilege-separation directory (shared by every sshd + // on the host). Its absence is the exact lockout G1 closes. + DefaultPrivsepDir = "/run/sshd" + // DefaultHealMarker is where felhom-mgmt-watchdog records a heal (RFC3339 UTC). On tmpfs, so it + // clears on reboot — "healed since boot" is the intended semantics. + DefaultHealMarker = "/run/felhom-mgmt-watchdog.healed" + // DefaultSshdPort is the stock sshd port G1 observes (H1 will pass the felhom-sshd port). + DefaultSshdPort = 22 + // dialTimeout bounds the sshd reachability probe (a local TCP connect is fast; never block a report). + dialTimeout = 2 * time.Second +) + +// Reporter observes the host management plane. All fields are injectable for tests. +type Reporter struct { + privsepDir string + marker string + sshdAddr string // host:port dialed for the reachability probe + statDir func(string) bool // dir-exists check (os.Stat wrapper; test seam) + readMarker func(string) (string, bool) // marker read → (timestamp, present) + dialSSHD func(ctx context.Context, addr string) bool +} + +// NewReporter builds the production reporter over the real filesystem + a real TCP dial. sshdPort<=0 +// falls back to DefaultSshdPort. +func NewReporter(privsepDir, marker string, sshdPort int) *Reporter { + if privsepDir == "" { + privsepDir = DefaultPrivsepDir + } + if marker == "" { + marker = DefaultHealMarker + } + if sshdPort <= 0 { + sshdPort = DefaultSshdPort + } + return &Reporter{ + privsepDir: privsepDir, + marker: marker, + sshdAddr: net.JoinHostPort("127.0.0.1", itoa(sshdPort)), + statDir: statIsDir, + readMarker: readMarkerFile, + dialSSHD: dialTCP, + } +} + +// MgmtPlaneStatus builds the heartbeat stanza. Never errors — every probe degrades to a boolean; a +// read failure means "not ok", never a crash. Implements hub.MgmtPlaneReporter. +func (r *Reporter) MgmtPlaneStatus(ctx context.Context) *hub.MgmtPlaneStatus { + st := &hub.MgmtPlaneStatus{ + PrivsepDirOK: r.statDir(r.privsepDir), + SshdReachable: r.dialSSHD(ctx, r.sshdAddr), + } + if ts, present := r.readMarker(r.marker); present { + st.HealedRecently = true + st.PrivsepHealedAt = ts + } + return st +} + +// --- production probe impls (all replaceable in tests) --- + +func statIsDir(path string) bool { + fi, err := os.Stat(path) + return err == nil && fi.IsDir() +} + +func readMarkerFile(path string) (string, bool) { + raw, err := os.ReadFile(path) + if err != nil { + return "", false + } + ts := strings.TrimSpace(string(raw)) + if ts == "" { + return "", false // an empty marker is treated as absent (never report a heal we can't timestamp) + } + return ts, true +} + +func dialTCP(ctx context.Context, addr string) bool { + d := net.Dialer{Timeout: dialTimeout} + conn, err := d.DialContext(ctx, "tcp", addr) + if err != nil { + return false + } + _ = conn.Close() + return true +} + +// itoa avoids importing strconv for one call. +func itoa(n int) string { + if n == 0 { + return "0" + } + neg := n < 0 + if neg { + n = -n + } + var b [20]byte + i := len(b) + for n > 0 { + i-- + b[i] = byte('0' + n%10) + n /= 10 + } + if neg { + i-- + b[i] = '-' + } + return string(b[i:]) +} diff --git a/internal/mgmtplane/mgmtplane_test.go b/internal/mgmtplane/mgmtplane_test.go new file mode 100644 index 0000000..1ddbabb --- /dev/null +++ b/internal/mgmtplane/mgmtplane_test.go @@ -0,0 +1,89 @@ +package mgmtplane + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +// newTestReporter builds a Reporter with injected probes so tests never touch the real /run or a +// real socket. privsepOK / sshdOK are the probe verdicts; markerContent is written to a temp marker +// file (empty string = no marker file at all). +func newTestReporter(t *testing.T, privsepOK, sshdOK bool, markerContent string) *Reporter { + t.Helper() + marker := filepath.Join(t.TempDir(), "healed") + if markerContent != "" { + if err := os.WriteFile(marker, []byte(markerContent), 0o600); err != nil { + t.Fatalf("write marker: %v", err) + } + } + return &Reporter{ + privsepDir: "/run/sshd", + marker: marker, + sshdAddr: "127.0.0.1:22", + statDir: func(string) bool { return privsepOK }, + readMarker: readMarkerFile, // the REAL marker reader — exercises the parse (red-proof target) + dialSSHD: func(context.Context, string) bool { return sshdOK }, + } +} + +func TestMgmtPlane_Healthy_NoHealMarker(t *testing.T) { + st := newTestReporter(t, true, true, "").MgmtPlaneStatus(context.Background()) + if !st.PrivsepDirOK || !st.SshdReachable { + t.Fatalf("healthy host: want dir+sshd ok, got %+v", st) + } + if st.HealedRecently || st.PrivsepHealedAt != "" { + t.Fatalf("no marker → HealedRecently must be false + no timestamp, got %+v", st) + } +} + +func TestMgmtPlane_PrivsepDirMissing_IsDetected(t *testing.T) { + // The load-bearing detector: /run/sshd absent = the KEXINIT-reset lockout condition. + st := newTestReporter(t, false, true, "").MgmtPlaneStatus(context.Background()) + if st.PrivsepDirOK { + t.Fatal("privsep dir missing must report PrivsepDirOK=false (the lockout detector)") + } + if !st.SshdReachable { + t.Fatal("listener still up while privsep gone — sshd_reachable should stay true (that's the trap: TCP up, sessions broken)") + } +} + +func TestMgmtPlane_HealMarkerPresent_SurfacesTimestamp(t *testing.T) { + const ts = "2026-07-05T16:42:17Z" + st := newTestReporter(t, true, true, ts).MgmtPlaneStatus(context.Background()) + if !st.HealedRecently { + t.Fatal("watchdog heal-marker present → HealedRecently must be true (the recurring-clobber signal)") + } + if st.PrivsepHealedAt != ts { + t.Fatalf("PrivsepHealedAt: want %q, got %q", ts, st.PrivsepHealedAt) + } +} + +func TestMgmtPlane_EmptyMarker_TreatedAsAbsent(t *testing.T) { + // A truncated/empty marker must NOT report a heal we can't timestamp (would raise a hub warning + // with an empty healed_at). Red-proof: if readMarkerFile returned ("",true) for an empty file, + // HealedRecently would wrongly be true. + st := newTestReporter(t, true, true, " \n").MgmtPlaneStatus(context.Background()) + if st.HealedRecently || st.PrivsepHealedAt != "" { + t.Fatalf("empty marker must be treated as no-heal, got %+v", st) + } +} + +func TestMgmtPlane_SshdUnreachable_Reported(t *testing.T) { + st := newTestReporter(t, true, false, "").MgmtPlaneStatus(context.Background()) + if st.SshdReachable { + t.Fatal("dial failing → sshd_reachable must be false") + } +} + +func TestItoa(t *testing.T) { + for _, c := range []struct { + in int + want string + }{{0, "0"}, {22, "22"}, {8822, "8822"}, {65535, "65535"}} { + if got := itoa(c.in); got != c.want { + t.Fatalf("itoa(%d)=%q want %q", c.in, got, c.want) + } + } +}