feat(mgmtplane): break-glass privsep-dir watchdog + mgmt_plane health (TASK G1) — v0.71.0
Prerequisite for felhom-sshd (H1). Closes the SPIKE-felhom-sshd §8 lockout: a second sshd's RuntimeDirectory=sshd removed the SHARED /run/sshd privsep dir and took stock sshd on :22 down (sessions reset after KEXINIT). Host artifacts (configs/, installed by felhom-host-install): - felhom-privsep.tmpfiles: layer 1, boot-persistent /run/sshd owned by no unit - felhom-mgmt-watchdog.sh/.service/.timer: layer 2, AGENT-INDEPENDENT ~60s heal (stat-first recreate + reset-failed sshd only if failed + heal-marker); never RuntimeDirectory=, never restarts stock sshd, never touches a healthy dir. Go (internal/mgmtplane): read-only Reporter → additive omitempty mgmt_plane heartbeat stanza (privsep_dir_ok/sshd_reachable/healed_recently/privsep_healed_at), wired via Collector.SetMgmtPlaneReporter. Non-hollow tests + red-proofs. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
+23
-1
@@ -1,4 +1,26 @@
|
||||
## v0.70.0 — agent self-update (operator-signed, A/B slots, crash-loop auto-rollback) (2026-07-05)
|
||||
## v0.71.0 — management-plane break-glass: privsep-dir watchdog + mgmt_plane health (TASK G1) (2026-07-05)
|
||||
|
||||
Prerequisite for the felhom-sshd OOB feature (H1). Closes the lockout from
|
||||
`felhom.eu/documentation/audits/SPIKE-felhom-sshd-2026-07-05.md` §8: a second sshd's
|
||||
`RuntimeDirectory=sshd` removed the SHARED `/run/sshd` privsep dir and took the stock sshd on :22 down
|
||||
too (sessions reset right after SSH2_MSG_KEXINIT) — a management lockout on a healthy box. Three
|
||||
independent layers; this repo ships the host artifacts + the agent reporter (hub vault + surfacing are
|
||||
the felhom.eu half).
|
||||
|
||||
- **Host artifacts** (`configs/`, installed by felhom-host-install): `felhom-privsep.tmpfiles`
|
||||
(`d /run/sshd 0755 root root -` — layer 1, boot-persistent, owned by no unit's lifecycle);
|
||||
`felhom-mgmt-watchdog.sh` (layer 2 heal action — stat-first recreate `/run/sshd`, `reset-failed`
|
||||
ssh ONLY when `failed`, write an RFC3339 heal-marker; NEVER restarts the stock sshd, NEVER touches a
|
||||
healthy dir — shellcheck-clean); `.service` (oneshot) + `.timer` (~60s, `Persistent`). The healer is
|
||||
**agent-INDEPENDENT** — it self-corrects with felhom-agent stopped (the whole point). **No unit
|
||||
declares `RuntimeDirectory=`** (that IS the incident cause; the installer refuses any that does).
|
||||
- **Go** (`internal/mgmtplane/`): a read-only `Reporter` (os.Stat `/run/sshd` + read the heal-marker +
|
||||
a short TCP dial to sshd:22) producing the additive `mgmt_plane` heartbeat stanza
|
||||
(`{privsep_dir_ok, sshd_reachable, healed_recently, privsep_healed_at}` — `omitempty`, the
|
||||
SelfUpdatePending precedent, no hub-schema change). Wired always-on via
|
||||
`Collector.SetMgmtPlaneReporter`. The hub raises a warning on a new `privsep_healed_at` so a
|
||||
recurring clobber surfaces BEFORE a lockout (complements host_staleness). Touches host `/run` + the
|
||||
stock sshd only — no guests.
|
||||
|
||||
Closes the update asymmetry: the root-adjacent agent was updated by manual SSH binary-replace while
|
||||
the lower-stakes controller already auto-updates. Design provenance:
|
||||
|
||||
@@ -36,6 +36,7 @@ import (
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
||||
applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/mgmtplane"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/pbs"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
@@ -646,6 +647,12 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
||||
})
|
||||
collector.SetSelfUpdateReporter(selfUpdateMgr) // heartbeat pending-status field
|
||||
|
||||
// G1: management-plane health observer (break-glass visibility). Read-only + always wired — the
|
||||
// dumb watchdog (configs/felhom-mgmt-watchdog.*) does the HEALING agent-independently; this only
|
||||
// REPORTS /run/sshd presence + sshd reachability + any recent auto-heal so the hub can surface a
|
||||
// recurring clobber before it locks the box out. Port 22 for G1 (H1 passes the felhom-sshd port).
|
||||
collector.SetMgmtPlaneReporter(mgmtplane.NewReporter(mgmtplane.DefaultPrivsepDir, mgmtplane.DefaultHealMarker, mgmtplane.DefaultSshdPort))
|
||||
|
||||
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec}, cfg.Hub.HostID, logger)
|
||||
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
||||
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# felhom-mgmt-watchdog.service (TASK G1) — install as /etc/systemd/system/felhom-mgmt-watchdog.service.
|
||||
#
|
||||
# LAYER 2 (auto-heal without login): a DUMB oneshot that runs /usr/local/sbin/felhom-mgmt-watchdog,
|
||||
# triggered every ~60s by felhom-mgmt-watchdog.timer. It recreates a missing /run/sshd and clears a
|
||||
# start-limited stock sshd — so a management-plane lockout self-corrects in ≤1 tick with NOBODY logged
|
||||
# in and WITHOUT the felhom-agent binary running (the agent only OBSERVES/reports; the heal must not
|
||||
# depend on it — SPIKE-felhom-sshd-2026-07-05 finding #9 / TASK G1 trap 1).
|
||||
#
|
||||
# CRITICAL: this unit MUST NOT declare `RuntimeDirectory=` — that directive (with value `sshd`) is the
|
||||
# incident it exists to fix. It writes only to /run (dir + heal marker) as root; no state dir needed.
|
||||
[Unit]
|
||||
Description=Felhom management-plane watchdog (privsep-dir auto-heal; agent-independent)
|
||||
# Ordering only — the timer drives cadence; no hard dependency so it runs even in degraded boots.
|
||||
After=systemd-tmpfiles-setup.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/felhom-mgmt-watchdog
|
||||
# Deliberately NO RuntimeDirectory=, NO PrivateTmp, NO sandboxing that would hide /run/sshd from it.
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
# felhom-mgmt-watchdog — the DUMB, agent-INDEPENDENT management-plane healer (TASK G1).
|
||||
#
|
||||
# WHY THIS EXISTS: /run/sshd is OpenSSH's compiled-in privilege-separation directory, SHARED by
|
||||
# every sshd on the host. If it goes missing, sshd's listener stays up but every new session RESETS
|
||||
# right after SSH2_MSG_KEXINIT — a management-plane lockout on an otherwise-healthy box
|
||||
# (SPIKE-felhom-sshd-2026-07-05 §8, caused live by a second unit's `RuntimeDirectory=sshd` cleanup).
|
||||
# The tmpfiles.d entry (felhom-privsep.tmpfiles) PREVENTS the known cause; this script AUTO-HEALS any
|
||||
# novel cause with ZERO login and ZERO dependency on the felhom-agent binary (so it self-corrects even
|
||||
# when the agent is down — the whole point). It is run by felhom-mgmt-watchdog.timer every ~60s.
|
||||
#
|
||||
# CONTRACT (all four are load-bearing):
|
||||
# 1. STAT-FIRST / IDEMPOTENT — a HEALTHY /run/sshd is NEVER touched (no mkdir/chmod/chown call), so
|
||||
# steady state is zero mutation and the dir's mtime is preserved. Only a MISSING or
|
||||
# wrong-mode/owner dir is corrected.
|
||||
# 2. reset-failed the STOCK sshd ONLY when it is in the `failed` state (a start-limit lockout) —
|
||||
# never otherwise, and NEVER `restart` it (mkdir alone restores it; PID stays, spike-proven).
|
||||
# 3. HEAL MARKER — writes an RFC3339 UTC timestamp to $MARKER only on a real heal, so the agent can
|
||||
# REPORT the condition to the hub (a recurring clobber must surface BEFORE it becomes a lockout).
|
||||
# 4. NO `RuntimeDirectory=` anywhere in this feature (that IS the incident cause) — the unit that
|
||||
# runs this script must not declare one either.
|
||||
#
|
||||
# Touches host /run + the stock sshd unit ONLY — no guests (pool-scoping is moot here). Fail-safe:
|
||||
# any single step failing is logged and does not abort the others (`|| true`); the next tick retries.
|
||||
|
||||
set -u
|
||||
|
||||
PRIVSEP=/run/sshd
|
||||
MARKER=/run/felhom-mgmt-watchdog.healed
|
||||
SSHD_UNIT=ssh.service # Debian/PVE stock sshd unit name (sshd.service is an alias)
|
||||
healed=0
|
||||
|
||||
log() { logger -t felhom-mgmt-watchdog "$*" 2>/dev/null || true; }
|
||||
|
||||
# 1. Privsep dir — stat-first: create only when missing; correct mode/owner only when wrong.
|
||||
if [ ! -d "$PRIVSEP" ]; then
|
||||
if mkdir -p "$PRIVSEP" && chown root:root "$PRIVSEP" && chmod 0755 "$PRIVSEP"; then
|
||||
healed=1
|
||||
log "recreated missing privsep dir $PRIVSEP (0755 root:root)"
|
||||
else
|
||||
log "ERROR: failed to recreate $PRIVSEP"
|
||||
fi
|
||||
else
|
||||
mode=$(stat -c %a "$PRIVSEP" 2>/dev/null || echo "")
|
||||
owner=$(stat -c %U:%G "$PRIVSEP" 2>/dev/null || echo "")
|
||||
if [ "$mode" != "755" ]; then
|
||||
chmod 0755 "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP mode ($mode -> 755)"
|
||||
fi
|
||||
if [ "$owner" != "root:root" ]; then
|
||||
chown root:root "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP owner ($owner -> root:root)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Stock sshd — clear a start-limit lockout ONLY when the unit is genuinely `failed`. Never restart
|
||||
# it (unnecessary — a recreated privsep dir is picked up by the next fork; the listener never died).
|
||||
if systemctl is-failed --quiet "$SSHD_UNIT" 2>/dev/null; then
|
||||
if systemctl reset-failed "$SSHD_UNIT" 2>/dev/null; then
|
||||
healed=1
|
||||
log "reset-failed $SSHD_UNIT (was in the failed state)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. Heal marker — only on a real heal, so the agent reports the condition (hub raises a warning).
|
||||
if [ "$healed" = "1" ]; then
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ > "$MARKER" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,18 @@
|
||||
# felhom-mgmt-watchdog.timer (TASK G1) — install as /etc/systemd/system/felhom-mgmt-watchdog.timer.
|
||||
#
|
||||
# Drives felhom-mgmt-watchdog.service on a ~60s cadence (the auto-heal tick). OnBootSec fires shortly
|
||||
# after boot (belt-and-suspenders with the tmpfiles layer); OnUnitActiveSec=60s gives the ≤1-tick
|
||||
# heal budget the G1 acceptance drill measures. Persistent=true runs a missed tick immediately after a
|
||||
# resume/late boot. The oneshot is idempotent, so a fast cadence never churns a healthy host.
|
||||
[Unit]
|
||||
Description=Felhom management-plane watchdog timer (~60s privsep-dir auto-heal tick)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitActiveSec=60s
|
||||
AccuracySec=5s
|
||||
Persistent=true
|
||||
Unit=felhom-mgmt-watchdog.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,15 @@
|
||||
# felhom privsep-dir persistence (TASK G1) — install as /etc/tmpfiles.d/felhom-privsep.conf.
|
||||
#
|
||||
# LAYER 1 (prevent): make OpenSSH's shared privilege-separation directory /run/sshd boot-persistent
|
||||
# and owned by NO systemd unit's lifecycle. This closes the exact incident cause from
|
||||
# SPIKE-felhom-sshd-2026-07-05 §8: a second sshd unit declaring `RuntimeDirectory=sshd` had systemd
|
||||
# REMOVE the shared /run/sshd on that unit's stop/failure, taking the stock sshd on :22 down with it
|
||||
# (sessions reset right after SSH2_MSG_KEXINIT). A tmpfiles.d entry recreates the dir at every boot
|
||||
# independently of any unit, so no unit's RuntimeDirectory cleanup can be the sole owner.
|
||||
#
|
||||
# systemd-tmpfiles is idempotent: `systemd-tmpfiles --create` re-run is a no-op on an existing,
|
||||
# correct dir (it only creates/fixes, never churns). Complemented at runtime by the
|
||||
# felhom-mgmt-watchdog timer (layer 2), which re-heals a dir removed AFTER boot.
|
||||
#
|
||||
# Type d = create the directory if absent (leaves an existing one, only fixing mode/owner).
|
||||
d /run/sshd 0755 root root -
|
||||
@@ -69,6 +69,7 @@ type Collector struct {
|
||||
leafFP string // v0.48.0: served local-API leaf fp (static per process; "" when local API disabled)
|
||||
wg WireguardReporter // S3: offsite-tunnel status (nil → stanza omitted)
|
||||
selfUpdate SelfUpdateReporter // D1: agent self-update pending status (nil → false)
|
||||
mgmtPlane MgmtPlaneReporter // G1: management-plane health (nil → stanza omitted)
|
||||
hostID string
|
||||
agentVersion string
|
||||
logger *slog.Logger
|
||||
@@ -140,6 +141,19 @@ func (c *Collector) SetSelfUpdateReporter(s SelfUpdateReporter) *Collector {
|
||||
return c
|
||||
}
|
||||
|
||||
// MgmtPlaneReporter is the G1 seam the mgmtplane observer plugs into (same consumer-side pattern —
|
||||
// hub does not import mgmtplane). nil (feature not wired) → no mgmt_plane stanza on the report.
|
||||
type MgmtPlaneReporter interface {
|
||||
MgmtPlaneStatus(ctx context.Context) *MgmtPlaneStatus
|
||||
}
|
||||
|
||||
// SetMgmtPlaneReporter wires the management-plane health source (G1; nil-safe → stanza omitted).
|
||||
// Returns the collector for chaining.
|
||||
func (c *Collector) SetMgmtPlaneReporter(m MgmtPlaneReporter) *Collector {
|
||||
c.mgmtPlane = m
|
||||
return c
|
||||
}
|
||||
|
||||
// Collect builds the report. Best-effort liveness: a failed NodeStatus is a hard
|
||||
// error (no useful report — the cycle skips the POST); a failed per-guest
|
||||
// GuestConfig degrades that guest to status="unknown" without spec but still sends;
|
||||
@@ -182,6 +196,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
|
||||
if c.selfUpdate != nil {
|
||||
report.SelfUpdatePending, report.SelfUpdatePendingVersion = c.selfUpdate.SelfUpdatePending()
|
||||
}
|
||||
// G1: management-plane health (nil reporter = feature not wired → stanza omitted).
|
||||
if c.mgmtPlane != nil {
|
||||
report.MgmtPlane = c.mgmtPlane.MgmtPlaneStatus(ctx)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeMgmtPlane is a MgmtPlaneReporter returning a fixed stanza (or nil).
|
||||
type fakeMgmtPlane struct{ st *MgmtPlaneStatus }
|
||||
|
||||
func (f fakeMgmtPlane) MgmtPlaneStatus(context.Context) *MgmtPlaneStatus { return f.st }
|
||||
|
||||
func TestCollect_MgmtPlaneOmittedWhenReporterNil(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger())
|
||||
r, err := c.Collect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Collect: %v", err)
|
||||
}
|
||||
if r.MgmtPlane != nil {
|
||||
t.Fatalf("no reporter wired → mgmt_plane must be omitted (nil), got %+v", r.MgmtPlane)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollect_MgmtPlanePopulatedWhenWired(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger())
|
||||
c.SetMgmtPlaneReporter(fakeMgmtPlane{st: &MgmtPlaneStatus{
|
||||
PrivsepDirOK: true, SshdReachable: true, HealedRecently: true, PrivsepHealedAt: "2026-07-05T16:42:17Z",
|
||||
}})
|
||||
r, err := c.Collect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Collect: %v", err)
|
||||
}
|
||||
if r.MgmtPlane == nil {
|
||||
t.Fatal("wired reporter → mgmt_plane must be present")
|
||||
}
|
||||
if !r.MgmtPlane.HealedRecently || r.MgmtPlane.PrivsepHealedAt != "2026-07-05T16:42:17Z" {
|
||||
t.Fatalf("mgmt_plane not carried through: %+v", r.MgmtPlane)
|
||||
}
|
||||
}
|
||||
@@ -66,6 +66,28 @@ type HostReport struct {
|
||||
// pending=false, the correct default.
|
||||
SelfUpdatePending bool `json:"selfupdate_pending,omitempty"`
|
||||
SelfUpdatePendingVersion string `json:"selfupdate_pending_version,omitempty"`
|
||||
|
||||
// MgmtPlane is the management-plane health stanza (TASK G1). It reports whether OpenSSH's SHARED
|
||||
// privilege-separation dir (/run/sshd) is present and whether the stock sshd listener answers, plus
|
||||
// whether the agent-independent watchdog auto-healed a missing dir since boot (and when). This is
|
||||
// the visibility half of the break-glass system: the dumb watchdog fixes /run/sshd with no login,
|
||||
// and this stanza surfaces a RECURRING clobber to the operator BEFORE it becomes a lockout —
|
||||
// complementing host_staleness (which only catches a box gone silent). `omitempty` (the
|
||||
// SelfUpdatePending precedent): stored opaquely hub-side, so these additive fields need no
|
||||
// hub-schema change and are absent when the reporter is not wired.
|
||||
MgmtPlane *MgmtPlaneStatus `json:"mgmt_plane,omitempty"`
|
||||
}
|
||||
|
||||
// MgmtPlaneStatus is the per-heartbeat management-plane health (TASK G1). Carries no secret.
|
||||
// HealedRecently is true while the watchdog's heal-marker exists (a privsep-dir heal happened this
|
||||
// boot); PrivsepHealedAt is that marker's RFC3339 timestamp (absent when no heal has occurred). The
|
||||
// hub raises a warning event on a PrivsepHealedAt it has not alerted on — a recurring auto-heal means
|
||||
// a persistent clobber cause worth investigating before the box locks out.
|
||||
type MgmtPlaneStatus struct {
|
||||
PrivsepDirOK bool `json:"privsep_dir_ok"` // /run/sshd exists (the KEXINIT-reset detector)
|
||||
SshdReachable bool `json:"sshd_reachable"` // the stock sshd listener accepts TCP
|
||||
HealedRecently bool `json:"healed_recently"` // the watchdog heal-marker is present (this boot)
|
||||
PrivsepHealedAt string `json:"privsep_healed_at,omitempty"` // marker timestamp; hub warns on a NEW value
|
||||
}
|
||||
|
||||
// WireguardStatus is the per-heartbeat offsite-tunnel status (S3). LastHandshakeAgeS is nil when
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
// Package mgmtplane is the agent-side OBSERVER for the management-plane break-glass system (TASK G1).
|
||||
//
|
||||
// It does NOT heal anything — the healing is done by the dumb, agent-independent
|
||||
// felhom-mgmt-watchdog timer (configs/felhom-mgmt-watchdog.{sh,service,timer}), precisely so the
|
||||
// heal works when the agent is down. This package only READS host state each heartbeat and produces
|
||||
// the report stanza the hub surfaces:
|
||||
//
|
||||
// - /run/sshd present? — OpenSSH's SHARED privsep dir; its absence is the KEXINIT-reset
|
||||
// lockout (SPIKE-felhom-sshd-2026-07-05 §8).
|
||||
// - stock sshd listener answers? — a TCP connect to the sshd port (22 for G1; H1 passes the
|
||||
// discovered felhom-sshd port later).
|
||||
// - did the watchdog auto-heal? — the watchdog writes an RFC3339 marker to /run when it heals;
|
||||
// its presence (+ timestamp) tells the hub a clobber recurred, so
|
||||
// the operator learns of a recurring cause BEFORE a lockout.
|
||||
//
|
||||
// Read-only and dependency-light: os.Stat + os.ReadFile + a short net.Dial, no exec, no sudo. Safe to
|
||||
// run every heartbeat.
|
||||
package mgmtplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultPrivsepDir is OpenSSH's compiled-in privilege-separation directory (shared by every sshd
|
||||
// on the host). Its absence is the exact lockout G1 closes.
|
||||
DefaultPrivsepDir = "/run/sshd"
|
||||
// DefaultHealMarker is where felhom-mgmt-watchdog records a heal (RFC3339 UTC). On tmpfs, so it
|
||||
// clears on reboot — "healed since boot" is the intended semantics.
|
||||
DefaultHealMarker = "/run/felhom-mgmt-watchdog.healed"
|
||||
// DefaultSshdPort is the stock sshd port G1 observes (H1 will pass the felhom-sshd port).
|
||||
DefaultSshdPort = 22
|
||||
// dialTimeout bounds the sshd reachability probe (a local TCP connect is fast; never block a report).
|
||||
dialTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// Reporter observes the host management plane. All fields are injectable for tests.
|
||||
type Reporter struct {
|
||||
privsepDir string
|
||||
marker string
|
||||
sshdAddr string // host:port dialed for the reachability probe
|
||||
statDir func(string) bool // dir-exists check (os.Stat wrapper; test seam)
|
||||
readMarker func(string) (string, bool) // marker read → (timestamp, present)
|
||||
dialSSHD func(ctx context.Context, addr string) bool
|
||||
}
|
||||
|
||||
// NewReporter builds the production reporter over the real filesystem + a real TCP dial. sshdPort<=0
|
||||
// falls back to DefaultSshdPort.
|
||||
func NewReporter(privsepDir, marker string, sshdPort int) *Reporter {
|
||||
if privsepDir == "" {
|
||||
privsepDir = DefaultPrivsepDir
|
||||
}
|
||||
if marker == "" {
|
||||
marker = DefaultHealMarker
|
||||
}
|
||||
if sshdPort <= 0 {
|
||||
sshdPort = DefaultSshdPort
|
||||
}
|
||||
return &Reporter{
|
||||
privsepDir: privsepDir,
|
||||
marker: marker,
|
||||
sshdAddr: net.JoinHostPort("127.0.0.1", itoa(sshdPort)),
|
||||
statDir: statIsDir,
|
||||
readMarker: readMarkerFile,
|
||||
dialSSHD: dialTCP,
|
||||
}
|
||||
}
|
||||
|
||||
// MgmtPlaneStatus builds the heartbeat stanza. Never errors — every probe degrades to a boolean; a
|
||||
// read failure means "not ok", never a crash. Implements hub.MgmtPlaneReporter.
|
||||
func (r *Reporter) MgmtPlaneStatus(ctx context.Context) *hub.MgmtPlaneStatus {
|
||||
st := &hub.MgmtPlaneStatus{
|
||||
PrivsepDirOK: r.statDir(r.privsepDir),
|
||||
SshdReachable: r.dialSSHD(ctx, r.sshdAddr),
|
||||
}
|
||||
if ts, present := r.readMarker(r.marker); present {
|
||||
st.HealedRecently = true
|
||||
st.PrivsepHealedAt = ts
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// --- production probe impls (all replaceable in tests) ---
|
||||
|
||||
func statIsDir(path string) bool {
|
||||
fi, err := os.Stat(path)
|
||||
return err == nil && fi.IsDir()
|
||||
}
|
||||
|
||||
func readMarkerFile(path string) (string, bool) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
ts := strings.TrimSpace(string(raw))
|
||||
if ts == "" {
|
||||
return "", false // an empty marker is treated as absent (never report a heal we can't timestamp)
|
||||
}
|
||||
return ts, true
|
||||
}
|
||||
|
||||
func dialTCP(ctx context.Context, addr string) bool {
|
||||
d := net.Dialer{Timeout: dialTimeout}
|
||||
conn, err := d.DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
// itoa avoids importing strconv for one call.
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
neg := n < 0
|
||||
if neg {
|
||||
n = -n
|
||||
}
|
||||
var b [20]byte
|
||||
i := len(b)
|
||||
for n > 0 {
|
||||
i--
|
||||
b[i] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
if neg {
|
||||
i--
|
||||
b[i] = '-'
|
||||
}
|
||||
return string(b[i:])
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package mgmtplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// newTestReporter builds a Reporter with injected probes so tests never touch the real /run or a
|
||||
// real socket. privsepOK / sshdOK are the probe verdicts; markerContent is written to a temp marker
|
||||
// file (empty string = no marker file at all).
|
||||
func newTestReporter(t *testing.T, privsepOK, sshdOK bool, markerContent string) *Reporter {
|
||||
t.Helper()
|
||||
marker := filepath.Join(t.TempDir(), "healed")
|
||||
if markerContent != "" {
|
||||
if err := os.WriteFile(marker, []byte(markerContent), 0o600); err != nil {
|
||||
t.Fatalf("write marker: %v", err)
|
||||
}
|
||||
}
|
||||
return &Reporter{
|
||||
privsepDir: "/run/sshd",
|
||||
marker: marker,
|
||||
sshdAddr: "127.0.0.1:22",
|
||||
statDir: func(string) bool { return privsepOK },
|
||||
readMarker: readMarkerFile, // the REAL marker reader — exercises the parse (red-proof target)
|
||||
dialSSHD: func(context.Context, string) bool { return sshdOK },
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_Healthy_NoHealMarker(t *testing.T) {
|
||||
st := newTestReporter(t, true, true, "").MgmtPlaneStatus(context.Background())
|
||||
if !st.PrivsepDirOK || !st.SshdReachable {
|
||||
t.Fatalf("healthy host: want dir+sshd ok, got %+v", st)
|
||||
}
|
||||
if st.HealedRecently || st.PrivsepHealedAt != "" {
|
||||
t.Fatalf("no marker → HealedRecently must be false + no timestamp, got %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_PrivsepDirMissing_IsDetected(t *testing.T) {
|
||||
// The load-bearing detector: /run/sshd absent = the KEXINIT-reset lockout condition.
|
||||
st := newTestReporter(t, false, true, "").MgmtPlaneStatus(context.Background())
|
||||
if st.PrivsepDirOK {
|
||||
t.Fatal("privsep dir missing must report PrivsepDirOK=false (the lockout detector)")
|
||||
}
|
||||
if !st.SshdReachable {
|
||||
t.Fatal("listener still up while privsep gone — sshd_reachable should stay true (that's the trap: TCP up, sessions broken)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_HealMarkerPresent_SurfacesTimestamp(t *testing.T) {
|
||||
const ts = "2026-07-05T16:42:17Z"
|
||||
st := newTestReporter(t, true, true, ts).MgmtPlaneStatus(context.Background())
|
||||
if !st.HealedRecently {
|
||||
t.Fatal("watchdog heal-marker present → HealedRecently must be true (the recurring-clobber signal)")
|
||||
}
|
||||
if st.PrivsepHealedAt != ts {
|
||||
t.Fatalf("PrivsepHealedAt: want %q, got %q", ts, st.PrivsepHealedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_EmptyMarker_TreatedAsAbsent(t *testing.T) {
|
||||
// A truncated/empty marker must NOT report a heal we can't timestamp (would raise a hub warning
|
||||
// with an empty healed_at). Red-proof: if readMarkerFile returned ("",true) for an empty file,
|
||||
// HealedRecently would wrongly be true.
|
||||
st := newTestReporter(t, true, true, " \n").MgmtPlaneStatus(context.Background())
|
||||
if st.HealedRecently || st.PrivsepHealedAt != "" {
|
||||
t.Fatalf("empty marker must be treated as no-heal, got %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_SshdUnreachable_Reported(t *testing.T) {
|
||||
st := newTestReporter(t, true, false, "").MgmtPlaneStatus(context.Background())
|
||||
if st.SshdReachable {
|
||||
t.Fatal("dial failing → sshd_reachable must be false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestItoa(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
in int
|
||||
want string
|
||||
}{{0, "0"}, {22, "22"}, {8822, "8822"}, {65535, "65535"}} {
|
||||
if got := itoa(c.in); got != c.want {
|
||||
t.Fatalf("itoa(%d)=%q want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user