feat(mgmtplane): break-glass privsep-dir watchdog + mgmt_plane health (TASK G1) — v0.71.0
Prerequisite for felhom-sshd (H1). Closes the SPIKE-felhom-sshd §8 lockout: a second sshd's RuntimeDirectory=sshd removed the SHARED /run/sshd privsep dir and took stock sshd on :22 down (sessions reset after KEXINIT). Host artifacts (configs/, installed by felhom-host-install): - felhom-privsep.tmpfiles: layer 1, boot-persistent /run/sshd owned by no unit - felhom-mgmt-watchdog.sh/.service/.timer: layer 2, AGENT-INDEPENDENT ~60s heal (stat-first recreate + reset-failed sshd only if failed + heal-marker); never RuntimeDirectory=, never restarts stock sshd, never touches a healthy dir. Go (internal/mgmtplane): read-only Reporter → additive omitempty mgmt_plane heartbeat stanza (privsep_dir_ok/sshd_reachable/healed_recently/privsep_healed_at), wired via Collector.SetMgmtPlaneReporter. Non-hollow tests + red-proofs. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
// Package mgmtplane is the agent-side OBSERVER for the management-plane break-glass system (TASK G1).
|
||||
//
|
||||
// It does NOT heal anything — the healing is done by the dumb, agent-independent
|
||||
// felhom-mgmt-watchdog timer (configs/felhom-mgmt-watchdog.{sh,service,timer}), precisely so the
|
||||
// heal works when the agent is down. This package only READS host state each heartbeat and produces
|
||||
// the report stanza the hub surfaces:
|
||||
//
|
||||
// - /run/sshd present? — OpenSSH's SHARED privsep dir; its absence is the KEXINIT-reset
|
||||
// lockout (SPIKE-felhom-sshd-2026-07-05 §8).
|
||||
// - stock sshd listener answers? — a TCP connect to the sshd port (22 for G1; H1 passes the
|
||||
// discovered felhom-sshd port later).
|
||||
// - did the watchdog auto-heal? — the watchdog writes an RFC3339 marker to /run when it heals;
|
||||
// its presence (+ timestamp) tells the hub a clobber recurred, so
|
||||
// the operator learns of a recurring cause BEFORE a lockout.
|
||||
//
|
||||
// Read-only and dependency-light: os.Stat + os.ReadFile + a short net.Dial, no exec, no sudo. Safe to
|
||||
// run every heartbeat.
|
||||
package mgmtplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultPrivsepDir is OpenSSH's compiled-in privilege-separation directory (shared by every sshd
|
||||
// on the host). Its absence is the exact lockout G1 closes.
|
||||
DefaultPrivsepDir = "/run/sshd"
|
||||
// DefaultHealMarker is where felhom-mgmt-watchdog records a heal (RFC3339 UTC). On tmpfs, so it
|
||||
// clears on reboot — "healed since boot" is the intended semantics.
|
||||
DefaultHealMarker = "/run/felhom-mgmt-watchdog.healed"
|
||||
// DefaultSshdPort is the stock sshd port G1 observes (H1 will pass the felhom-sshd port).
|
||||
DefaultSshdPort = 22
|
||||
// dialTimeout bounds the sshd reachability probe (a local TCP connect is fast; never block a report).
|
||||
dialTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// Reporter observes the host management plane. All fields are injectable for tests.
|
||||
type Reporter struct {
|
||||
privsepDir string
|
||||
marker string
|
||||
sshdAddr string // host:port dialed for the reachability probe
|
||||
statDir func(string) bool // dir-exists check (os.Stat wrapper; test seam)
|
||||
readMarker func(string) (string, bool) // marker read → (timestamp, present)
|
||||
dialSSHD func(ctx context.Context, addr string) bool
|
||||
}
|
||||
|
||||
// NewReporter builds the production reporter over the real filesystem + a real TCP dial. sshdPort<=0
|
||||
// falls back to DefaultSshdPort.
|
||||
func NewReporter(privsepDir, marker string, sshdPort int) *Reporter {
|
||||
if privsepDir == "" {
|
||||
privsepDir = DefaultPrivsepDir
|
||||
}
|
||||
if marker == "" {
|
||||
marker = DefaultHealMarker
|
||||
}
|
||||
if sshdPort <= 0 {
|
||||
sshdPort = DefaultSshdPort
|
||||
}
|
||||
return &Reporter{
|
||||
privsepDir: privsepDir,
|
||||
marker: marker,
|
||||
sshdAddr: net.JoinHostPort("127.0.0.1", itoa(sshdPort)),
|
||||
statDir: statIsDir,
|
||||
readMarker: readMarkerFile,
|
||||
dialSSHD: dialTCP,
|
||||
}
|
||||
}
|
||||
|
||||
// MgmtPlaneStatus builds the heartbeat stanza. Never errors — every probe degrades to a boolean; a
|
||||
// read failure means "not ok", never a crash. Implements hub.MgmtPlaneReporter.
|
||||
func (r *Reporter) MgmtPlaneStatus(ctx context.Context) *hub.MgmtPlaneStatus {
|
||||
st := &hub.MgmtPlaneStatus{
|
||||
PrivsepDirOK: r.statDir(r.privsepDir),
|
||||
SshdReachable: r.dialSSHD(ctx, r.sshdAddr),
|
||||
}
|
||||
if ts, present := r.readMarker(r.marker); present {
|
||||
st.HealedRecently = true
|
||||
st.PrivsepHealedAt = ts
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// --- production probe impls (all replaceable in tests) ---
|
||||
|
||||
func statIsDir(path string) bool {
|
||||
fi, err := os.Stat(path)
|
||||
return err == nil && fi.IsDir()
|
||||
}
|
||||
|
||||
func readMarkerFile(path string) (string, bool) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
ts := strings.TrimSpace(string(raw))
|
||||
if ts == "" {
|
||||
return "", false // an empty marker is treated as absent (never report a heal we can't timestamp)
|
||||
}
|
||||
return ts, true
|
||||
}
|
||||
|
||||
func dialTCP(ctx context.Context, addr string) bool {
|
||||
d := net.Dialer{Timeout: dialTimeout}
|
||||
conn, err := d.DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = conn.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
// itoa avoids importing strconv for one call.
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
neg := n < 0
|
||||
if neg {
|
||||
n = -n
|
||||
}
|
||||
var b [20]byte
|
||||
i := len(b)
|
||||
for n > 0 {
|
||||
i--
|
||||
b[i] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
if neg {
|
||||
i--
|
||||
b[i] = '-'
|
||||
}
|
||||
return string(b[i:])
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package mgmtplane
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// newTestReporter builds a Reporter with injected probes so tests never touch the real /run or a
|
||||
// real socket. privsepOK / sshdOK are the probe verdicts; markerContent is written to a temp marker
|
||||
// file (empty string = no marker file at all).
|
||||
func newTestReporter(t *testing.T, privsepOK, sshdOK bool, markerContent string) *Reporter {
|
||||
t.Helper()
|
||||
marker := filepath.Join(t.TempDir(), "healed")
|
||||
if markerContent != "" {
|
||||
if err := os.WriteFile(marker, []byte(markerContent), 0o600); err != nil {
|
||||
t.Fatalf("write marker: %v", err)
|
||||
}
|
||||
}
|
||||
return &Reporter{
|
||||
privsepDir: "/run/sshd",
|
||||
marker: marker,
|
||||
sshdAddr: "127.0.0.1:22",
|
||||
statDir: func(string) bool { return privsepOK },
|
||||
readMarker: readMarkerFile, // the REAL marker reader — exercises the parse (red-proof target)
|
||||
dialSSHD: func(context.Context, string) bool { return sshdOK },
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_Healthy_NoHealMarker(t *testing.T) {
|
||||
st := newTestReporter(t, true, true, "").MgmtPlaneStatus(context.Background())
|
||||
if !st.PrivsepDirOK || !st.SshdReachable {
|
||||
t.Fatalf("healthy host: want dir+sshd ok, got %+v", st)
|
||||
}
|
||||
if st.HealedRecently || st.PrivsepHealedAt != "" {
|
||||
t.Fatalf("no marker → HealedRecently must be false + no timestamp, got %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_PrivsepDirMissing_IsDetected(t *testing.T) {
|
||||
// The load-bearing detector: /run/sshd absent = the KEXINIT-reset lockout condition.
|
||||
st := newTestReporter(t, false, true, "").MgmtPlaneStatus(context.Background())
|
||||
if st.PrivsepDirOK {
|
||||
t.Fatal("privsep dir missing must report PrivsepDirOK=false (the lockout detector)")
|
||||
}
|
||||
if !st.SshdReachable {
|
||||
t.Fatal("listener still up while privsep gone — sshd_reachable should stay true (that's the trap: TCP up, sessions broken)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_HealMarkerPresent_SurfacesTimestamp(t *testing.T) {
|
||||
const ts = "2026-07-05T16:42:17Z"
|
||||
st := newTestReporter(t, true, true, ts).MgmtPlaneStatus(context.Background())
|
||||
if !st.HealedRecently {
|
||||
t.Fatal("watchdog heal-marker present → HealedRecently must be true (the recurring-clobber signal)")
|
||||
}
|
||||
if st.PrivsepHealedAt != ts {
|
||||
t.Fatalf("PrivsepHealedAt: want %q, got %q", ts, st.PrivsepHealedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_EmptyMarker_TreatedAsAbsent(t *testing.T) {
|
||||
// A truncated/empty marker must NOT report a heal we can't timestamp (would raise a hub warning
|
||||
// with an empty healed_at). Red-proof: if readMarkerFile returned ("",true) for an empty file,
|
||||
// HealedRecently would wrongly be true.
|
||||
st := newTestReporter(t, true, true, " \n").MgmtPlaneStatus(context.Background())
|
||||
if st.HealedRecently || st.PrivsepHealedAt != "" {
|
||||
t.Fatalf("empty marker must be treated as no-heal, got %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMgmtPlane_SshdUnreachable_Reported(t *testing.T) {
|
||||
st := newTestReporter(t, true, false, "").MgmtPlaneStatus(context.Background())
|
||||
if st.SshdReachable {
|
||||
t.Fatal("dial failing → sshd_reachable must be false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestItoa(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
in int
|
||||
want string
|
||||
}{{0, "0"}, {22, "22"}, {8822, "8822"}, {65535, "65535"}} {
|
||||
if got := itoa(c.in); got != c.want {
|
||||
t.Fatalf("itoa(%d)=%q want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user