feat(felhomsshd): dedicated OOB sshd instance + port-adaptive belt (H1 Parts 2-4 agent)
internal/felhomsshd: agent-managed felhom-sshd (claim port [8822,2222,8022,62222] loud-fail-on-exhaustion; render config→sshd -t→reload never-restart-on-change [SF-2]; operator authorized_keys from the hub block outside ~/.ssh [SF-3]); the static-table nft belt mutating ONLY @operator_ips + @ssh_port [trap 4]; health/heal (reset-failed-then-restart with 10min cooldown, NEVER restart onto an invalid config) + the oob heartbeat stanza. configs/felhom-sshd.service (SAFE, no RuntimeDirectory [SF-1]). FELHOM_SSHD + FELHOM_OOB sudoers (set-elements only). oob.enabled config DEFAULT FALSE. Wired into main like wgtunnel. Non-hollow tests: claim clean/contention/idempotent/exhaustion; config safe+byte-stable+refuses-:22; belt mutate-then-idempotent + never-touches-rules; heal no-restart-on-invalid-config + cooldown; status reflects block. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -31,6 +31,7 @@ import (
|
|||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/desired"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/desired"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/dr"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/dr"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/felhomsshd"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/guesthook"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/guesthook"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver"
|
||||||
@@ -706,6 +707,28 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OOB operator access (H1): the dedicated felhom-sshd instance + port-adaptive belt + health.
|
||||||
|
// DEFAULT OFF (oob.enabled) until the operator endpoint + static belt table exist. Consumes the
|
||||||
|
// SAME wireguard desired-state block (oob_peer_ip → belt, oob_operator_ssh_key → authorized_keys).
|
||||||
|
oobServers := 0
|
||||||
|
var oobLoop *felhomsshd.Loop
|
||||||
|
{
|
||||||
|
oc := cfg.OOB.WithDefaults()
|
||||||
|
if oc.Enabled {
|
||||||
|
oobMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
||||||
|
if oobMode == "" {
|
||||||
|
oobMode = proxmox.RunnerSudo
|
||||||
|
}
|
||||||
|
oobRunner := &proxmox.ExecRunner{Mode: oobMode, SudoPath: cfg.Privileged.SudoPath}
|
||||||
|
oobMgr := felhomsshd.NewManager(oobRunner, oc.StateDir, logger)
|
||||||
|
oobBelt := felhomsshd.NewBelt(oobRunner, logger)
|
||||||
|
oobLoop = felhomsshd.NewLoop(oobMgr, oobBelt, time.Duration(oc.IntervalSeconds)*time.Second, logger)
|
||||||
|
desiredSyncer.AddConsumer(oobLoop) // raw desired-state → oob_peer_ip + operator key
|
||||||
|
collector.SetOOBReporter(oobLoop) // heartbeat oob health stanza
|
||||||
|
logger.Info("felhomsshd (OOB): enabled", "interval_s", oc.IntervalSeconds, "state_dir", oc.StateDir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, the PBS
|
// Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, the PBS
|
||||||
// verify loop, (optionally) the local-API server, and (optionally) the LAN resolver loop
|
// verify loop, (optionally) the local-API server, and (optionally) the LAN resolver loop
|
||||||
// concurrently; any one returning ends the daemon (ctx cancel tears down the rest).
|
// concurrently; any one returning ends the daemon (ctx cancel tears down the rest).
|
||||||
@@ -768,6 +791,10 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
|||||||
wgServers = 1
|
wgServers = 1
|
||||||
go func() { errc <- wgLoop.Run(ctx) }()
|
go func() { errc <- wgLoop.Run(ctx) }()
|
||||||
}
|
}
|
||||||
|
if oobLoop != nil {
|
||||||
|
oobServers = 1
|
||||||
|
go func() { errc <- oobLoop.Run(ctx) }()
|
||||||
|
}
|
||||||
|
|
||||||
// TASK D1 Scenario D: if this process is a JUST-FLIPPED self-update (a pending marker names THIS
|
// TASK D1 Scenario D: if this process is a JUST-FLIPPED self-update (a pending marker names THIS
|
||||||
// version), commit it after a clean dwell — but only now that core init is done (config parsed,
|
// version), commit it after a clean dwell — but only now that core init is done (config parsed,
|
||||||
@@ -777,8 +804,8 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
|||||||
go selfUpdateMgr.MaybeCommit(ctx)
|
go selfUpdateMgr.MaybeCommit(ctx)
|
||||||
|
|
||||||
err = <-errc
|
err = <-errc
|
||||||
stop() // tear down the siblings on the first exit
|
stop() // tear down the siblings on the first exit
|
||||||
for i := 0; i < 4+localServers+lanServers+wgServers; i++ { // wait for the other goroutines
|
for i := 0; i < 4+localServers+lanServers+wgServers+oobServers; i++ { // wait for the other goroutines
|
||||||
<-errc
|
<-errc
|
||||||
}
|
}
|
||||||
if err != nil && err != context.Canceled {
|
if err != nil && err != context.Canceled {
|
||||||
|
|||||||
@@ -178,4 +178,35 @@ Cmnd_Alias FELHOM_SELFUPDATE = \
|
|||||||
/usr/local/sbin/felhom-selfupdate-guarded commit, \
|
/usr/local/sbin/felhom-selfupdate-guarded commit, \
|
||||||
/usr/local/sbin/felhom-selfupdate-guarded rollback
|
/usr/local/sbin/felhom-selfupdate-guarded rollback
|
||||||
|
|
||||||
felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE
|
# Dedicated OOB sshd (TASK H1). The agent manages felhom-sshd like wg-felhom/dnsmasq: it RENDERS the
|
||||||
|
# config (Port from its claim) + the operator's authorized_keys, validates with `sshd -t`, and reloads
|
||||||
|
# (never restart-on-change [SF-2]). Both install SOURCES are the agent-owned staged files under
|
||||||
|
# StateDir; both DESTINATIONS are FIXED. `sshd -t/-T` are the validate/discover reads. The
|
||||||
|
# systemctl verbs are SCOPED to felhom-sshd only. reset-failed precedes a deliberate restart [SF-5].
|
||||||
|
# NOTHING here can touch the stock sshd, :22, or /etc/ssh.
|
||||||
|
Cmnd_Alias FELHOM_SSHD = \
|
||||||
|
/usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/felhom-sshd/sshd_config /etc/felhom-sshd/sshd_config, \
|
||||||
|
/usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/felhom-sshd/authorized_keys.felhom-op /etc/felhom-sshd/authorized_keys/felhom-op, \
|
||||||
|
/usr/sbin/sshd -t -f /var/lib/felhom-agent/felhom-sshd/sshd_config, \
|
||||||
|
/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config, \
|
||||||
|
/usr/sbin/sshd -T -f /etc/felhom-sshd/sshd_config, \
|
||||||
|
/usr/bin/systemctl enable --now felhom-sshd, \
|
||||||
|
/usr/bin/systemctl reload felhom-sshd, \
|
||||||
|
/usr/bin/systemctl restart felhom-sshd, \
|
||||||
|
/usr/bin/systemctl reset-failed felhom-sshd, \
|
||||||
|
/usr/bin/wg show wg-felhom latest-handshakes
|
||||||
|
|
||||||
|
# OOB nft belt (TASK H1). The STATIC table `inet felhom_oob` is installed once by host-install; the
|
||||||
|
# agent mutates ONLY its two SETS — @operator_ips (the operator /32) + @ssh_port (the claimed port).
|
||||||
|
# SET ELEMENTS ONLY [trap 4]: NO `nft add rule`, NO `nft -f`, NO `flush ruleset/table` — a rule grant
|
||||||
|
# would let the agent firewall anything. The agent fine-validates every element (netip / int range)
|
||||||
|
# before exec; the trailing wildcards are the coarse allowlist (values only).
|
||||||
|
Cmnd_Alias FELHOM_OOB = \
|
||||||
|
/usr/sbin/nft list set inet felhom_oob operator_ips, \
|
||||||
|
/usr/sbin/nft list set inet felhom_oob ssh_port, \
|
||||||
|
/usr/sbin/nft flush set inet felhom_oob operator_ips, \
|
||||||
|
/usr/sbin/nft flush set inet felhom_oob ssh_port, \
|
||||||
|
/usr/sbin/nft add element inet felhom_oob operator_ips *, \
|
||||||
|
/usr/sbin/nft add element inet felhom_oob ssh_port *
|
||||||
|
|
||||||
|
felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
# felhom-sshd.service (TASK H1) — install as /etc/systemd/system/felhom-sshd.service.
|
||||||
|
#
|
||||||
|
# The dedicated OOB sshd instance: a SECOND sshd on a claimed non-22 port, COEXISTING with the
|
||||||
|
# customer's/stock sshd on :22 (never touched). Config is AGENT-RENDERED at /etc/felhom-sshd/sshd_config
|
||||||
|
# (Port from the agent's claim); the agent reloads on change (never restart-on-change [SF-2]).
|
||||||
|
#
|
||||||
|
# CRITICAL [SF-1]: this unit MUST NOT declare `RuntimeDirectory=` — that directive (value `sshd`)
|
||||||
|
# removed the SHARED /run/sshd and broke the stock sshd live (SPIKE-felhom-sshd §8). /run/sshd is
|
||||||
|
# guaranteed by G1 (tmpfiles + watchdog); the ExecStartPre mkdir is a harmless extra guarantee. The
|
||||||
|
# host-install RuntimeDirectory guard will REJECT this unit if the directive is ever added.
|
||||||
|
[Unit]
|
||||||
|
Description=Felhom OOB sshd (dedicated instance, agent-managed; H1)
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=notify
|
||||||
|
ExecStartPre=-/usr/bin/mkdir -p /run/sshd
|
||||||
|
ExecStartPre=/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config
|
||||||
|
ExecStart=/usr/sbin/sshd -D -f /etc/felhom-sshd/sshd_config
|
||||||
|
# Config-change path: validate then HUP — a bad reload is REFUSED and the running daemon SURVIVES
|
||||||
|
# [SF-2]. The agent always writes→sshd -t→reload; it never `restart`s on a config change.
|
||||||
|
ExecReload=/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config
|
||||||
|
ExecReload=/bin/kill -HUP $MAINPID
|
||||||
|
KillMode=process
|
||||||
|
Restart=on-failure
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -32,10 +32,32 @@ type Config struct {
|
|||||||
LocalAPI LocalAPIConfig `json:"local_api"`
|
LocalAPI LocalAPIConfig `json:"local_api"`
|
||||||
LANResolver LANResolverConfig `json:"lan_resolver"`
|
LANResolver LANResolverConfig `json:"lan_resolver"`
|
||||||
WGTunnel WGTunnelConfig `json:"wg_tunnel"`
|
WGTunnel WGTunnelConfig `json:"wg_tunnel"`
|
||||||
|
OOB OOBConfig `json:"oob"`
|
||||||
SelfUpdate SelfUpdateConfig `json:"selfupdate"`
|
SelfUpdate SelfUpdateConfig `json:"selfupdate"`
|
||||||
LogLevel string `json:"log_level"` // debug|info|warn|error (default info)
|
LogLevel string `json:"log_level"` // debug|info|warn|error (default info)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OOBConfig configures the dedicated felhom-sshd OOB access instance + belt (TASK H1). **Enabled
|
||||||
|
// DEFAULTS TO FALSE** — a rollout to a box without explicit oob.enabled=true is a no-op (no port
|
||||||
|
// claim, no config render, no belt mutation, no oob report stanza) until the operator endpoint +
|
||||||
|
// static belt table exist.
|
||||||
|
type OOBConfig struct {
|
||||||
|
Enabled bool `json:"enabled"`
|
||||||
|
IntervalSeconds int `json:"interval_seconds"` // reconcile cadence; default 60
|
||||||
|
StateDir string `json:"state_dir"` // staged config/authkeys under <StateDir>/felhom-sshd/; default /var/lib/felhom-agent
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithDefaults fills the OOB reconcile cadence + state dir.
|
||||||
|
func (o OOBConfig) WithDefaults() OOBConfig {
|
||||||
|
if o.IntervalSeconds == 0 {
|
||||||
|
o.IntervalSeconds = 60
|
||||||
|
}
|
||||||
|
if o.StateDir == "" {
|
||||||
|
o.StateDir = "/var/lib/felhom-agent"
|
||||||
|
}
|
||||||
|
return o
|
||||||
|
}
|
||||||
|
|
||||||
// SelfUpdateConfig configures the operator-signed agent self-update (TASK D1). The artifact HOST
|
// SelfUpdateConfig configures the operator-signed agent self-update (TASK D1). The artifact HOST
|
||||||
// is operator-controlled config; the artifact INTEGRITY comes only from the sha256 pinned inside
|
// is operator-controlled config; the artifact INTEGRITY comes only from the sha256 pinned inside
|
||||||
// the operator-signed op — the hub's Day-0 manifest plays no role here, and a compromised Gitea
|
// the operator-signed op — the hub's Day-0 manifest plays no role here, and a compromised Gitea
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
package felhomsshd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"net/netip"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Belt keeps the STATIC nft table `inet felhom_oob` (installed by host-install) converged on the
|
||||||
|
// current OOB state by mutating only its SETS — @operator_ips (the operator /32 allowed to reach
|
||||||
|
// felhom-sshd over wg-felhom) and @ssh_port (felhom-sshd's claimed port). The agent NEVER adds/removes
|
||||||
|
// RULES [trap 4]: set-element mutation can't change rule semantics, so the narrow sudoers grant stays
|
||||||
|
// safe. Idempotent: reads the current elements and mutates only on a difference (no churn).
|
||||||
|
type Belt struct {
|
||||||
|
runner proxmox.Runner
|
||||||
|
logger *slog.Logger
|
||||||
|
|
||||||
|
table string // "felhom_oob"
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBelt builds a Belt over the narrow runner. table defaults to "felhom_oob".
|
||||||
|
func NewBelt(runner proxmox.Runner, logger *slog.Logger) *Belt {
|
||||||
|
if logger == nil {
|
||||||
|
logger = slog.Default()
|
||||||
|
}
|
||||||
|
return &Belt{runner: runner, logger: logger, table: "felhom_oob"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync converges @operator_ips + @ssh_port on the desired values. operatorIP "" empties @operator_ips
|
||||||
|
// (belt drops all tunnel SSH — OOB off). port 0 empties @ssh_port. Every value is validated before it
|
||||||
|
// reaches nft (netip for the IP, int range for the port), so the coarse sudoers wildcard can never be
|
||||||
|
// abused. A missing table (host-install not run / pre-H1 box) degrades to a single logged warning.
|
||||||
|
func (b *Belt) Sync(ctx context.Context, port int, operatorIP string) {
|
||||||
|
// desired sets
|
||||||
|
wantOps := []string{}
|
||||||
|
if operatorIP != "" {
|
||||||
|
ip, err := netip.ParseAddr(operatorIP)
|
||||||
|
if err != nil || !ip.Is4() {
|
||||||
|
b.logger.Error("felhomsshd belt: operator ip invalid — refusing", "ip", operatorIP)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
wantOps = []string{ip.String()}
|
||||||
|
}
|
||||||
|
wantPorts := []string{}
|
||||||
|
if port > 0 && port <= 65535 {
|
||||||
|
wantPorts = []string{strconv.Itoa(port)}
|
||||||
|
}
|
||||||
|
|
||||||
|
b.syncSet(ctx, "operator_ips", wantOps)
|
||||||
|
b.syncSet(ctx, "ssh_port", wantPorts)
|
||||||
|
}
|
||||||
|
|
||||||
|
var nftElemRe = regexp.MustCompile(`elements\s*=\s*\{([^}]*)\}`)
|
||||||
|
|
||||||
|
// syncSet converges one named set on `want` (sorted, deduped). Reads current elements; if they match,
|
||||||
|
// ZERO nft mutations (the idempotency the scenario asserts). Otherwise flush + add the desired
|
||||||
|
// elements. A read failure (table/set absent) → one warning, no mutation.
|
||||||
|
func (b *Belt) syncSet(ctx context.Context, setName string, want []string) {
|
||||||
|
want = sortedUnique(want)
|
||||||
|
cur, ok := b.readSet(ctx, setName)
|
||||||
|
if !ok {
|
||||||
|
b.logger.Warn("felhomsshd belt: set unreadable (table not installed?) — skipping", "set", setName)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if equalStringSlices(cur, want) {
|
||||||
|
return // steady state: no mutation
|
||||||
|
}
|
||||||
|
if _, errOut, err := b.runner.Run(ctx, "nft", "flush", "set", "inet", b.table, setName); err != nil {
|
||||||
|
b.logger.Error("felhomsshd belt: flush failed", "set", setName, "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, e := range want {
|
||||||
|
if _, errOut, err := b.runner.Run(ctx, "nft", "add", "element", "inet", b.table, setName, "{ "+e+" }"); err != nil {
|
||||||
|
b.logger.Error("felhomsshd belt: add element failed", "set", setName, "elem", e, "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.logger.Info("felhomsshd belt: set synced", "set", setName, "elements", strings.Join(want, ","))
|
||||||
|
}
|
||||||
|
|
||||||
|
// readSet returns the current elements of a named set (sorted), or ok=false if the set can't be read.
|
||||||
|
func (b *Belt) readSet(ctx context.Context, setName string) ([]string, bool) {
|
||||||
|
out, _, err := b.runner.Run(ctx, "nft", "list", "set", "inet", b.table, setName)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
m := nftElemRe.FindSubmatch(out)
|
||||||
|
if m == nil {
|
||||||
|
return []string{}, true // set exists but empty (no "elements = {}" block)
|
||||||
|
}
|
||||||
|
var elems []string
|
||||||
|
for _, f := range strings.Split(string(m[1]), ",") {
|
||||||
|
f = strings.TrimSpace(f)
|
||||||
|
if f != "" {
|
||||||
|
elems = append(elems, f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sortedUnique(elems), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedUnique(in []string) []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
out := make([]string, 0, len(in))
|
||||||
|
for _, s := range in {
|
||||||
|
if !seen[s] {
|
||||||
|
seen[s] = true
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func equalStringSlices(a, b []string) bool {
|
||||||
|
if len(a) != len(b) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range a {
|
||||||
|
if a[i] != b[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -0,0 +1,178 @@
|
|||||||
|
package felhomsshd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||||
|
)
|
||||||
|
|
||||||
|
// scriptRunner records every exec and returns scripted stdout per "name+first-arg" key; scripted
|
||||||
|
// errors per the same key. Everything else returns empty success.
|
||||||
|
type scriptRunner struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
calls [][]string
|
||||||
|
stdout map[string]string
|
||||||
|
errs map[string]error
|
||||||
|
}
|
||||||
|
|
||||||
|
func newScriptRunner() *scriptRunner {
|
||||||
|
return &scriptRunner{stdout: map[string]string{}, errs: map[string]error{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func key(name string, args ...string) string {
|
||||||
|
// `nft list set inet felhom_oob <setName>` → key by the set name (last arg), so the two set reads
|
||||||
|
// are distinguishable. Everything else keys by first arg.
|
||||||
|
if name == "nft" && len(args) > 0 && args[0] == "list" {
|
||||||
|
return "nft list " + args[len(args)-1]
|
||||||
|
}
|
||||||
|
if len(args) > 0 {
|
||||||
|
return name + " " + args[0]
|
||||||
|
}
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *scriptRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
r.mu.Lock()
|
||||||
|
r.calls = append(r.calls, append([]string{name}, args...))
|
||||||
|
k := key(name, args...)
|
||||||
|
out := r.stdout[k]
|
||||||
|
err := r.errs[k]
|
||||||
|
r.mu.Unlock()
|
||||||
|
if err != nil {
|
||||||
|
return nil, []byte("scripted failure"), err
|
||||||
|
}
|
||||||
|
return []byte(out), nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *scriptRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
return r.Run(ctx, name, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *scriptRunner) count(name, firstArg string) int {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
n := 0
|
||||||
|
for _, c := range r.calls {
|
||||||
|
if c[0] == name && len(c) > 1 && c[1] == firstArg {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *scriptRunner) sawRuleMutation() bool {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
for _, c := range r.calls {
|
||||||
|
// trap 4: the agent must NEVER run `nft add rule` / `nft -f` / `nft flush ruleset|table`.
|
||||||
|
if c[0] == "nft" && len(c) > 1 {
|
||||||
|
if c[1] == "-f" || (len(c) > 2 && c[1] == "add" && c[2] == "rule") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if c[1] == "flush" && len(c) > 2 && (c[2] == "ruleset" || c[2] == "table") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBelt_SyncMutatesThenIdempotent(t *testing.T) {
|
||||||
|
r := newScriptRunner()
|
||||||
|
// first read: both sets empty (no "elements" block)
|
||||||
|
r.stdout["nft list operator_ips"] = "set operator_ips {\n\ttype ipv4_addr\n}"
|
||||||
|
r.stdout["nft list ssh_port"] = "set ssh_port {\n\ttype inet_service\n}"
|
||||||
|
b := NewBelt(r, nil)
|
||||||
|
b.Sync(context.Background(), 8822, "10.77.0.250")
|
||||||
|
|
||||||
|
// mutations happened: flush + add for BOTH sets
|
||||||
|
if r.count("nft", "flush") < 2 || r.count("nft", "add") < 2 {
|
||||||
|
t.Fatalf("first sync must flush+add both sets; flushes=%d adds=%d", r.count("nft", "flush"), r.count("nft", "add"))
|
||||||
|
}
|
||||||
|
if r.sawRuleMutation() {
|
||||||
|
t.Fatal("belt must NEVER mutate rules — only set elements (trap 4)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// second sync: the reads now return the desired elements → ZERO mutations (idempotent)
|
||||||
|
r2 := newScriptRunner()
|
||||||
|
r2.stdout["nft list operator_ips"] = "elements = { 10.77.0.250 }"
|
||||||
|
r2.stdout["nft list ssh_port"] = "elements = { 8822 }"
|
||||||
|
b2 := NewBelt(r2, nil)
|
||||||
|
b2.Sync(context.Background(), 8822, "10.77.0.250")
|
||||||
|
if r2.count("nft", "flush") != 0 || r2.count("nft", "add") != 0 {
|
||||||
|
t.Fatalf("idempotent sync must do ZERO mutations; flushes=%d adds=%d", r2.count("nft", "flush"), r2.count("nft", "add"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBelt_OperatorEmptyEmptiesSet(t *testing.T) {
|
||||||
|
r := newScriptRunner()
|
||||||
|
r.stdout["nft list operator_ips"] = "elements = { 10.77.0.250 }" // currently has an operator IP
|
||||||
|
r.stdout["nft list ssh_port"] = "elements = { 8822 }"
|
||||||
|
b := NewBelt(r, nil)
|
||||||
|
b.Sync(context.Background(), 8822, "") // OOB off → operator set must be emptied
|
||||||
|
// operator_ips: current {250} != desired {} → flush, no add. ssh_port: {} vs {8822}...
|
||||||
|
if r.count("nft", "flush") < 1 {
|
||||||
|
t.Fatal("emptying the operator set must flush it")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// health: inactive + INVALID config → NO restart (the red-proof); inactive + valid → restart.
|
||||||
|
func TestHeal_NoRestartOnInvalidConfig(t *testing.T) {
|
||||||
|
r := newScriptRunner()
|
||||||
|
r.errs["sshd -t"] = context.DeadlineExceeded // sshd -t FAILS (config invalid)
|
||||||
|
m := newTestManager(r, false) // inactive
|
||||||
|
m.HealAndCheck(context.Background(), 8822)
|
||||||
|
if r.count("systemctl", "restart") != 0 {
|
||||||
|
t.Fatal("a broken config must NOT trigger a restart (never degraded→dead)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHeal_RestartsWhenDownWithValidConfigThenCooldown(t *testing.T) {
|
||||||
|
r := newScriptRunner() // sshd -t passes (no error)
|
||||||
|
m := newTestManager(r, false)
|
||||||
|
m.HealAndCheck(context.Background(), 8822)
|
||||||
|
if r.count("systemctl", "restart") != 1 {
|
||||||
|
t.Fatalf("down + valid config → exactly one restart, got %d", r.count("systemctl", "restart"))
|
||||||
|
}
|
||||||
|
// cooldown: a second call inside the window → still one restart
|
||||||
|
m.HealAndCheck(context.Background(), 8822)
|
||||||
|
if r.count("systemctl", "restart") != 1 {
|
||||||
|
t.Fatalf("restart cooldown violated: %d restarts", r.count("systemctl", "restart"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatus_ReflectsBlockAndPort(t *testing.T) {
|
||||||
|
r := newScriptRunner()
|
||||||
|
r.stdout["sshd -T"] = "port 8822\nsomethingelse yes\n"
|
||||||
|
m := newTestManager(r, true) // active
|
||||||
|
m.port = 8822
|
||||||
|
st := m.Status(context.Background(), &hub.WireWireguard{OOBPeerIP: "10.77.0.250", OOBOperatorSSHKey: "ssh-ed25519 AAAA op"})
|
||||||
|
if !st.FelhomSshdActive || st.FelhomSshdPort != 8822 {
|
||||||
|
t.Fatalf("status: %+v", st)
|
||||||
|
}
|
||||||
|
if !st.OperatorPeerConfigured || !st.OperatorKeyConfigured {
|
||||||
|
t.Fatalf("status must reflect operator peer + key configured: %+v", st)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// newTestManager builds a Manager with injected active-state + a now clock; sshd -T/-t go through the
|
||||||
|
// runner. isFree unused here.
|
||||||
|
func newTestManager(r *scriptRunner, active bool) *Manager {
|
||||||
|
m := NewManager(r, "/tmp/felhomsshd-test", nil)
|
||||||
|
m.isActive = func(context.Context) bool { return active }
|
||||||
|
m.isFailed = func(context.Context) bool { return false }
|
||||||
|
m.now = func() time.Time { return time.Unix(1783270000, 0) }
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfig_NoRuntimeDirectoryString(t *testing.T) {
|
||||||
|
c, _ := renderConfig(2222)
|
||||||
|
if strings.Contains(c, "RuntimeDirectory") {
|
||||||
|
t.Fatal("SF-1: config must never contain RuntimeDirectory")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package felhomsshd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Candidates is the ordered OOB-port candidate list (spike §2). First free wins; NEVER :22 or a
|
||||||
|
// random port. Package-var (not const) so tests can shrink it.
|
||||||
|
var Candidates = []int{8822, 2222, 8022, 62222}
|
||||||
|
|
||||||
|
// ErrPortsExhausted is returned when every candidate is busy — a LOUD failure [SF-4/trap 6], never a
|
||||||
|
// silent fallback to :22 or a random high port.
|
||||||
|
var ErrPortsExhausted = fmt.Errorf("felhomsshd: all candidate OOB ports are busy — refusing to fall back to :22 or a random port")
|
||||||
|
|
||||||
|
// portProbe reports whether a TCP port is free (nothing listening AND a real bind succeeds). Injected
|
||||||
|
// for tests; production impl = probeFree (ss + net.Listen).
|
||||||
|
type portProbe func(port int) bool
|
||||||
|
|
||||||
|
// claimPort returns the OOB port, mirroring the spike's shell algorithm:
|
||||||
|
// - if a persisted port exists AND is still free → keep it (idempotent, no thrash),
|
||||||
|
// - else the FIRST free candidate → persist + return,
|
||||||
|
// - else ErrPortsExhausted (LOUD).
|
||||||
|
//
|
||||||
|
// persist writes PortFile; readPersisted reads it. isFree is the probe. All injected for tests.
|
||||||
|
func claimPort(candidates []int, isFree portProbe, readPersisted func() (int, bool), persist func(int) error) (int, error) {
|
||||||
|
if cur, ok := readPersisted(); ok && cur != 22 && isFree(cur) {
|
||||||
|
return cur, nil
|
||||||
|
}
|
||||||
|
for _, p := range candidates {
|
||||||
|
if p == 22 {
|
||||||
|
continue // defensive: never :22
|
||||||
|
}
|
||||||
|
if isFree(p) {
|
||||||
|
if err := persist(p); err != nil {
|
||||||
|
return 0, fmt.Errorf("felhomsshd: persist claimed port %d: %w", p, err)
|
||||||
|
}
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, ErrPortsExhausted
|
||||||
|
}
|
||||||
|
|
||||||
|
// readPortFile parses PortFile → (port, ok). A missing/garbage file → (0,false).
|
||||||
|
func readPortFile() (int, bool) {
|
||||||
|
raw, err := os.ReadFile(PortFile)
|
||||||
|
if err != nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
p, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||||
|
if err != nil || p < 1 || p > 65535 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return p, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// writePortFile persists the claimed port (0644 — a port is not a secret).
|
||||||
|
func writePortFile(port int) error {
|
||||||
|
if err := os.MkdirAll(ConfDir, 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(PortFile, []byte(strconv.Itoa(port)+"\n"), 0o644)
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
// Package felhomsshd manages the dedicated OOB sshd instance (TASK H1). It is a SECOND sshd —
|
||||||
|
// separate port, config, host keys, AuthorizedKeysFile, and systemd unit — that COEXISTS with the
|
||||||
|
// customer's/stock sshd on :22 (never touched). Design + safety earned by
|
||||||
|
// SPIKE-felhom-sshd-2026-07-05 (§2 claim, §3 SAFE unit, §5 reload-not-restart, §7 AuthorizedKeysFile
|
||||||
|
// isolation) and SPIKE-oob-wg-operator-peer-2026-07-05 (the tunnel-only belt).
|
||||||
|
//
|
||||||
|
// The agent RENDERS the config (Port from the claim) and reloads on change — the wg-felhom pattern.
|
||||||
|
// It NEVER declares RuntimeDirectory= (G1 [SF-1]) and NEVER restarts on a config change [SF-2].
|
||||||
|
package felhomsshd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ConfDir is the dedicated config tree (host-install creates it; the agent renders the config).
|
||||||
|
ConfDir = "/etc/felhom-sshd"
|
||||||
|
// ConfPath is the rendered sshd config (referenced by the static unit's ExecStart/ExecReload).
|
||||||
|
ConfPath = ConfDir + "/sshd_config"
|
||||||
|
// HostKeyPath is the dedicated ed25519 host key (host-install generates it; stable across reloads).
|
||||||
|
HostKeyPath = ConfDir + "/ssh_host_ed25519_key"
|
||||||
|
// AuthKeysDir holds per-user authorized_keys OUTSIDE ~/.ssh, so the customer's sshd (which reads
|
||||||
|
// ~/.ssh/authorized_keys) structurally cannot honour the operator key [SF-3/§7].
|
||||||
|
AuthKeysDir = ConfDir + "/authorized_keys"
|
||||||
|
// PortFile persists the claimed port (idempotent re-pick).
|
||||||
|
PortFile = ConfDir + "/port"
|
||||||
|
// PidFile is the instance pidfile (NOT a RuntimeDirectory — that is the G1 incident cause).
|
||||||
|
PidFile = "/run/felhom-sshd.pid"
|
||||||
|
// Unit is the systemd unit name.
|
||||||
|
Unit = "felhom-sshd"
|
||||||
|
// OperatorUser is the default operator login (scoped sudo; key in AuthKeysDir only).
|
||||||
|
OperatorUser = "felhom-op"
|
||||||
|
)
|
||||||
|
|
||||||
|
// renderConfig builds the felhom-sshd config for a claimed port. Pure + deterministic (byte-stable
|
||||||
|
// for a given port → a stable conf-hash, no reload churn). The security posture is the SAFE template
|
||||||
|
// from the spike §3: key-only, dedicated host key + AuthorizedKeysFile, AllowUsers scoped to
|
||||||
|
// root+felhom-op, binds 0.0.0.0 (+ ::) so it never waits on a late interface, no RuntimeDirectory.
|
||||||
|
func renderConfig(port int) (string, error) {
|
||||||
|
if port < 1 || port > 65535 {
|
||||||
|
return "", fmt.Errorf("felhomsshd: port %d out of range", port)
|
||||||
|
}
|
||||||
|
if port == 22 {
|
||||||
|
// The whole point is coexistence — the dedicated instance must NEVER claim :22 [SF-4/trap 5].
|
||||||
|
return "", fmt.Errorf("felhomsshd: refusing to render on :22 (the stock/customer sshd port)")
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString("# felhom OOB sshd — agent-managed (H1); DO NOT EDIT\n")
|
||||||
|
fmt.Fprintf(&b, "Port %d\n", port)
|
||||||
|
b.WriteString("ListenAddress 0.0.0.0\n")
|
||||||
|
b.WriteString("ListenAddress ::\n")
|
||||||
|
fmt.Fprintf(&b, "HostKey %s\n", HostKeyPath)
|
||||||
|
fmt.Fprintf(&b, "PidFile %s\n", PidFile)
|
||||||
|
fmt.Fprintf(&b, "AuthorizedKeysFile %s/%%u\n", AuthKeysDir)
|
||||||
|
b.WriteString("PasswordAuthentication no\n")
|
||||||
|
b.WriteString("PermitRootLogin prohibit-password\n")
|
||||||
|
b.WriteString("PubkeyAuthentication yes\n")
|
||||||
|
b.WriteString("KbdInteractiveAuthentication no\n")
|
||||||
|
b.WriteString("UsePAM yes\n")
|
||||||
|
fmt.Fprintf(&b, "AllowUsers root %s\n", OperatorUser)
|
||||||
|
b.WriteString("X11Forwarding no\n")
|
||||||
|
b.WriteString("Subsystem sftp internal-sftp\n")
|
||||||
|
return b.String(), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package felhomsshd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRenderConfig_SafeTemplateAndByteStable(t *testing.T) {
|
||||||
|
c, err := renderConfig(8822)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, must := range []string{
|
||||||
|
"Port 8822\n",
|
||||||
|
"AuthorizedKeysFile /etc/felhom-sshd/authorized_keys/%u\n",
|
||||||
|
"HostKey /etc/felhom-sshd/ssh_host_ed25519_key\n",
|
||||||
|
"PasswordAuthentication no\n",
|
||||||
|
"PermitRootLogin prohibit-password\n",
|
||||||
|
"AllowUsers root felhom-op\n",
|
||||||
|
"PidFile /run/felhom-sshd.pid\n",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(c, must) {
|
||||||
|
t.Errorf("config missing %q:\n%s", must, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// [SF-1] the incident cause must NEVER appear.
|
||||||
|
if strings.Contains(c, "RuntimeDirectory") {
|
||||||
|
t.Fatal("config/unit must never mention RuntimeDirectory")
|
||||||
|
}
|
||||||
|
// byte-stable for a given port (conf-hash stability → no reload churn).
|
||||||
|
c2, _ := renderConfig(8822)
|
||||||
|
if c != c2 {
|
||||||
|
t.Fatal("renderConfig not deterministic")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderConfig_RefusesPort22AndOutOfRange(t *testing.T) {
|
||||||
|
if _, err := renderConfig(22); err == nil {
|
||||||
|
t.Fatal("renderConfig(22) must be refused — never claim the stock/customer port")
|
||||||
|
}
|
||||||
|
if _, err := renderConfig(0); err == nil {
|
||||||
|
t.Fatal("port 0 accepted")
|
||||||
|
}
|
||||||
|
if _, err := renderConfig(70000); err == nil {
|
||||||
|
t.Fatal("out-of-range port accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// claim harness: a fake free-set + an in-memory persisted port.
|
||||||
|
func claimHarness(free map[int]bool, persisted int) (portProbe, func() (int, bool), func(int) error, *int) {
|
||||||
|
stored := persisted
|
||||||
|
isFree := func(p int) bool { return free[p] }
|
||||||
|
read := func() (int, bool) {
|
||||||
|
if stored == 0 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return stored, true
|
||||||
|
}
|
||||||
|
write := func(p int) error { stored = p; return nil }
|
||||||
|
return isFree, read, write, &stored
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClaimPort_CleanContentionIdempotentExhaustion(t *testing.T) {
|
||||||
|
cands := []int{8822, 2222, 8022, 62222}
|
||||||
|
|
||||||
|
// clean → first candidate
|
||||||
|
isFree, read, write, stored := claimHarness(map[int]bool{8822: true, 2222: true, 8022: true, 62222: true}, 0)
|
||||||
|
if p, err := claimPort(cands, isFree, read, write); err != nil || p != 8822 {
|
||||||
|
t.Fatalf("clean claim = %d / %v, want 8822", p, err)
|
||||||
|
}
|
||||||
|
if *stored != 8822 {
|
||||||
|
t.Fatalf("clean claim not persisted, stored=%d", *stored)
|
||||||
|
}
|
||||||
|
|
||||||
|
// contention: 8822 busy → 2222
|
||||||
|
isFree, read, write, _ = claimHarness(map[int]bool{8822: false, 2222: true, 8022: true, 62222: true}, 0)
|
||||||
|
if p, err := claimPort(cands, isFree, read, write); err != nil || p != 2222 {
|
||||||
|
t.Fatalf("contention claim = %d / %v, want 2222", p, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// idempotent: persisted 2222 still free (even though 8822 is now free) → keep 2222 (no thrash)
|
||||||
|
isFree, read, write, _ = claimHarness(map[int]bool{8822: true, 2222: true, 8022: true, 62222: true}, 2222)
|
||||||
|
if p, err := claimPort(cands, isFree, read, write); err != nil || p != 2222 {
|
||||||
|
t.Fatalf("idempotent claim = %d / %v, want 2222 (kept)", p, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// exhaustion: all busy → LOUD error, no fallback
|
||||||
|
isFree, read, write, stored = claimHarness(map[int]bool{8822: false, 2222: false, 8022: false, 62222: false}, 0)
|
||||||
|
p, err := claimPort(cands, isFree, read, write)
|
||||||
|
if err != ErrPortsExhausted {
|
||||||
|
t.Fatalf("exhaustion must return ErrPortsExhausted, got %d / %v", p, err)
|
||||||
|
}
|
||||||
|
if p == 22 || p != 0 {
|
||||||
|
t.Fatalf("exhaustion must NOT yield a port (esp. :22), got %d", p)
|
||||||
|
}
|
||||||
|
if *stored != 0 {
|
||||||
|
t.Fatalf("exhaustion must persist nothing, stored=%d", *stored)
|
||||||
|
}
|
||||||
|
|
||||||
|
// persisted port that is now BUSY → re-claim a fresh free one (not the stale persisted)
|
||||||
|
isFree, read, write, _ = claimHarness(map[int]bool{8822: false, 2222: true, 8022: true, 62222: true}, 8822)
|
||||||
|
if p, err := claimPort(cands, isFree, read, write); err != nil || p != 2222 {
|
||||||
|
t.Fatalf("stale-persisted re-claim = %d / %v, want 2222", p, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
package felhomsshd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HealMarkerPath records the last felhom-sshd auto-heal (RFC3339), so the heartbeat surfaces a
|
||||||
|
// recurring failure to the operator (the mgmt_plane pattern, scoped to the OOB daemon).
|
||||||
|
const HealMarkerPath = "/run/felhom-sshd.healed"
|
||||||
|
|
||||||
|
// HealAndCheck restores felhom-sshd if it is down AND its config is valid — ONE restart per cooldown
|
||||||
|
// [SF-5 / spike §6], so a persistently-broken instance is reported (Status), not restart-stormed. It
|
||||||
|
// NEVER restarts onto a broken config [trap 8]: if `sshd -t` fails, it hands off (Status reports
|
||||||
|
// config_invalid; the hub warns). A healthy/active instance is untouched.
|
||||||
|
func (m *Manager) HealAndCheck(ctx context.Context, port int) {
|
||||||
|
if m.isActive(ctx) {
|
||||||
|
return // healthy — nothing to heal
|
||||||
|
}
|
||||||
|
// Down. Only restart if the config is VALID — never convert degraded into dead.
|
||||||
|
if _, errOut, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil {
|
||||||
|
m.logger.Warn("felhomsshd: down AND config invalid — NOT restarting (report-only)",
|
||||||
|
"stderr", strings.TrimSpace(string(errOut)))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Cooldown: at most one deliberate restart per window.
|
||||||
|
now := m.now()
|
||||||
|
if !m.lastRestartAt.IsZero() && now.Sub(m.lastRestartAt) < restartCooldown {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.lastRestartAt = now
|
||||||
|
// reset-failed BEFORE restart [SF-5]: a start-limit lockout otherwise refuses the restart.
|
||||||
|
if m.isFailed(ctx) {
|
||||||
|
_, _, _ = m.runner.Run(ctx, "systemctl", "reset-failed", Unit)
|
||||||
|
}
|
||||||
|
if err := m.systemctl(ctx, "restart"); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// record the heal (best-effort; a failed marker write never fails the heal)
|
||||||
|
_ = os.WriteFile(HealMarkerPath, []byte(now.UTC().Format(time.RFC3339)+"\n"), 0o644)
|
||||||
|
m.logger.Warn("felhomsshd: was down with a valid config — restarted (heal)", "port", port)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status builds the OOB heartbeat stanza (Part 4). Read-only. Discovers the effective port(s) via
|
||||||
|
// `sshd -T` (authoritative — catches a non-default/multi-Port config), dials the port locally to
|
||||||
|
// prove reachability, reads the wg-felhom handshake age, and reflects operator-peer/key config from
|
||||||
|
// the desired-state block. NEVER `wg show dump` (the S1 ban) — `latest-handshakes` only.
|
||||||
|
func (m *Manager) Status(ctx context.Context, block *hub.WireWireguard) *hub.OOBStatus {
|
||||||
|
st := &hub.OOBStatus{
|
||||||
|
FelhomSshdActive: m.isActive(ctx),
|
||||||
|
FelhomSshdPort: m.port,
|
||||||
|
}
|
||||||
|
// Authoritative port(s) from `sshd -T` (may differ from m.port if the config was hand-edited).
|
||||||
|
if ports := m.sshdEffectivePorts(ctx); len(ports) > 0 {
|
||||||
|
st.FelhomSshdPort = ports[0]
|
||||||
|
}
|
||||||
|
// config validity
|
||||||
|
if _, _, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil {
|
||||||
|
st.ConfigInvalid = true
|
||||||
|
}
|
||||||
|
// local reachability (a TCP dial to the OOB port)
|
||||||
|
if st.FelhomSshdPort > 0 {
|
||||||
|
st.Reachable = dialLocal(ctx, st.FelhomSshdPort)
|
||||||
|
}
|
||||||
|
// wg-felhom handshake age (the OOB path rides the tunnel)
|
||||||
|
if age, ok := m.wgHandshakeAge(ctx); ok {
|
||||||
|
st.WGHandshakeAgeS = &age
|
||||||
|
}
|
||||||
|
// desired-state config reflection
|
||||||
|
if block != nil {
|
||||||
|
st.OperatorPeerConfigured = block.OOBPeerIP != ""
|
||||||
|
st.OperatorKeyConfigured = strings.TrimSpace(block.OOBOperatorSSHKey) != ""
|
||||||
|
}
|
||||||
|
// last auto-heal
|
||||||
|
if raw, err := os.ReadFile(HealMarkerPath); err == nil {
|
||||||
|
st.HealedAt = strings.TrimSpace(string(raw))
|
||||||
|
}
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
|
||||||
|
// sshdEffectivePorts parses `sshd -T -f <conf>` for the effective Port line(s) (handles multi-Port).
|
||||||
|
func (m *Manager) sshdEffectivePorts(ctx context.Context) []int {
|
||||||
|
out, _, err := m.runner.Run(ctx, "sshd", "-T", "-f", ConfPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var ports []int
|
||||||
|
for _, line := range strings.Split(string(out), "\n") {
|
||||||
|
f := strings.Fields(strings.ToLower(line))
|
||||||
|
if len(f) == 2 && f[0] == "port" {
|
||||||
|
if p, err := strconv.Atoi(f[1]); err == nil && p > 0 && p <= 65535 {
|
||||||
|
ports = append(ports, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ports
|
||||||
|
}
|
||||||
|
|
||||||
|
// wgHandshakeAge reads wg-felhom's latest-handshake age in seconds (latest-handshakes ONLY — the
|
||||||
|
// dump ban). ok=false when the tunnel is down or unreadable.
|
||||||
|
func (m *Manager) wgHandshakeAge(ctx context.Context) (int64, bool) {
|
||||||
|
out, _, err := m.runner.Run(ctx, "wg", "show", "wg-felhom", "latest-handshakes")
|
||||||
|
if err != nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||||
|
f := strings.Fields(line)
|
||||||
|
if len(f) != 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
epoch, err := strconv.ParseInt(f[1], 10, 64)
|
||||||
|
if err != nil || epoch <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
age := m.now().Unix() - epoch
|
||||||
|
if age < 0 {
|
||||||
|
age = 0
|
||||||
|
}
|
||||||
|
return age, true
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// dialLocal reports whether a TCP connect to 127.0.0.1:port succeeds within a short timeout.
|
||||||
|
func dialLocal(ctx context.Context, port int) bool {
|
||||||
|
d := net.Dialer{Timeout: 2 * time.Second}
|
||||||
|
conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_ = conn.Close()
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package felhomsshd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Loop drives the felhom-sshd Manager on its own cadence (the wgtunnel/lanresolver shape) and
|
||||||
|
// consumes the hub desired-state's wireguard block via the desired.Syncer raw-consumer seam (for
|
||||||
|
// oob_peer_ip → the belt, and oob_operator_ssh_key → authorized_keys). Each tick: claim/render/reload
|
||||||
|
// the instance, sync the belt sets, and run the health/heal check.
|
||||||
|
type Loop struct {
|
||||||
|
mgr *Manager
|
||||||
|
belt *Belt // Part 3 (nil-safe: no belt sync when unset)
|
||||||
|
interval time.Duration
|
||||||
|
logger *slog.Logger
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
block *hub.WireWireguard
|
||||||
|
|
||||||
|
nudge chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewLoop builds the loop. interval defaults to 60s. belt may be nil (belt sync skipped).
|
||||||
|
func NewLoop(mgr *Manager, belt *Belt, interval time.Duration, logger *slog.Logger) *Loop {
|
||||||
|
if interval <= 0 {
|
||||||
|
interval = 60 * time.Second
|
||||||
|
}
|
||||||
|
if logger == nil {
|
||||||
|
logger = slog.Default()
|
||||||
|
}
|
||||||
|
return &Loop{mgr: mgr, belt: belt, interval: interval, logger: logger, nudge: make(chan struct{}, 1)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// OnDesiredState implements desired.RawConsumer: store the latest wireguard block and nudge.
|
||||||
|
func (l *Loop) OnDesiredState(_ context.Context, resp *hub.DesiredStateResponse) {
|
||||||
|
if resp == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
l.mu.Lock()
|
||||||
|
l.block = resp.DesiredState.Wireguard
|
||||||
|
l.mu.Unlock()
|
||||||
|
select {
|
||||||
|
case l.nudge <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *Loop) snapshot() *hub.WireWireguard {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
return l.block
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run reconciles immediately, then on every tick or desired-state nudge, until ctx is cancelled.
|
||||||
|
func (l *Loop) Run(ctx context.Context) error {
|
||||||
|
l.reconcile(ctx)
|
||||||
|
t := time.NewTicker(l.interval)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-t.C:
|
||||||
|
case <-l.nudge:
|
||||||
|
}
|
||||||
|
l.reconcile(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// reconcile runs one full pass: instance apply → belt sync → health/heal.
|
||||||
|
func (l *Loop) reconcile(ctx context.Context) {
|
||||||
|
block := l.snapshot()
|
||||||
|
port, err := l.mgr.Apply(ctx, block)
|
||||||
|
if err != nil {
|
||||||
|
return // Apply logged; a claim/exhaustion or install error — retry next tick
|
||||||
|
}
|
||||||
|
if l.belt != nil {
|
||||||
|
l.belt.Sync(ctx, port, oobPeerIP(block))
|
||||||
|
}
|
||||||
|
l.mgr.HealAndCheck(ctx, port)
|
||||||
|
}
|
||||||
|
|
||||||
|
// oobPeerIP extracts the operator /32 source (bare IP) from the block, or "" when OOB is off.
|
||||||
|
func oobPeerIP(block *hub.WireWireguard) string {
|
||||||
|
if block == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return block.OOBPeerIP
|
||||||
|
}
|
||||||
|
|
||||||
|
// OOBStatus implements the hub collector's reporter seam (Part 4).
|
||||||
|
func (l *Loop) OOBStatus(ctx context.Context) *hub.OOBStatus {
|
||||||
|
return l.mgr.Status(ctx, l.snapshot())
|
||||||
|
}
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
package felhomsshd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AuthKeysUserPath is felhom-op's authorized_keys (the default operator identity — key OUTSIDE
|
||||||
|
// ~/.ssh so the customer's sshd never honours it [SF-3]). FIXED (sudoers install target).
|
||||||
|
const AuthKeysUserPath = AuthKeysDir + "/" + OperatorUser
|
||||||
|
|
||||||
|
const stagedConfName = "sshd_config"
|
||||||
|
|
||||||
|
// Manager renders + applies the felhom-sshd config and drives the unit through the narrow-sudoers
|
||||||
|
// runner. Config changes go write→`sshd -t`→`reload` (NEVER restart-on-change [SF-2]); a deliberate
|
||||||
|
// restart is `reset-failed`-then-`restart` [SF-5], rate-limited by a cooldown (no flap).
|
||||||
|
type Manager struct {
|
||||||
|
runner proxmox.Runner
|
||||||
|
stateDir string
|
||||||
|
logger *slog.Logger
|
||||||
|
|
||||||
|
// injectable seams (tests)
|
||||||
|
isActive func(ctx context.Context) bool
|
||||||
|
isFailed func(ctx context.Context) bool
|
||||||
|
isFree portProbe
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
port int // the claimed port (0 until Apply claims it)
|
||||||
|
hash string // sha256 of the last-applied config (steady-state zero-exec gate)
|
||||||
|
akHash string // sha256 of the last-applied felhom-op authorized_keys
|
||||||
|
lastRestartAt time.Time // heal cooldown (Part 4)
|
||||||
|
}
|
||||||
|
|
||||||
|
// restartCooldown bounds deliberate felhom-sshd restarts (heal path) — one attempt per window, so a
|
||||||
|
// persistently-broken instance is reported, not restart-stormed (spike §6 / notification-cooldown shape).
|
||||||
|
const restartCooldown = 10 * time.Minute
|
||||||
|
|
||||||
|
// NewManager builds a Manager. stateDir is the agent state dir (staged config lives under it).
|
||||||
|
func NewManager(runner proxmox.Runner, stateDir string, logger *slog.Logger) *Manager {
|
||||||
|
if logger == nil {
|
||||||
|
logger = slog.Default()
|
||||||
|
}
|
||||||
|
return &Manager{
|
||||||
|
runner: runner,
|
||||||
|
stateDir: stateDir,
|
||||||
|
logger: logger,
|
||||||
|
isActive: func(ctx context.Context) bool {
|
||||||
|
out, _ := exec.CommandContext(ctx, "systemctl", "is-active", Unit).Output()
|
||||||
|
return strings.TrimSpace(string(out)) == "active"
|
||||||
|
},
|
||||||
|
isFailed: func(ctx context.Context) bool {
|
||||||
|
out, _ := exec.CommandContext(ctx, "systemctl", "is-failed", Unit).Output()
|
||||||
|
return strings.TrimSpace(string(out)) == "failed"
|
||||||
|
},
|
||||||
|
isFree: probeFree,
|
||||||
|
now: time.Now,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Port returns the claimed port (0 before the first successful Apply).
|
||||||
|
func (m *Manager) Port() int { return m.port }
|
||||||
|
|
||||||
|
func (m *Manager) sshdDir() string { return filepath.Join(m.stateDir, "felhom-sshd") }
|
||||||
|
func (m *Manager) stagedConfPath() string { return filepath.Join(m.sshdDir(), stagedConfName) }
|
||||||
|
|
||||||
|
// Apply claims the port, renders the config, reconciles the running unit, and (when the desired-state
|
||||||
|
// block carries it) installs the operator's authorized_keys. Idempotent: no change → at most an
|
||||||
|
// is-active check. block may be nil (no desired-state yet) — the instance still runs; only the
|
||||||
|
// operator login/belt inputs are skipped. Returns the claimed port (0 on a claim/exhaustion error) —
|
||||||
|
// the caller (belt sync) needs it.
|
||||||
|
func (m *Manager) Apply(ctx context.Context, block *hub.WireWireguard) (int, error) {
|
||||||
|
port, err := claimPort(Candidates, m.isFree, readPortFile, writePortFile)
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Error("felhomsshd: port claim failed", "err", err)
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
m.port = port
|
||||||
|
|
||||||
|
// Operator authorized_keys (from the hub-driven block) — written to felhom-op's dedicated file,
|
||||||
|
// outside ~/.ssh [SF-3]. Independent of the config-reload path (a key change never reloads sshd).
|
||||||
|
if block != nil {
|
||||||
|
m.applyAuthorizedKeys(ctx, block.OOBOperatorSSHKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
conf, err := renderConfig(port)
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Error("felhomsshd: refusing to apply invalid config", "err", err)
|
||||||
|
return port, err
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256([]byte(conf))
|
||||||
|
hash := hex.EncodeToString(sum[:])
|
||||||
|
active := m.isActive(ctx)
|
||||||
|
|
||||||
|
if m.hash == hash && active {
|
||||||
|
return port, nil // steady state: zero execs
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stage → validate → install → reload/enable. Never restart on a config change [SF-2].
|
||||||
|
if err := os.MkdirAll(m.sshdDir(), 0o700); err != nil {
|
||||||
|
m.logger.Error("felhomsshd: state dir", "err", err)
|
||||||
|
return port, err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(m.stagedConfPath(), []byte(conf), 0o600); err != nil {
|
||||||
|
m.logger.Error("felhomsshd: staging config", "err", err)
|
||||||
|
return port, err
|
||||||
|
}
|
||||||
|
// Validate the STAGED config before it is installed — a bad render never reaches the live path.
|
||||||
|
if _, errOut, err := m.runner.Run(ctx, "sshd", "-t", "-f", m.stagedConfPath()); err != nil {
|
||||||
|
m.logger.Error("felhomsshd: staged config failed sshd -t — NOT installing", "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||||
|
return port, err
|
||||||
|
}
|
||||||
|
if _, errOut, err := m.runner.Run(ctx, "install", "-o", "root", "-g", "root", "-m", "0644", "--", m.stagedConfPath(), ConfPath); err != nil {
|
||||||
|
m.logger.Error("felhomsshd: config install failed", "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||||
|
return port, err
|
||||||
|
}
|
||||||
|
verb := "reload"
|
||||||
|
if !active {
|
||||||
|
verb = "enable" // first apply / down → enable --now brings it up
|
||||||
|
}
|
||||||
|
if err := m.systemctl(ctx, verb); err != nil {
|
||||||
|
return port, err
|
||||||
|
}
|
||||||
|
m.hash = hash
|
||||||
|
m.logger.Info("felhomsshd: config applied", "port", port, "action", verb)
|
||||||
|
return port, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyAuthorizedKeys installs (or clears) felhom-op's authorized_keys from the hub-delivered
|
||||||
|
// operator SSH key. Hash-gated (no churn); the key is public, never a secret. A write failure is
|
||||||
|
// logged, not fatal — the instance keeps running. NO sshd reload needed (sshd reads the file per-auth).
|
||||||
|
func (m *Manager) applyAuthorizedKeys(ctx context.Context, sshKey string) {
|
||||||
|
content := ""
|
||||||
|
if k := strings.TrimSpace(sshKey); k != "" {
|
||||||
|
content = k + "\n"
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256([]byte(content))
|
||||||
|
h := hex.EncodeToString(sum[:])
|
||||||
|
if h == m.akHash {
|
||||||
|
return // unchanged
|
||||||
|
}
|
||||||
|
staged := filepath.Join(m.sshdDir(), "authorized_keys."+OperatorUser)
|
||||||
|
if err := os.MkdirAll(m.sshdDir(), 0o700); err != nil {
|
||||||
|
m.logger.Error("felhomsshd: state dir for authorized_keys", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(staged, []byte(content), 0o600); err != nil {
|
||||||
|
m.logger.Error("felhomsshd: staging authorized_keys", "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, errOut, err := m.runner.Run(ctx, "install", "-o", "root", "-g", "root", "-m", "0644", "--", staged, AuthKeysUserPath); err != nil {
|
||||||
|
m.logger.Error("felhomsshd: authorized_keys install failed", "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
m.akHash = h
|
||||||
|
m.logger.Info("felhomsshd: operator authorized_keys updated", "user", OperatorUser, "present", content != "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// systemctl runs the reconcile verbs through the narrow runner. `enable` = `enable --now`; `reload`
|
||||||
|
// HUPs (config change, running instance survives a bad reload via the unit's ExecReload sshd -t gate).
|
||||||
|
func (m *Manager) systemctl(ctx context.Context, verb string) error {
|
||||||
|
var args []string
|
||||||
|
switch verb {
|
||||||
|
case "enable":
|
||||||
|
args = []string{"enable", "--now", Unit}
|
||||||
|
case "reload", "restart":
|
||||||
|
args = []string{verb, Unit}
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, errOut, err := m.runner.Run(ctx, "systemctl", args...); err != nil {
|
||||||
|
m.logger.Error("felhomsshd: systemctl "+verb+" failed", "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// probeFree is the production port probe: nothing LISTENing (ss) AND a real bind succeeds (a bind
|
||||||
|
// that succeeds-then-closes proves the port is actually claimable, not just ss-silent).
|
||||||
|
func probeFree(port int) bool {
|
||||||
|
p := strconv.Itoa(port)
|
||||||
|
out, err := exec.Command("ss", "-Htln", "sport = :"+p).Output()
|
||||||
|
if err == nil && strings.TrimSpace(string(out)) != "" {
|
||||||
|
return false // something is listening
|
||||||
|
}
|
||||||
|
ln, err := net.Listen("tcp", "0.0.0.0:"+p)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_ = ln.Close()
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -70,6 +70,7 @@ type Collector struct {
|
|||||||
wg WireguardReporter // S3: offsite-tunnel status (nil → stanza omitted)
|
wg WireguardReporter // S3: offsite-tunnel status (nil → stanza omitted)
|
||||||
selfUpdate SelfUpdateReporter // D1: agent self-update pending status (nil → false)
|
selfUpdate SelfUpdateReporter // D1: agent self-update pending status (nil → false)
|
||||||
mgmtPlane MgmtPlaneReporter // G1: management-plane health (nil → stanza omitted)
|
mgmtPlane MgmtPlaneReporter // G1: management-plane health (nil → stanza omitted)
|
||||||
|
oob OOBReporter // H1: operator-access health (nil → stanza omitted)
|
||||||
hostID string
|
hostID string
|
||||||
agentVersion string
|
agentVersion string
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
@@ -154,6 +155,17 @@ func (c *Collector) SetMgmtPlaneReporter(m MgmtPlaneReporter) *Collector {
|
|||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OOBReporter is the H1 seam the felhom-sshd loop plugs into (nil → no oob stanza).
|
||||||
|
type OOBReporter interface {
|
||||||
|
OOBStatus(ctx context.Context) *OOBStatus
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetOOBReporter wires the operator-access health source (H1; nil-safe → stanza omitted).
|
||||||
|
func (c *Collector) SetOOBReporter(o OOBReporter) *Collector {
|
||||||
|
c.oob = o
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
// Collect builds the report. Best-effort liveness: a failed NodeStatus is a hard
|
// Collect builds the report. Best-effort liveness: a failed NodeStatus is a hard
|
||||||
// error (no useful report — the cycle skips the POST); a failed per-guest
|
// error (no useful report — the cycle skips the POST); a failed per-guest
|
||||||
// GuestConfig degrades that guest to status="unknown" without spec but still sends;
|
// GuestConfig degrades that guest to status="unknown" without spec but still sends;
|
||||||
@@ -200,6 +212,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
|
|||||||
if c.mgmtPlane != nil {
|
if c.mgmtPlane != nil {
|
||||||
report.MgmtPlane = c.mgmtPlane.MgmtPlaneStatus(ctx)
|
report.MgmtPlane = c.mgmtPlane.MgmtPlaneStatus(ctx)
|
||||||
}
|
}
|
||||||
|
// H1: operator-access (OOB) health (nil reporter = feature not wired → stanza omitted).
|
||||||
|
if c.oob != nil {
|
||||||
|
report.OOB = c.oob.OOBStatus(ctx)
|
||||||
|
}
|
||||||
return report, nil
|
return report, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+29
-3
@@ -76,6 +76,26 @@ type HostReport struct {
|
|||||||
// SelfUpdatePending precedent): stored opaquely hub-side, so these additive fields need no
|
// SelfUpdatePending precedent): stored opaquely hub-side, so these additive fields need no
|
||||||
// hub-schema change and are absent when the reporter is not wired.
|
// hub-schema change and are absent when the reporter is not wired.
|
||||||
MgmtPlane *MgmtPlaneStatus `json:"mgmt_plane,omitempty"`
|
MgmtPlane *MgmtPlaneStatus `json:"mgmt_plane,omitempty"`
|
||||||
|
|
||||||
|
// OOB is the operator-access health stanza (TASK H1). It answers the operator's question — "can I
|
||||||
|
// get into this box right now, and if not, why" — from the hub: felhom-sshd up + on which port,
|
||||||
|
// locally reachable, the tunnel handshake age (the OOB path rides wg-felhom), whether the operator
|
||||||
|
// peer/key is configured, and whether the config is invalid. The heartbeat reaches the hub over
|
||||||
|
// HTTPS even when felhom-sshd or the tunnel is DOWN (channel independence). `omitempty`: absent
|
||||||
|
// when the feature is not wired (pre-H1) — additive, no hub-schema change.
|
||||||
|
OOB *OOBStatus `json:"oob,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// OOBStatus is the per-heartbeat operator-access health (TASK H1). Carries no secret.
|
||||||
|
type OOBStatus struct {
|
||||||
|
FelhomSshdActive bool `json:"felhom_sshd_active"` // the felhom-sshd unit is active
|
||||||
|
FelhomSshdPort int `json:"felhom_sshd_port"` // the claimed OOB port (0 = unclaimed)
|
||||||
|
Reachable bool `json:"reachable"` // a local TCP dial to the OOB port succeeds
|
||||||
|
ConfigInvalid bool `json:"config_invalid"` // `sshd -t` on the OOB config fails
|
||||||
|
OperatorPeerConfigured bool `json:"operator_peer_configured"` // oob_peer_ip present in desired-state
|
||||||
|
OperatorKeyConfigured bool `json:"operator_key_configured"` // operator authorized_key installed
|
||||||
|
WGHandshakeAgeS *int64 `json:"wg_handshake_age_s,omitempty"` // wg-felhom last handshake age (nil = unknown)
|
||||||
|
HealedAt string `json:"healed_at,omitempty"` // last felhom-sshd auto-heal (RFC3339)
|
||||||
}
|
}
|
||||||
|
|
||||||
// MgmtPlaneStatus is the per-heartbeat management-plane health (TASK G1). Carries no secret.
|
// MgmtPlaneStatus is the per-heartbeat management-plane health (TASK G1). Carries no secret.
|
||||||
@@ -84,9 +104,9 @@ type HostReport struct {
|
|||||||
// hub raises a warning event on a PrivsepHealedAt it has not alerted on — a recurring auto-heal means
|
// hub raises a warning event on a PrivsepHealedAt it has not alerted on — a recurring auto-heal means
|
||||||
// a persistent clobber cause worth investigating before the box locks out.
|
// a persistent clobber cause worth investigating before the box locks out.
|
||||||
type MgmtPlaneStatus struct {
|
type MgmtPlaneStatus struct {
|
||||||
PrivsepDirOK bool `json:"privsep_dir_ok"` // /run/sshd exists (the KEXINIT-reset detector)
|
PrivsepDirOK bool `json:"privsep_dir_ok"` // /run/sshd exists (the KEXINIT-reset detector)
|
||||||
SshdReachable bool `json:"sshd_reachable"` // the stock sshd listener accepts TCP
|
SshdReachable bool `json:"sshd_reachable"` // the stock sshd listener accepts TCP
|
||||||
HealedRecently bool `json:"healed_recently"` // the watchdog heal-marker is present (this boot)
|
HealedRecently bool `json:"healed_recently"` // the watchdog heal-marker is present (this boot)
|
||||||
PrivsepHealedAt string `json:"privsep_healed_at,omitempty"` // marker timestamp; hub warns on a NEW value
|
PrivsepHealedAt string `json:"privsep_healed_at,omitempty"` // marker timestamp; hub warns on a NEW value
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -364,6 +384,12 @@ type WireWireguard struct {
|
|||||||
// set`) so it survives self-heal/restart/reboot ([OF-1]: a runtime widening was wiped by the
|
// set`) so it survives self-heal/restart/reboot ([OF-1]: a runtime widening was wiped by the
|
||||||
// agent's own self-heal). `omitempty`: absent = OOB off, byte-identical conf for pre-H1 hosts.
|
// agent's own self-heal). `omitempty`: absent = OOB off, byte-identical conf for pre-H1 hosts.
|
||||||
OOBPeerIP string `json:"oob_peer_ip,omitempty"`
|
OOBPeerIP string `json:"oob_peer_ip,omitempty"`
|
||||||
|
|
||||||
|
// OOBOperatorSSHKey is the operator's SSH PUBLIC key (an authorized_keys line, TASK H1). The agent
|
||||||
|
// writes it to felhom-sshd's dedicated AuthorizedKeysFile (/etc/felhom-sshd/authorized_keys/{root,
|
||||||
|
// felhom-op}) — OUTSIDE ~/.ssh, so the customer's sshd never honours it [SF-3]. Hub-driven so it
|
||||||
|
// rotates fleet-wide. `omitempty`: absent = no operator login installed. NOT a secret (public key).
|
||||||
|
OOBOperatorSSHKey string `json:"oob_operator_ssh_key,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// WireWireguardEndpoint is the endpoint half of the wireguard block.
|
// WireWireguardEndpoint is the endpoint half of the wireguard block.
|
||||||
|
|||||||
Reference in New Issue
Block a user