diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index ce4127b..6ddbfb7 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -31,6 +31,7 @@ import ( "gitea.dooplex.hu/admin/felhom-agent/internal/desired" "gitea.dooplex.hu/admin/felhom-agent/internal/dr" "gitea.dooplex.hu/admin/felhom-agent/internal/escrow" + "gitea.dooplex.hu/admin/felhom-agent/internal/felhomsshd" "gitea.dooplex.hu/admin/felhom-agent/internal/guesthook" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" "gitea.dooplex.hu/admin/felhom-agent/internal/lanresolver" @@ -706,6 +707,28 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { } } + // OOB operator access (H1): the dedicated felhom-sshd instance + port-adaptive belt + health. + // DEFAULT OFF (oob.enabled) until the operator endpoint + static belt table exist. Consumes the + // SAME wireguard desired-state block (oob_peer_ip → belt, oob_operator_ssh_key → authorized_keys). + oobServers := 0 + var oobLoop *felhomsshd.Loop + { + oc := cfg.OOB.WithDefaults() + if oc.Enabled { + oobMode := proxmox.RunnerMode(cfg.Privileged.Mode) + if oobMode == "" { + oobMode = proxmox.RunnerSudo + } + oobRunner := &proxmox.ExecRunner{Mode: oobMode, SudoPath: cfg.Privileged.SudoPath} + oobMgr := felhomsshd.NewManager(oobRunner, oc.StateDir, logger) + oobBelt := felhomsshd.NewBelt(oobRunner, logger) + oobLoop = felhomsshd.NewLoop(oobMgr, oobBelt, time.Duration(oc.IntervalSeconds)*time.Second, logger) + desiredSyncer.AddConsumer(oobLoop) // raw desired-state → oob_peer_ip + operator key + collector.SetOOBReporter(oobLoop) // heartbeat oob health stanza + logger.Info("felhomsshd (OOB): enabled", "interval_s", oc.IntervalSeconds, "state_dir", oc.StateDir) + } + } + // Run reconcile, the hub loop, the storage watchdog, the restore-test scheduler, the PBS // verify loop, (optionally) the local-API server, and (optionally) the LAN resolver loop // concurrently; any one returning ends the daemon (ctx cancel tears down the rest). @@ -768,6 +791,10 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { wgServers = 1 go func() { errc <- wgLoop.Run(ctx) }() } + if oobLoop != nil { + oobServers = 1 + go func() { errc <- oobLoop.Run(ctx) }() + } // TASK D1 Scenario D: if this process is a JUST-FLIPPED self-update (a pending marker names THIS // version), commit it after a clean dwell — but only now that core init is done (config parsed, @@ -777,8 +804,8 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { go selfUpdateMgr.MaybeCommit(ctx) err = <-errc - stop() // tear down the siblings on the first exit - for i := 0; i < 4+localServers+lanServers+wgServers; i++ { // wait for the other goroutines + stop() // tear down the siblings on the first exit + for i := 0; i < 4+localServers+lanServers+wgServers+oobServers; i++ { // wait for the other goroutines <-errc } if err != nil && err != context.Canceled { diff --git a/configs/felhom-agent.sudoers b/configs/felhom-agent.sudoers index 9119640..f6b7fb3 100644 --- a/configs/felhom-agent.sudoers +++ b/configs/felhom-agent.sudoers @@ -178,4 +178,35 @@ Cmnd_Alias FELHOM_SELFUPDATE = \ /usr/local/sbin/felhom-selfupdate-guarded commit, \ /usr/local/sbin/felhom-selfupdate-guarded rollback -felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE +# Dedicated OOB sshd (TASK H1). The agent manages felhom-sshd like wg-felhom/dnsmasq: it RENDERS the +# config (Port from its claim) + the operator's authorized_keys, validates with `sshd -t`, and reloads +# (never restart-on-change [SF-2]). Both install SOURCES are the agent-owned staged files under +# StateDir; both DESTINATIONS are FIXED. `sshd -t/-T` are the validate/discover reads. The +# systemctl verbs are SCOPED to felhom-sshd only. reset-failed precedes a deliberate restart [SF-5]. +# NOTHING here can touch the stock sshd, :22, or /etc/ssh. +Cmnd_Alias FELHOM_SSHD = \ + /usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/felhom-sshd/sshd_config /etc/felhom-sshd/sshd_config, \ + /usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/felhom-sshd/authorized_keys.felhom-op /etc/felhom-sshd/authorized_keys/felhom-op, \ + /usr/sbin/sshd -t -f /var/lib/felhom-agent/felhom-sshd/sshd_config, \ + /usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config, \ + /usr/sbin/sshd -T -f /etc/felhom-sshd/sshd_config, \ + /usr/bin/systemctl enable --now felhom-sshd, \ + /usr/bin/systemctl reload felhom-sshd, \ + /usr/bin/systemctl restart felhom-sshd, \ + /usr/bin/systemctl reset-failed felhom-sshd, \ + /usr/bin/wg show wg-felhom latest-handshakes + +# OOB nft belt (TASK H1). The STATIC table `inet felhom_oob` is installed once by host-install; the +# agent mutates ONLY its two SETS — @operator_ips (the operator /32) + @ssh_port (the claimed port). +# SET ELEMENTS ONLY [trap 4]: NO `nft add rule`, NO `nft -f`, NO `flush ruleset/table` — a rule grant +# would let the agent firewall anything. The agent fine-validates every element (netip / int range) +# before exec; the trailing wildcards are the coarse allowlist (values only). +Cmnd_Alias FELHOM_OOB = \ + /usr/sbin/nft list set inet felhom_oob operator_ips, \ + /usr/sbin/nft list set inet felhom_oob ssh_port, \ + /usr/sbin/nft flush set inet felhom_oob operator_ips, \ + /usr/sbin/nft flush set inet felhom_oob ssh_port, \ + /usr/sbin/nft add element inet felhom_oob operator_ips *, \ + /usr/sbin/nft add element inet felhom_oob ssh_port * + +felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB diff --git a/configs/felhom-sshd.service b/configs/felhom-sshd.service new file mode 100644 index 0000000..cf28b8f --- /dev/null +++ b/configs/felhom-sshd.service @@ -0,0 +1,28 @@ +# felhom-sshd.service (TASK H1) — install as /etc/systemd/system/felhom-sshd.service. +# +# The dedicated OOB sshd instance: a SECOND sshd on a claimed non-22 port, COEXISTING with the +# customer's/stock sshd on :22 (never touched). Config is AGENT-RENDERED at /etc/felhom-sshd/sshd_config +# (Port from the agent's claim); the agent reloads on change (never restart-on-change [SF-2]). +# +# CRITICAL [SF-1]: this unit MUST NOT declare `RuntimeDirectory=` — that directive (value `sshd`) +# removed the SHARED /run/sshd and broke the stock sshd live (SPIKE-felhom-sshd §8). /run/sshd is +# guaranteed by G1 (tmpfiles + watchdog); the ExecStartPre mkdir is a harmless extra guarantee. The +# host-install RuntimeDirectory guard will REJECT this unit if the directive is ever added. +[Unit] +Description=Felhom OOB sshd (dedicated instance, agent-managed; H1) +After=network.target + +[Service] +Type=notify +ExecStartPre=-/usr/bin/mkdir -p /run/sshd +ExecStartPre=/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config +ExecStart=/usr/sbin/sshd -D -f /etc/felhom-sshd/sshd_config +# Config-change path: validate then HUP — a bad reload is REFUSED and the running daemon SURVIVES +# [SF-2]. The agent always writes→sshd -t→reload; it never `restart`s on a config change. +ExecReload=/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config +ExecReload=/bin/kill -HUP $MAINPID +KillMode=process +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/internal/config/config.go b/internal/config/config.go index 90dd251..0f31095 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,10 +32,32 @@ type Config struct { LocalAPI LocalAPIConfig `json:"local_api"` LANResolver LANResolverConfig `json:"lan_resolver"` WGTunnel WGTunnelConfig `json:"wg_tunnel"` + OOB OOBConfig `json:"oob"` SelfUpdate SelfUpdateConfig `json:"selfupdate"` LogLevel string `json:"log_level"` // debug|info|warn|error (default info) } +// OOBConfig configures the dedicated felhom-sshd OOB access instance + belt (TASK H1). **Enabled +// DEFAULTS TO FALSE** — a rollout to a box without explicit oob.enabled=true is a no-op (no port +// claim, no config render, no belt mutation, no oob report stanza) until the operator endpoint + +// static belt table exist. +type OOBConfig struct { + Enabled bool `json:"enabled"` + IntervalSeconds int `json:"interval_seconds"` // reconcile cadence; default 60 + StateDir string `json:"state_dir"` // staged config/authkeys under /felhom-sshd/; default /var/lib/felhom-agent +} + +// WithDefaults fills the OOB reconcile cadence + state dir. +func (o OOBConfig) WithDefaults() OOBConfig { + if o.IntervalSeconds == 0 { + o.IntervalSeconds = 60 + } + if o.StateDir == "" { + o.StateDir = "/var/lib/felhom-agent" + } + return o +} + // SelfUpdateConfig configures the operator-signed agent self-update (TASK D1). The artifact HOST // is operator-controlled config; the artifact INTEGRITY comes only from the sha256 pinned inside // the operator-signed op — the hub's Day-0 manifest plays no role here, and a compromised Gitea diff --git a/internal/felhomsshd/belt.go b/internal/felhomsshd/belt.go new file mode 100644 index 0000000..c61f945 --- /dev/null +++ b/internal/felhomsshd/belt.go @@ -0,0 +1,130 @@ +package felhomsshd + +import ( + "context" + "log/slog" + "net/netip" + "regexp" + "sort" + "strconv" + "strings" + + "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" +) + +// Belt keeps the STATIC nft table `inet felhom_oob` (installed by host-install) converged on the +// current OOB state by mutating only its SETS — @operator_ips (the operator /32 allowed to reach +// felhom-sshd over wg-felhom) and @ssh_port (felhom-sshd's claimed port). The agent NEVER adds/removes +// RULES [trap 4]: set-element mutation can't change rule semantics, so the narrow sudoers grant stays +// safe. Idempotent: reads the current elements and mutates only on a difference (no churn). +type Belt struct { + runner proxmox.Runner + logger *slog.Logger + + table string // "felhom_oob" +} + +// NewBelt builds a Belt over the narrow runner. table defaults to "felhom_oob". +func NewBelt(runner proxmox.Runner, logger *slog.Logger) *Belt { + if logger == nil { + logger = slog.Default() + } + return &Belt{runner: runner, logger: logger, table: "felhom_oob"} +} + +// Sync converges @operator_ips + @ssh_port on the desired values. operatorIP "" empties @operator_ips +// (belt drops all tunnel SSH — OOB off). port 0 empties @ssh_port. Every value is validated before it +// reaches nft (netip for the IP, int range for the port), so the coarse sudoers wildcard can never be +// abused. A missing table (host-install not run / pre-H1 box) degrades to a single logged warning. +func (b *Belt) Sync(ctx context.Context, port int, operatorIP string) { + // desired sets + wantOps := []string{} + if operatorIP != "" { + ip, err := netip.ParseAddr(operatorIP) + if err != nil || !ip.Is4() { + b.logger.Error("felhomsshd belt: operator ip invalid — refusing", "ip", operatorIP) + return + } + wantOps = []string{ip.String()} + } + wantPorts := []string{} + if port > 0 && port <= 65535 { + wantPorts = []string{strconv.Itoa(port)} + } + + b.syncSet(ctx, "operator_ips", wantOps) + b.syncSet(ctx, "ssh_port", wantPorts) +} + +var nftElemRe = regexp.MustCompile(`elements\s*=\s*\{([^}]*)\}`) + +// syncSet converges one named set on `want` (sorted, deduped). Reads current elements; if they match, +// ZERO nft mutations (the idempotency the scenario asserts). Otherwise flush + add the desired +// elements. A read failure (table/set absent) → one warning, no mutation. +func (b *Belt) syncSet(ctx context.Context, setName string, want []string) { + want = sortedUnique(want) + cur, ok := b.readSet(ctx, setName) + if !ok { + b.logger.Warn("felhomsshd belt: set unreadable (table not installed?) — skipping", "set", setName) + return + } + if equalStringSlices(cur, want) { + return // steady state: no mutation + } + if _, errOut, err := b.runner.Run(ctx, "nft", "flush", "set", "inet", b.table, setName); err != nil { + b.logger.Error("felhomsshd belt: flush failed", "set", setName, "err", err, "stderr", strings.TrimSpace(string(errOut))) + return + } + for _, e := range want { + if _, errOut, err := b.runner.Run(ctx, "nft", "add", "element", "inet", b.table, setName, "{ "+e+" }"); err != nil { + b.logger.Error("felhomsshd belt: add element failed", "set", setName, "elem", e, "err", err, "stderr", strings.TrimSpace(string(errOut))) + return + } + } + b.logger.Info("felhomsshd belt: set synced", "set", setName, "elements", strings.Join(want, ",")) +} + +// readSet returns the current elements of a named set (sorted), or ok=false if the set can't be read. +func (b *Belt) readSet(ctx context.Context, setName string) ([]string, bool) { + out, _, err := b.runner.Run(ctx, "nft", "list", "set", "inet", b.table, setName) + if err != nil { + return nil, false + } + m := nftElemRe.FindSubmatch(out) + if m == nil { + return []string{}, true // set exists but empty (no "elements = {}" block) + } + var elems []string + for _, f := range strings.Split(string(m[1]), ",") { + f = strings.TrimSpace(f) + if f != "" { + elems = append(elems, f) + } + } + return sortedUnique(elems), true +} + +func sortedUnique(in []string) []string { + seen := map[string]bool{} + out := make([]string, 0, len(in)) + for _, s := range in { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.Strings(out) + return out +} + +func equalStringSlices(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/felhomsshd/belt_health_test.go b/internal/felhomsshd/belt_health_test.go new file mode 100644 index 0000000..031b17d --- /dev/null +++ b/internal/felhomsshd/belt_health_test.go @@ -0,0 +1,178 @@ +package felhomsshd + +import ( + "context" + "io" + "strings" + "sync" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// scriptRunner records every exec and returns scripted stdout per "name+first-arg" key; scripted +// errors per the same key. Everything else returns empty success. +type scriptRunner struct { + mu sync.Mutex + calls [][]string + stdout map[string]string + errs map[string]error +} + +func newScriptRunner() *scriptRunner { + return &scriptRunner{stdout: map[string]string{}, errs: map[string]error{}} +} + +func key(name string, args ...string) string { + // `nft list set inet felhom_oob ` → key by the set name (last arg), so the two set reads + // are distinguishable. Everything else keys by first arg. + if name == "nft" && len(args) > 0 && args[0] == "list" { + return "nft list " + args[len(args)-1] + } + if len(args) > 0 { + return name + " " + args[0] + } + return name +} + +func (r *scriptRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) { + r.mu.Lock() + r.calls = append(r.calls, append([]string{name}, args...)) + k := key(name, args...) + out := r.stdout[k] + err := r.errs[k] + r.mu.Unlock() + if err != nil { + return nil, []byte("scripted failure"), err + } + return []byte(out), nil, nil +} + +func (r *scriptRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) { + return r.Run(ctx, name, args...) +} + +func (r *scriptRunner) count(name, firstArg string) int { + r.mu.Lock() + defer r.mu.Unlock() + n := 0 + for _, c := range r.calls { + if c[0] == name && len(c) > 1 && c[1] == firstArg { + n++ + } + } + return n +} + +func (r *scriptRunner) sawRuleMutation() bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, c := range r.calls { + // trap 4: the agent must NEVER run `nft add rule` / `nft -f` / `nft flush ruleset|table`. + if c[0] == "nft" && len(c) > 1 { + if c[1] == "-f" || (len(c) > 2 && c[1] == "add" && c[2] == "rule") { + return true + } + if c[1] == "flush" && len(c) > 2 && (c[2] == "ruleset" || c[2] == "table") { + return true + } + } + } + return false +} + +func TestBelt_SyncMutatesThenIdempotent(t *testing.T) { + r := newScriptRunner() + // first read: both sets empty (no "elements" block) + r.stdout["nft list operator_ips"] = "set operator_ips {\n\ttype ipv4_addr\n}" + r.stdout["nft list ssh_port"] = "set ssh_port {\n\ttype inet_service\n}" + b := NewBelt(r, nil) + b.Sync(context.Background(), 8822, "10.77.0.250") + + // mutations happened: flush + add for BOTH sets + if r.count("nft", "flush") < 2 || r.count("nft", "add") < 2 { + t.Fatalf("first sync must flush+add both sets; flushes=%d adds=%d", r.count("nft", "flush"), r.count("nft", "add")) + } + if r.sawRuleMutation() { + t.Fatal("belt must NEVER mutate rules — only set elements (trap 4)") + } + + // second sync: the reads now return the desired elements → ZERO mutations (idempotent) + r2 := newScriptRunner() + r2.stdout["nft list operator_ips"] = "elements = { 10.77.0.250 }" + r2.stdout["nft list ssh_port"] = "elements = { 8822 }" + b2 := NewBelt(r2, nil) + b2.Sync(context.Background(), 8822, "10.77.0.250") + if r2.count("nft", "flush") != 0 || r2.count("nft", "add") != 0 { + t.Fatalf("idempotent sync must do ZERO mutations; flushes=%d adds=%d", r2.count("nft", "flush"), r2.count("nft", "add")) + } +} + +func TestBelt_OperatorEmptyEmptiesSet(t *testing.T) { + r := newScriptRunner() + r.stdout["nft list operator_ips"] = "elements = { 10.77.0.250 }" // currently has an operator IP + r.stdout["nft list ssh_port"] = "elements = { 8822 }" + b := NewBelt(r, nil) + b.Sync(context.Background(), 8822, "") // OOB off → operator set must be emptied + // operator_ips: current {250} != desired {} → flush, no add. ssh_port: {} vs {8822}... + if r.count("nft", "flush") < 1 { + t.Fatal("emptying the operator set must flush it") + } +} + +// health: inactive + INVALID config → NO restart (the red-proof); inactive + valid → restart. +func TestHeal_NoRestartOnInvalidConfig(t *testing.T) { + r := newScriptRunner() + r.errs["sshd -t"] = context.DeadlineExceeded // sshd -t FAILS (config invalid) + m := newTestManager(r, false) // inactive + m.HealAndCheck(context.Background(), 8822) + if r.count("systemctl", "restart") != 0 { + t.Fatal("a broken config must NOT trigger a restart (never degraded→dead)") + } +} + +func TestHeal_RestartsWhenDownWithValidConfigThenCooldown(t *testing.T) { + r := newScriptRunner() // sshd -t passes (no error) + m := newTestManager(r, false) + m.HealAndCheck(context.Background(), 8822) + if r.count("systemctl", "restart") != 1 { + t.Fatalf("down + valid config → exactly one restart, got %d", r.count("systemctl", "restart")) + } + // cooldown: a second call inside the window → still one restart + m.HealAndCheck(context.Background(), 8822) + if r.count("systemctl", "restart") != 1 { + t.Fatalf("restart cooldown violated: %d restarts", r.count("systemctl", "restart")) + } +} + +func TestStatus_ReflectsBlockAndPort(t *testing.T) { + r := newScriptRunner() + r.stdout["sshd -T"] = "port 8822\nsomethingelse yes\n" + m := newTestManager(r, true) // active + m.port = 8822 + st := m.Status(context.Background(), &hub.WireWireguard{OOBPeerIP: "10.77.0.250", OOBOperatorSSHKey: "ssh-ed25519 AAAA op"}) + if !st.FelhomSshdActive || st.FelhomSshdPort != 8822 { + t.Fatalf("status: %+v", st) + } + if !st.OperatorPeerConfigured || !st.OperatorKeyConfigured { + t.Fatalf("status must reflect operator peer + key configured: %+v", st) + } +} + +// newTestManager builds a Manager with injected active-state + a now clock; sshd -T/-t go through the +// runner. isFree unused here. +func newTestManager(r *scriptRunner, active bool) *Manager { + m := NewManager(r, "/tmp/felhomsshd-test", nil) + m.isActive = func(context.Context) bool { return active } + m.isFailed = func(context.Context) bool { return false } + m.now = func() time.Time { return time.Unix(1783270000, 0) } + return m +} + +func TestConfig_NoRuntimeDirectoryString(t *testing.T) { + c, _ := renderConfig(2222) + if strings.Contains(c, "RuntimeDirectory") { + t.Fatal("SF-1: config must never contain RuntimeDirectory") + } +} diff --git a/internal/felhomsshd/claim.go b/internal/felhomsshd/claim.go new file mode 100644 index 0000000..844010e --- /dev/null +++ b/internal/felhomsshd/claim.go @@ -0,0 +1,65 @@ +package felhomsshd + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +// Candidates is the ordered OOB-port candidate list (spike §2). First free wins; NEVER :22 or a +// random port. Package-var (not const) so tests can shrink it. +var Candidates = []int{8822, 2222, 8022, 62222} + +// ErrPortsExhausted is returned when every candidate is busy — a LOUD failure [SF-4/trap 6], never a +// silent fallback to :22 or a random high port. +var ErrPortsExhausted = fmt.Errorf("felhomsshd: all candidate OOB ports are busy — refusing to fall back to :22 or a random port") + +// portProbe reports whether a TCP port is free (nothing listening AND a real bind succeeds). Injected +// for tests; production impl = probeFree (ss + net.Listen). +type portProbe func(port int) bool + +// claimPort returns the OOB port, mirroring the spike's shell algorithm: +// - if a persisted port exists AND is still free → keep it (idempotent, no thrash), +// - else the FIRST free candidate → persist + return, +// - else ErrPortsExhausted (LOUD). +// +// persist writes PortFile; readPersisted reads it. isFree is the probe. All injected for tests. +func claimPort(candidates []int, isFree portProbe, readPersisted func() (int, bool), persist func(int) error) (int, error) { + if cur, ok := readPersisted(); ok && cur != 22 && isFree(cur) { + return cur, nil + } + for _, p := range candidates { + if p == 22 { + continue // defensive: never :22 + } + if isFree(p) { + if err := persist(p); err != nil { + return 0, fmt.Errorf("felhomsshd: persist claimed port %d: %w", p, err) + } + return p, nil + } + } + return 0, ErrPortsExhausted +} + +// readPortFile parses PortFile → (port, ok). A missing/garbage file → (0,false). +func readPortFile() (int, bool) { + raw, err := os.ReadFile(PortFile) + if err != nil { + return 0, false + } + p, err := strconv.Atoi(strings.TrimSpace(string(raw))) + if err != nil || p < 1 || p > 65535 { + return 0, false + } + return p, true +} + +// writePortFile persists the claimed port (0644 — a port is not a secret). +func writePortFile(port int) error { + if err := os.MkdirAll(ConfDir, 0o755); err != nil { + return err + } + return os.WriteFile(PortFile, []byte(strconv.Itoa(port)+"\n"), 0o644) +} diff --git a/internal/felhomsshd/config.go b/internal/felhomsshd/config.go new file mode 100644 index 0000000..3ee1fe2 --- /dev/null +++ b/internal/felhomsshd/config.go @@ -0,0 +1,65 @@ +// Package felhomsshd manages the dedicated OOB sshd instance (TASK H1). It is a SECOND sshd — +// separate port, config, host keys, AuthorizedKeysFile, and systemd unit — that COEXISTS with the +// customer's/stock sshd on :22 (never touched). Design + safety earned by +// SPIKE-felhom-sshd-2026-07-05 (§2 claim, §3 SAFE unit, §5 reload-not-restart, §7 AuthorizedKeysFile +// isolation) and SPIKE-oob-wg-operator-peer-2026-07-05 (the tunnel-only belt). +// +// The agent RENDERS the config (Port from the claim) and reloads on change — the wg-felhom pattern. +// It NEVER declares RuntimeDirectory= (G1 [SF-1]) and NEVER restarts on a config change [SF-2]. +package felhomsshd + +import ( + "fmt" + "strings" +) + +const ( + // ConfDir is the dedicated config tree (host-install creates it; the agent renders the config). + ConfDir = "/etc/felhom-sshd" + // ConfPath is the rendered sshd config (referenced by the static unit's ExecStart/ExecReload). + ConfPath = ConfDir + "/sshd_config" + // HostKeyPath is the dedicated ed25519 host key (host-install generates it; stable across reloads). + HostKeyPath = ConfDir + "/ssh_host_ed25519_key" + // AuthKeysDir holds per-user authorized_keys OUTSIDE ~/.ssh, so the customer's sshd (which reads + // ~/.ssh/authorized_keys) structurally cannot honour the operator key [SF-3/§7]. + AuthKeysDir = ConfDir + "/authorized_keys" + // PortFile persists the claimed port (idempotent re-pick). + PortFile = ConfDir + "/port" + // PidFile is the instance pidfile (NOT a RuntimeDirectory — that is the G1 incident cause). + PidFile = "/run/felhom-sshd.pid" + // Unit is the systemd unit name. + Unit = "felhom-sshd" + // OperatorUser is the default operator login (scoped sudo; key in AuthKeysDir only). + OperatorUser = "felhom-op" +) + +// renderConfig builds the felhom-sshd config for a claimed port. Pure + deterministic (byte-stable +// for a given port → a stable conf-hash, no reload churn). The security posture is the SAFE template +// from the spike §3: key-only, dedicated host key + AuthorizedKeysFile, AllowUsers scoped to +// root+felhom-op, binds 0.0.0.0 (+ ::) so it never waits on a late interface, no RuntimeDirectory. +func renderConfig(port int) (string, error) { + if port < 1 || port > 65535 { + return "", fmt.Errorf("felhomsshd: port %d out of range", port) + } + if port == 22 { + // The whole point is coexistence — the dedicated instance must NEVER claim :22 [SF-4/trap 5]. + return "", fmt.Errorf("felhomsshd: refusing to render on :22 (the stock/customer sshd port)") + } + var b strings.Builder + b.WriteString("# felhom OOB sshd — agent-managed (H1); DO NOT EDIT\n") + fmt.Fprintf(&b, "Port %d\n", port) + b.WriteString("ListenAddress 0.0.0.0\n") + b.WriteString("ListenAddress ::\n") + fmt.Fprintf(&b, "HostKey %s\n", HostKeyPath) + fmt.Fprintf(&b, "PidFile %s\n", PidFile) + fmt.Fprintf(&b, "AuthorizedKeysFile %s/%%u\n", AuthKeysDir) + b.WriteString("PasswordAuthentication no\n") + b.WriteString("PermitRootLogin prohibit-password\n") + b.WriteString("PubkeyAuthentication yes\n") + b.WriteString("KbdInteractiveAuthentication no\n") + b.WriteString("UsePAM yes\n") + fmt.Fprintf(&b, "AllowUsers root %s\n", OperatorUser) + b.WriteString("X11Forwarding no\n") + b.WriteString("Subsystem sftp internal-sftp\n") + return b.String(), nil +} diff --git a/internal/felhomsshd/felhomsshd_test.go b/internal/felhomsshd/felhomsshd_test.go new file mode 100644 index 0000000..2e0ac63 --- /dev/null +++ b/internal/felhomsshd/felhomsshd_test.go @@ -0,0 +1,105 @@ +package felhomsshd + +import ( + "strings" + "testing" +) + +func TestRenderConfig_SafeTemplateAndByteStable(t *testing.T) { + c, err := renderConfig(8822) + if err != nil { + t.Fatal(err) + } + for _, must := range []string{ + "Port 8822\n", + "AuthorizedKeysFile /etc/felhom-sshd/authorized_keys/%u\n", + "HostKey /etc/felhom-sshd/ssh_host_ed25519_key\n", + "PasswordAuthentication no\n", + "PermitRootLogin prohibit-password\n", + "AllowUsers root felhom-op\n", + "PidFile /run/felhom-sshd.pid\n", + } { + if !strings.Contains(c, must) { + t.Errorf("config missing %q:\n%s", must, c) + } + } + // [SF-1] the incident cause must NEVER appear. + if strings.Contains(c, "RuntimeDirectory") { + t.Fatal("config/unit must never mention RuntimeDirectory") + } + // byte-stable for a given port (conf-hash stability → no reload churn). + c2, _ := renderConfig(8822) + if c != c2 { + t.Fatal("renderConfig not deterministic") + } +} + +func TestRenderConfig_RefusesPort22AndOutOfRange(t *testing.T) { + if _, err := renderConfig(22); err == nil { + t.Fatal("renderConfig(22) must be refused — never claim the stock/customer port") + } + if _, err := renderConfig(0); err == nil { + t.Fatal("port 0 accepted") + } + if _, err := renderConfig(70000); err == nil { + t.Fatal("out-of-range port accepted") + } +} + +// claim harness: a fake free-set + an in-memory persisted port. +func claimHarness(free map[int]bool, persisted int) (portProbe, func() (int, bool), func(int) error, *int) { + stored := persisted + isFree := func(p int) bool { return free[p] } + read := func() (int, bool) { + if stored == 0 { + return 0, false + } + return stored, true + } + write := func(p int) error { stored = p; return nil } + return isFree, read, write, &stored +} + +func TestClaimPort_CleanContentionIdempotentExhaustion(t *testing.T) { + cands := []int{8822, 2222, 8022, 62222} + + // clean → first candidate + isFree, read, write, stored := claimHarness(map[int]bool{8822: true, 2222: true, 8022: true, 62222: true}, 0) + if p, err := claimPort(cands, isFree, read, write); err != nil || p != 8822 { + t.Fatalf("clean claim = %d / %v, want 8822", p, err) + } + if *stored != 8822 { + t.Fatalf("clean claim not persisted, stored=%d", *stored) + } + + // contention: 8822 busy → 2222 + isFree, read, write, _ = claimHarness(map[int]bool{8822: false, 2222: true, 8022: true, 62222: true}, 0) + if p, err := claimPort(cands, isFree, read, write); err != nil || p != 2222 { + t.Fatalf("contention claim = %d / %v, want 2222", p, err) + } + + // idempotent: persisted 2222 still free (even though 8822 is now free) → keep 2222 (no thrash) + isFree, read, write, _ = claimHarness(map[int]bool{8822: true, 2222: true, 8022: true, 62222: true}, 2222) + if p, err := claimPort(cands, isFree, read, write); err != nil || p != 2222 { + t.Fatalf("idempotent claim = %d / %v, want 2222 (kept)", p, err) + } + + // exhaustion: all busy → LOUD error, no fallback + isFree, read, write, stored = claimHarness(map[int]bool{8822: false, 2222: false, 8022: false, 62222: false}, 0) + p, err := claimPort(cands, isFree, read, write) + if err != ErrPortsExhausted { + t.Fatalf("exhaustion must return ErrPortsExhausted, got %d / %v", p, err) + } + if p == 22 || p != 0 { + t.Fatalf("exhaustion must NOT yield a port (esp. :22), got %d", p) + } + if *stored != 0 { + t.Fatalf("exhaustion must persist nothing, stored=%d", *stored) + } + + // persisted port that is now BUSY → re-claim a fresh free one (not the stale persisted) + isFree, read, write, _ = claimHarness(map[int]bool{8822: false, 2222: true, 8022: true, 62222: true}, 8822) + if p, err := claimPort(cands, isFree, read, write); err != nil || p != 2222 { + t.Fatalf("stale-persisted re-claim = %d / %v, want 2222", p, err) + } +} diff --git a/internal/felhomsshd/health.go b/internal/felhomsshd/health.go new file mode 100644 index 0000000..41d9e73 --- /dev/null +++ b/internal/felhomsshd/health.go @@ -0,0 +1,139 @@ +package felhomsshd + +import ( + "context" + "net" + "os" + "strconv" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// HealMarkerPath records the last felhom-sshd auto-heal (RFC3339), so the heartbeat surfaces a +// recurring failure to the operator (the mgmt_plane pattern, scoped to the OOB daemon). +const HealMarkerPath = "/run/felhom-sshd.healed" + +// HealAndCheck restores felhom-sshd if it is down AND its config is valid — ONE restart per cooldown +// [SF-5 / spike §6], so a persistently-broken instance is reported (Status), not restart-stormed. It +// NEVER restarts onto a broken config [trap 8]: if `sshd -t` fails, it hands off (Status reports +// config_invalid; the hub warns). A healthy/active instance is untouched. +func (m *Manager) HealAndCheck(ctx context.Context, port int) { + if m.isActive(ctx) { + return // healthy — nothing to heal + } + // Down. Only restart if the config is VALID — never convert degraded into dead. + if _, errOut, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil { + m.logger.Warn("felhomsshd: down AND config invalid — NOT restarting (report-only)", + "stderr", strings.TrimSpace(string(errOut))) + return + } + // Cooldown: at most one deliberate restart per window. + now := m.now() + if !m.lastRestartAt.IsZero() && now.Sub(m.lastRestartAt) < restartCooldown { + return + } + m.lastRestartAt = now + // reset-failed BEFORE restart [SF-5]: a start-limit lockout otherwise refuses the restart. + if m.isFailed(ctx) { + _, _, _ = m.runner.Run(ctx, "systemctl", "reset-failed", Unit) + } + if err := m.systemctl(ctx, "restart"); err != nil { + return + } + // record the heal (best-effort; a failed marker write never fails the heal) + _ = os.WriteFile(HealMarkerPath, []byte(now.UTC().Format(time.RFC3339)+"\n"), 0o644) + m.logger.Warn("felhomsshd: was down with a valid config — restarted (heal)", "port", port) +} + +// Status builds the OOB heartbeat stanza (Part 4). Read-only. Discovers the effective port(s) via +// `sshd -T` (authoritative — catches a non-default/multi-Port config), dials the port locally to +// prove reachability, reads the wg-felhom handshake age, and reflects operator-peer/key config from +// the desired-state block. NEVER `wg show dump` (the S1 ban) — `latest-handshakes` only. +func (m *Manager) Status(ctx context.Context, block *hub.WireWireguard) *hub.OOBStatus { + st := &hub.OOBStatus{ + FelhomSshdActive: m.isActive(ctx), + FelhomSshdPort: m.port, + } + // Authoritative port(s) from `sshd -T` (may differ from m.port if the config was hand-edited). + if ports := m.sshdEffectivePorts(ctx); len(ports) > 0 { + st.FelhomSshdPort = ports[0] + } + // config validity + if _, _, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil { + st.ConfigInvalid = true + } + // local reachability (a TCP dial to the OOB port) + if st.FelhomSshdPort > 0 { + st.Reachable = dialLocal(ctx, st.FelhomSshdPort) + } + // wg-felhom handshake age (the OOB path rides the tunnel) + if age, ok := m.wgHandshakeAge(ctx); ok { + st.WGHandshakeAgeS = &age + } + // desired-state config reflection + if block != nil { + st.OperatorPeerConfigured = block.OOBPeerIP != "" + st.OperatorKeyConfigured = strings.TrimSpace(block.OOBOperatorSSHKey) != "" + } + // last auto-heal + if raw, err := os.ReadFile(HealMarkerPath); err == nil { + st.HealedAt = strings.TrimSpace(string(raw)) + } + return st +} + +// sshdEffectivePorts parses `sshd -T -f ` for the effective Port line(s) (handles multi-Port). +func (m *Manager) sshdEffectivePorts(ctx context.Context) []int { + out, _, err := m.runner.Run(ctx, "sshd", "-T", "-f", ConfPath) + if err != nil { + return nil + } + var ports []int + for _, line := range strings.Split(string(out), "\n") { + f := strings.Fields(strings.ToLower(line)) + if len(f) == 2 && f[0] == "port" { + if p, err := strconv.Atoi(f[1]); err == nil && p > 0 && p <= 65535 { + ports = append(ports, p) + } + } + } + return ports +} + +// wgHandshakeAge reads wg-felhom's latest-handshake age in seconds (latest-handshakes ONLY — the +// dump ban). ok=false when the tunnel is down or unreadable. +func (m *Manager) wgHandshakeAge(ctx context.Context) (int64, bool) { + out, _, err := m.runner.Run(ctx, "wg", "show", "wg-felhom", "latest-handshakes") + if err != nil { + return 0, false + } + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + f := strings.Fields(line) + if len(f) != 2 { + continue + } + epoch, err := strconv.ParseInt(f[1], 10, 64) + if err != nil || epoch <= 0 { + continue + } + age := m.now().Unix() - epoch + if age < 0 { + age = 0 + } + return age, true + } + return 0, false +} + +// dialLocal reports whether a TCP connect to 127.0.0.1:port succeeds within a short timeout. +func dialLocal(ctx context.Context, port int) bool { + d := net.Dialer{Timeout: 2 * time.Second} + conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + if err != nil { + return false + } + _ = conn.Close() + return true +} diff --git a/internal/felhomsshd/loop.go b/internal/felhomsshd/loop.go new file mode 100644 index 0000000..6ec3bb1 --- /dev/null +++ b/internal/felhomsshd/loop.go @@ -0,0 +1,99 @@ +package felhomsshd + +import ( + "context" + "log/slog" + "sync" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// Loop drives the felhom-sshd Manager on its own cadence (the wgtunnel/lanresolver shape) and +// consumes the hub desired-state's wireguard block via the desired.Syncer raw-consumer seam (for +// oob_peer_ip → the belt, and oob_operator_ssh_key → authorized_keys). Each tick: claim/render/reload +// the instance, sync the belt sets, and run the health/heal check. +type Loop struct { + mgr *Manager + belt *Belt // Part 3 (nil-safe: no belt sync when unset) + interval time.Duration + logger *slog.Logger + + mu sync.Mutex + block *hub.WireWireguard + + nudge chan struct{} +} + +// NewLoop builds the loop. interval defaults to 60s. belt may be nil (belt sync skipped). +func NewLoop(mgr *Manager, belt *Belt, interval time.Duration, logger *slog.Logger) *Loop { + if interval <= 0 { + interval = 60 * time.Second + } + if logger == nil { + logger = slog.Default() + } + return &Loop{mgr: mgr, belt: belt, interval: interval, logger: logger, nudge: make(chan struct{}, 1)} +} + +// OnDesiredState implements desired.RawConsumer: store the latest wireguard block and nudge. +func (l *Loop) OnDesiredState(_ context.Context, resp *hub.DesiredStateResponse) { + if resp == nil { + return + } + l.mu.Lock() + l.block = resp.DesiredState.Wireguard + l.mu.Unlock() + select { + case l.nudge <- struct{}{}: + default: + } +} + +func (l *Loop) snapshot() *hub.WireWireguard { + l.mu.Lock() + defer l.mu.Unlock() + return l.block +} + +// Run reconciles immediately, then on every tick or desired-state nudge, until ctx is cancelled. +func (l *Loop) Run(ctx context.Context) error { + l.reconcile(ctx) + t := time.NewTicker(l.interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + case <-l.nudge: + } + l.reconcile(ctx) + } +} + +// reconcile runs one full pass: instance apply → belt sync → health/heal. +func (l *Loop) reconcile(ctx context.Context) { + block := l.snapshot() + port, err := l.mgr.Apply(ctx, block) + if err != nil { + return // Apply logged; a claim/exhaustion or install error — retry next tick + } + if l.belt != nil { + l.belt.Sync(ctx, port, oobPeerIP(block)) + } + l.mgr.HealAndCheck(ctx, port) +} + +// oobPeerIP extracts the operator /32 source (bare IP) from the block, or "" when OOB is off. +func oobPeerIP(block *hub.WireWireguard) string { + if block == nil { + return "" + } + return block.OOBPeerIP +} + +// OOBStatus implements the hub collector's reporter seam (Part 4). +func (l *Loop) OOBStatus(ctx context.Context) *hub.OOBStatus { + return l.mgr.Status(ctx, l.snapshot()) +} diff --git a/internal/felhomsshd/manager.go b/internal/felhomsshd/manager.go new file mode 100644 index 0000000..2f7268a --- /dev/null +++ b/internal/felhomsshd/manager.go @@ -0,0 +1,203 @@ +package felhomsshd + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "log/slog" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" + "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" +) + +// AuthKeysUserPath is felhom-op's authorized_keys (the default operator identity — key OUTSIDE +// ~/.ssh so the customer's sshd never honours it [SF-3]). FIXED (sudoers install target). +const AuthKeysUserPath = AuthKeysDir + "/" + OperatorUser + +const stagedConfName = "sshd_config" + +// Manager renders + applies the felhom-sshd config and drives the unit through the narrow-sudoers +// runner. Config changes go write→`sshd -t`→`reload` (NEVER restart-on-change [SF-2]); a deliberate +// restart is `reset-failed`-then-`restart` [SF-5], rate-limited by a cooldown (no flap). +type Manager struct { + runner proxmox.Runner + stateDir string + logger *slog.Logger + + // injectable seams (tests) + isActive func(ctx context.Context) bool + isFailed func(ctx context.Context) bool + isFree portProbe + now func() time.Time + + port int // the claimed port (0 until Apply claims it) + hash string // sha256 of the last-applied config (steady-state zero-exec gate) + akHash string // sha256 of the last-applied felhom-op authorized_keys + lastRestartAt time.Time // heal cooldown (Part 4) +} + +// restartCooldown bounds deliberate felhom-sshd restarts (heal path) — one attempt per window, so a +// persistently-broken instance is reported, not restart-stormed (spike §6 / notification-cooldown shape). +const restartCooldown = 10 * time.Minute + +// NewManager builds a Manager. stateDir is the agent state dir (staged config lives under it). +func NewManager(runner proxmox.Runner, stateDir string, logger *slog.Logger) *Manager { + if logger == nil { + logger = slog.Default() + } + return &Manager{ + runner: runner, + stateDir: stateDir, + logger: logger, + isActive: func(ctx context.Context) bool { + out, _ := exec.CommandContext(ctx, "systemctl", "is-active", Unit).Output() + return strings.TrimSpace(string(out)) == "active" + }, + isFailed: func(ctx context.Context) bool { + out, _ := exec.CommandContext(ctx, "systemctl", "is-failed", Unit).Output() + return strings.TrimSpace(string(out)) == "failed" + }, + isFree: probeFree, + now: time.Now, + } +} + +// Port returns the claimed port (0 before the first successful Apply). +func (m *Manager) Port() int { return m.port } + +func (m *Manager) sshdDir() string { return filepath.Join(m.stateDir, "felhom-sshd") } +func (m *Manager) stagedConfPath() string { return filepath.Join(m.sshdDir(), stagedConfName) } + +// Apply claims the port, renders the config, reconciles the running unit, and (when the desired-state +// block carries it) installs the operator's authorized_keys. Idempotent: no change → at most an +// is-active check. block may be nil (no desired-state yet) — the instance still runs; only the +// operator login/belt inputs are skipped. Returns the claimed port (0 on a claim/exhaustion error) — +// the caller (belt sync) needs it. +func (m *Manager) Apply(ctx context.Context, block *hub.WireWireguard) (int, error) { + port, err := claimPort(Candidates, m.isFree, readPortFile, writePortFile) + if err != nil { + m.logger.Error("felhomsshd: port claim failed", "err", err) + return 0, err + } + m.port = port + + // Operator authorized_keys (from the hub-driven block) — written to felhom-op's dedicated file, + // outside ~/.ssh [SF-3]. Independent of the config-reload path (a key change never reloads sshd). + if block != nil { + m.applyAuthorizedKeys(ctx, block.OOBOperatorSSHKey) + } + + conf, err := renderConfig(port) + if err != nil { + m.logger.Error("felhomsshd: refusing to apply invalid config", "err", err) + return port, err + } + sum := sha256.Sum256([]byte(conf)) + hash := hex.EncodeToString(sum[:]) + active := m.isActive(ctx) + + if m.hash == hash && active { + return port, nil // steady state: zero execs + } + + // Stage → validate → install → reload/enable. Never restart on a config change [SF-2]. + if err := os.MkdirAll(m.sshdDir(), 0o700); err != nil { + m.logger.Error("felhomsshd: state dir", "err", err) + return port, err + } + if err := os.WriteFile(m.stagedConfPath(), []byte(conf), 0o600); err != nil { + m.logger.Error("felhomsshd: staging config", "err", err) + return port, err + } + // Validate the STAGED config before it is installed — a bad render never reaches the live path. + if _, errOut, err := m.runner.Run(ctx, "sshd", "-t", "-f", m.stagedConfPath()); err != nil { + m.logger.Error("felhomsshd: staged config failed sshd -t — NOT installing", "err", err, "stderr", strings.TrimSpace(string(errOut))) + return port, err + } + if _, errOut, err := m.runner.Run(ctx, "install", "-o", "root", "-g", "root", "-m", "0644", "--", m.stagedConfPath(), ConfPath); err != nil { + m.logger.Error("felhomsshd: config install failed", "err", err, "stderr", strings.TrimSpace(string(errOut))) + return port, err + } + verb := "reload" + if !active { + verb = "enable" // first apply / down → enable --now brings it up + } + if err := m.systemctl(ctx, verb); err != nil { + return port, err + } + m.hash = hash + m.logger.Info("felhomsshd: config applied", "port", port, "action", verb) + return port, nil +} + +// applyAuthorizedKeys installs (or clears) felhom-op's authorized_keys from the hub-delivered +// operator SSH key. Hash-gated (no churn); the key is public, never a secret. A write failure is +// logged, not fatal — the instance keeps running. NO sshd reload needed (sshd reads the file per-auth). +func (m *Manager) applyAuthorizedKeys(ctx context.Context, sshKey string) { + content := "" + if k := strings.TrimSpace(sshKey); k != "" { + content = k + "\n" + } + sum := sha256.Sum256([]byte(content)) + h := hex.EncodeToString(sum[:]) + if h == m.akHash { + return // unchanged + } + staged := filepath.Join(m.sshdDir(), "authorized_keys."+OperatorUser) + if err := os.MkdirAll(m.sshdDir(), 0o700); err != nil { + m.logger.Error("felhomsshd: state dir for authorized_keys", "err", err) + return + } + if err := os.WriteFile(staged, []byte(content), 0o600); err != nil { + m.logger.Error("felhomsshd: staging authorized_keys", "err", err) + return + } + if _, errOut, err := m.runner.Run(ctx, "install", "-o", "root", "-g", "root", "-m", "0644", "--", staged, AuthKeysUserPath); err != nil { + m.logger.Error("felhomsshd: authorized_keys install failed", "err", err, "stderr", strings.TrimSpace(string(errOut))) + return + } + m.akHash = h + m.logger.Info("felhomsshd: operator authorized_keys updated", "user", OperatorUser, "present", content != "") +} + +// systemctl runs the reconcile verbs through the narrow runner. `enable` = `enable --now`; `reload` +// HUPs (config change, running instance survives a bad reload via the unit's ExecReload sshd -t gate). +func (m *Manager) systemctl(ctx context.Context, verb string) error { + var args []string + switch verb { + case "enable": + args = []string{"enable", "--now", Unit} + case "reload", "restart": + args = []string{verb, Unit} + default: + return nil + } + if _, errOut, err := m.runner.Run(ctx, "systemctl", args...); err != nil { + m.logger.Error("felhomsshd: systemctl "+verb+" failed", "err", err, "stderr", strings.TrimSpace(string(errOut))) + return err + } + return nil +} + +// probeFree is the production port probe: nothing LISTENing (ss) AND a real bind succeeds (a bind +// that succeeds-then-closes proves the port is actually claimable, not just ss-silent). +func probeFree(port int) bool { + p := strconv.Itoa(port) + out, err := exec.Command("ss", "-Htln", "sport = :"+p).Output() + if err == nil && strings.TrimSpace(string(out)) != "" { + return false // something is listening + } + ln, err := net.Listen("tcp", "0.0.0.0:"+p) + if err != nil { + return false + } + _ = ln.Close() + return true +} diff --git a/internal/hub/collect.go b/internal/hub/collect.go index 58a2ad4..c09a82a 100644 --- a/internal/hub/collect.go +++ b/internal/hub/collect.go @@ -70,6 +70,7 @@ type Collector struct { wg WireguardReporter // S3: offsite-tunnel status (nil → stanza omitted) selfUpdate SelfUpdateReporter // D1: agent self-update pending status (nil → false) mgmtPlane MgmtPlaneReporter // G1: management-plane health (nil → stanza omitted) + oob OOBReporter // H1: operator-access health (nil → stanza omitted) hostID string agentVersion string logger *slog.Logger @@ -154,6 +155,17 @@ func (c *Collector) SetMgmtPlaneReporter(m MgmtPlaneReporter) *Collector { return c } +// OOBReporter is the H1 seam the felhom-sshd loop plugs into (nil → no oob stanza). +type OOBReporter interface { + OOBStatus(ctx context.Context) *OOBStatus +} + +// SetOOBReporter wires the operator-access health source (H1; nil-safe → stanza omitted). +func (c *Collector) SetOOBReporter(o OOBReporter) *Collector { + c.oob = o + return c +} + // Collect builds the report. Best-effort liveness: a failed NodeStatus is a hard // error (no useful report — the cycle skips the POST); a failed per-guest // GuestConfig degrades that guest to status="unknown" without spec but still sends; @@ -200,6 +212,10 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { if c.mgmtPlane != nil { report.MgmtPlane = c.mgmtPlane.MgmtPlaneStatus(ctx) } + // H1: operator-access (OOB) health (nil reporter = feature not wired → stanza omitted). + if c.oob != nil { + report.OOB = c.oob.OOBStatus(ctx) + } return report, nil } diff --git a/internal/hub/report.go b/internal/hub/report.go index 4d957fe..22deeae 100644 --- a/internal/hub/report.go +++ b/internal/hub/report.go @@ -76,6 +76,26 @@ type HostReport struct { // SelfUpdatePending precedent): stored opaquely hub-side, so these additive fields need no // hub-schema change and are absent when the reporter is not wired. MgmtPlane *MgmtPlaneStatus `json:"mgmt_plane,omitempty"` + + // OOB is the operator-access health stanza (TASK H1). It answers the operator's question — "can I + // get into this box right now, and if not, why" — from the hub: felhom-sshd up + on which port, + // locally reachable, the tunnel handshake age (the OOB path rides wg-felhom), whether the operator + // peer/key is configured, and whether the config is invalid. The heartbeat reaches the hub over + // HTTPS even when felhom-sshd or the tunnel is DOWN (channel independence). `omitempty`: absent + // when the feature is not wired (pre-H1) — additive, no hub-schema change. + OOB *OOBStatus `json:"oob,omitempty"` +} + +// OOBStatus is the per-heartbeat operator-access health (TASK H1). Carries no secret. +type OOBStatus struct { + FelhomSshdActive bool `json:"felhom_sshd_active"` // the felhom-sshd unit is active + FelhomSshdPort int `json:"felhom_sshd_port"` // the claimed OOB port (0 = unclaimed) + Reachable bool `json:"reachable"` // a local TCP dial to the OOB port succeeds + ConfigInvalid bool `json:"config_invalid"` // `sshd -t` on the OOB config fails + OperatorPeerConfigured bool `json:"operator_peer_configured"` // oob_peer_ip present in desired-state + OperatorKeyConfigured bool `json:"operator_key_configured"` // operator authorized_key installed + WGHandshakeAgeS *int64 `json:"wg_handshake_age_s,omitempty"` // wg-felhom last handshake age (nil = unknown) + HealedAt string `json:"healed_at,omitempty"` // last felhom-sshd auto-heal (RFC3339) } // MgmtPlaneStatus is the per-heartbeat management-plane health (TASK G1). Carries no secret. @@ -84,9 +104,9 @@ type HostReport struct { // hub raises a warning event on a PrivsepHealedAt it has not alerted on — a recurring auto-heal means // a persistent clobber cause worth investigating before the box locks out. type MgmtPlaneStatus struct { - PrivsepDirOK bool `json:"privsep_dir_ok"` // /run/sshd exists (the KEXINIT-reset detector) - SshdReachable bool `json:"sshd_reachable"` // the stock sshd listener accepts TCP - HealedRecently bool `json:"healed_recently"` // the watchdog heal-marker is present (this boot) + PrivsepDirOK bool `json:"privsep_dir_ok"` // /run/sshd exists (the KEXINIT-reset detector) + SshdReachable bool `json:"sshd_reachable"` // the stock sshd listener accepts TCP + HealedRecently bool `json:"healed_recently"` // the watchdog heal-marker is present (this boot) PrivsepHealedAt string `json:"privsep_healed_at,omitempty"` // marker timestamp; hub warns on a NEW value } @@ -364,6 +384,12 @@ type WireWireguard struct { // set`) so it survives self-heal/restart/reboot ([OF-1]: a runtime widening was wiped by the // agent's own self-heal). `omitempty`: absent = OOB off, byte-identical conf for pre-H1 hosts. OOBPeerIP string `json:"oob_peer_ip,omitempty"` + + // OOBOperatorSSHKey is the operator's SSH PUBLIC key (an authorized_keys line, TASK H1). The agent + // writes it to felhom-sshd's dedicated AuthorizedKeysFile (/etc/felhom-sshd/authorized_keys/{root, + // felhom-op}) — OUTSIDE ~/.ssh, so the customer's sshd never honours it [SF-3]. Hub-driven so it + // rotates fleet-wide. `omitempty`: absent = no operator login installed. NOT a secret (public key). + OOBOperatorSSHKey string `json:"oob_operator_ssh_key,omitempty"` } // WireWireguardEndpoint is the endpoint half of the wireguard block.