package felhomsshd import ( "context" "os" "os/exec" "strconv" "strings" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" ) // HealMarkerPath records the last felhom-sshd auto-heal (RFC3339), so the heartbeat surfaces a // recurring failure to the operator (the mgmt_plane pattern, scoped to the OOB daemon). const HealMarkerPath = "/run/felhom-sshd.healed" // HealAndCheck restores felhom-sshd if it is down AND its config is valid — ONE restart per cooldown // [SF-5 / spike §6], so a persistently-broken instance is reported (Status), not restart-stormed. It // NEVER restarts onto a broken config [trap 8]: if `sshd -t` fails, it hands off (Status reports // config_invalid; the hub warns). A healthy/active instance is untouched. func (m *Manager) HealAndCheck(ctx context.Context, port int) { if m.isActive(ctx) { return // healthy — nothing to heal } // Down. Only restart if the config is VALID — never convert degraded into dead. if _, errOut, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil { m.logger.Warn("felhomsshd: down AND config invalid — NOT restarting (report-only)", "stderr", strings.TrimSpace(string(errOut))) return } // Cooldown: at most one deliberate restart per window. now := m.now() if !m.lastRestartAt.IsZero() && now.Sub(m.lastRestartAt) < restartCooldown { return } m.lastRestartAt = now // reset-failed BEFORE restart [SF-5]: a start-limit lockout otherwise refuses the restart. if m.isFailed(ctx) { _, _, _ = m.runner.Run(ctx, "systemctl", "reset-failed", Unit) } if err := m.systemctl(ctx, "restart"); err != nil { return } // record the heal (best-effort; a failed marker write never fails the heal) _ = os.WriteFile(HealMarkerPath, []byte(now.UTC().Format(time.RFC3339)+"\n"), 0o644) m.logger.Warn("felhomsshd: was down with a valid config — restarted (heal)", "port", port) } // Status builds the OOB heartbeat stanza (Part 4). Read-only. Discovers the effective port(s) via // `sshd -T` (authoritative — catches a non-default/multi-Port config), dials the port locally to // prove reachability, reads the wg-felhom handshake age, and reflects operator-peer/key config from // the desired-state block. NEVER `wg show dump` (the S1 ban) — `latest-handshakes` only. func (m *Manager) Status(ctx context.Context, block *hub.WireWireguard) *hub.OOBStatus { st := &hub.OOBStatus{ FelhomSshdActive: m.isActive(ctx), FelhomSshdPort: m.port, } // Authoritative port(s) from `sshd -T` (may differ from m.port if the config was hand-edited). if ports := m.sshdEffectivePorts(ctx); len(ports) > 0 { st.FelhomSshdPort = ports[0] } // config validity if _, _, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil { st.ConfigInvalid = true } // Reachability = a LISTENER is bound on the OOB port (catches "active but crashed post-fork"). We // do NOT dial: the belt (correctly) drops even localhost→felhom-sshd (tunnel-only), so a local // dial always fails and would misreport a healthy daemon as unreachable. if st.FelhomSshdPort > 0 { st.Reachable = listenerPresent(st.FelhomSshdPort) } // wg-felhom handshake age (the OOB path rides the tunnel) if age, ok := m.wgHandshakeAge(ctx); ok { st.WGHandshakeAgeS = &age } // Operator-peer configured: from the in-memory block when fetched, OR (robust across an agent // restart, before the next desired-state fetch) from the PERSISTENT rendered wg-felhom.conf — a // second /32 in AllowedIPs is the operator peer. This keeps the report (and the oob_degraded alert // gate) accurate immediately after a restart, not only after the next 900s heartbeat fetch. st.OperatorPeerConfigured = wgConfHasOperatorPeer() if block != nil { if block.OOBPeerIP != "" { st.OperatorPeerConfigured = true } st.OperatorKeyConfigured = strings.TrimSpace(block.OOBOperatorSSHKey) != "" } // Operator key: robust across restart via the installed authorized_keys file. if fi, err := os.Stat(AuthKeysUserPath); err == nil && fi.Size() > 0 { st.OperatorKeyConfigured = true } // last auto-heal if raw, err := os.ReadFile(HealMarkerPath); err == nil { st.HealedAt = strings.TrimSpace(string(raw)) } return st } // sshdEffectivePorts parses `sshd -T -f ` for the effective Port line(s) (handles multi-Port). func (m *Manager) sshdEffectivePorts(ctx context.Context) []int { out, _, err := m.runner.Run(ctx, "sshd", "-T", "-f", ConfPath) if err != nil { return nil } var ports []int for _, line := range strings.Split(string(out), "\n") { f := strings.Fields(strings.ToLower(line)) if len(f) == 2 && f[0] == "port" { if p, err := strconv.Atoi(f[1]); err == nil && p > 0 && p <= 65535 { ports = append(ports, p) } } } return ports } // wgHandshakeAge reads wg-felhom's latest-handshake age in seconds (latest-handshakes ONLY — the // dump ban). ok=false when the tunnel is down or unreadable. func (m *Manager) wgHandshakeAge(ctx context.Context) (int64, bool) { out, _, err := m.runner.Run(ctx, "wg", "show", "wg-felhom", "latest-handshakes") if err != nil { return 0, false } for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { f := strings.Fields(line) if len(f) != 2 { continue } epoch, err := strconv.ParseInt(f[1], 10, 64) if err != nil || epoch <= 0 { continue } age := m.now().Unix() - epoch if age < 0 { age = 0 } return age, true } return 0, false } // listenerPresent reports whether something is LISTENing on the port (via ss — reads kernel state, // so the belt never blocks it, unlike a real dial). func listenerPresent(port int) bool { out, err := exec.Command("ss", "-Htln", "sport = :"+strconv.Itoa(port)).Output() return err == nil && strings.TrimSpace(string(out)) != "" } // wgConfHasOperatorPeer reports whether the rendered wg-felhom.conf carries a SECOND AllowedIPs /32 // (the operator OOB peer, alongside the PBS /32). A pure file read — always current, survives restart. func wgConfHasOperatorPeer() bool { raw, err := os.ReadFile("/etc/wireguard/wg-felhom.conf") if err != nil { return false } for _, line := range strings.Split(string(raw), "\n") { if strings.HasPrefix(strings.TrimSpace(line), "AllowedIPs") && strings.Contains(line, ",") { return true // two or more /32s = PBS + operator } } return false }