felhom-agent/internal/felhomsshd/health.go

package felhomsshd

import (
	"context"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)

// HealMarkerPath records the last felhom-sshd auto-heal (RFC3339), so the heartbeat surfaces a
// recurring failure to the operator (the mgmt_plane pattern, scoped to the OOB daemon).
const HealMarkerPath = "/run/felhom-sshd.healed"

// HealAndCheck restores felhom-sshd if it is down AND its config is valid — ONE restart per cooldown
// [SF-5 / spike §6], so a persistently-broken instance is reported (Status), not restart-stormed. It
// NEVER restarts onto a broken config [trap 8]: if `sshd -t` fails, it hands off (Status reports
// config_invalid; the hub warns). A healthy/active instance is untouched.
func (m *Manager) HealAndCheck(ctx context.Context, port int) {
	if m.isActive(ctx) {
		return // healthy — nothing to heal
	}
	// Down. Only restart if the config is VALID — never convert degraded into dead.
	if _, errOut, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil {
		m.logger.Warn("felhomsshd: down AND config invalid — NOT restarting (report-only)",
			"stderr", strings.TrimSpace(string(errOut)))
		return
	}
	// Cooldown: at most one deliberate restart per window.
	now := m.now()
	if !m.lastRestartAt.IsZero() && now.Sub(m.lastRestartAt) < restartCooldown {
		return
	}
	m.lastRestartAt = now
	// reset-failed BEFORE restart [SF-5]: a start-limit lockout otherwise refuses the restart.
	if m.isFailed(ctx) {
		_, _, _ = m.runner.Run(ctx, "systemctl", "reset-failed", Unit)
	}
	if err := m.systemctl(ctx, "restart"); err != nil {
		return
	}
	// record the heal (best-effort; a failed marker write never fails the heal)
	_ = os.WriteFile(HealMarkerPath, []byte(now.UTC().Format(time.RFC3339)+"\n"), 0o644)
	m.logger.Warn("felhomsshd: was down with a valid config — restarted (heal)", "port", port)
}

// Status builds the OOB heartbeat stanza (Part 4). Read-only. Discovers the effective port(s) via
// `sshd -T` (authoritative — catches a non-default/multi-Port config), dials the port locally to
// prove reachability, reads the wg-felhom handshake age, and reflects operator-peer/key config from
// the desired-state block. NEVER `wg show dump` (the S1 ban) — `latest-handshakes` only.
func (m *Manager) Status(ctx context.Context, block *hub.WireWireguard) *hub.OOBStatus {
	st := &hub.OOBStatus{
		FelhomSshdActive: m.isActive(ctx),
		FelhomSshdPort:   m.port,
	}
	// Authoritative port(s) from `sshd -T` (may differ from m.port if the config was hand-edited).
	if ports := m.sshdEffectivePorts(ctx); len(ports) > 0 {
		st.FelhomSshdPort = ports[0]
	}
	// config validity
	if _, _, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil {
		st.ConfigInvalid = true
	}
	// Reachability = a LISTENER is bound on the OOB port (catches "active but crashed post-fork"). We
	// do NOT dial: the belt (correctly) drops even localhost→felhom-sshd (tunnel-only), so a local
	// dial always fails and would misreport a healthy daemon as unreachable.
	if st.FelhomSshdPort > 0 {
		st.Reachable = listenerPresent(st.FelhomSshdPort)
	}
	// wg-felhom handshake age (the OOB path rides the tunnel)
	if age, ok := m.wgHandshakeAge(ctx); ok {
		st.WGHandshakeAgeS = &age
	}
	// Operator-peer configured: from the in-memory block when fetched, OR (robust across an agent
	// restart, before the next desired-state fetch) from the PERSISTENT belt @operator_ips set — the
	// agent can read it via the sudo nft-list grant (unlike the 0600 root-owned wg-felhom.conf). This
	// keeps the report + the oob_degraded alert gate accurate immediately after a restart.
	st.OperatorPeerConfigured = m.beltOperatorConfigured(ctx)
	if block != nil {
		if block.OOBPeerIP != "" {
			st.OperatorPeerConfigured = true
		}
		st.OperatorKeyConfigured = strings.TrimSpace(block.OOBOperatorSSHKey) != ""
	}
	// Operator key: robust across restart via the installed authorized_keys file.
	if fi, err := os.Stat(AuthKeysUserPath); err == nil && fi.Size() > 0 {
		st.OperatorKeyConfigured = true
	}
	// last auto-heal
	if raw, err := os.ReadFile(HealMarkerPath); err == nil {
		st.HealedAt = strings.TrimSpace(string(raw))
	}
	return st
}

// sshdEffectivePorts parses `sshd -T -f <conf>` for the effective Port line(s) (handles multi-Port).
func (m *Manager) sshdEffectivePorts(ctx context.Context) []int {
	out, _, err := m.runner.Run(ctx, "sshd", "-T", "-f", ConfPath)
	if err != nil {
		return nil
	}
	var ports []int
	for _, line := range strings.Split(string(out), "\n") {
		f := strings.Fields(strings.ToLower(line))
		if len(f) == 2 && f[0] == "port" {
			if p, err := strconv.Atoi(f[1]); err == nil && p > 0 && p <= 65535 {
				ports = append(ports, p)
			}
		}
	}
	return ports
}

// wgHandshakeAge reads wg-felhom's latest-handshake age in seconds (latest-handshakes ONLY — the
// dump ban). ok=false when the tunnel is down or unreadable.
func (m *Manager) wgHandshakeAge(ctx context.Context) (int64, bool) {
	out, _, err := m.runner.Run(ctx, "wg", "show", "wg-felhom", "latest-handshakes")
	if err != nil {
		return 0, false
	}
	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		f := strings.Fields(line)
		if len(f) != 2 {
			continue
		}
		epoch, err := strconv.ParseInt(f[1], 10, 64)
		if err != nil || epoch <= 0 {
			continue
		}
		age := m.now().Unix() - epoch
		if age < 0 {
			age = 0
		}
		return age, true
	}
	return 0, false
}

// listenerPresent reports whether something is LISTENing on the port (via ss — reads kernel state,
// so the belt never blocks it, unlike a real dial).
func listenerPresent(port int) bool {
	out, err := exec.Command("ss", "-Htln", "sport = :"+strconv.Itoa(port)).Output()
	return err == nil && strings.TrimSpace(string(out)) != ""
}

// beltOperatorConfigured reports whether the belt's @operator_ips set is non-empty (an operator /32
// is allowed to reach felhom-sshd). Read via the sudo nft-list grant — persistent + agent-readable.
func (m *Manager) beltOperatorConfigured(ctx context.Context) bool {
	out, _, err := m.runner.Run(ctx, "nft", "list", "set", "inet", "felhom_oob", "operator_ips")
	if err != nil {
		return false
	}
	return strings.Contains(string(out), "elements")
}