felhom-agent/internal/felhomsshd/health.go

package felhomsshd

import (
	"context"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)

// HealMarkerPath records the last felhom-sshd auto-heal (RFC3339), so the heartbeat surfaces a
// recurring failure to the operator (the mgmt_plane pattern, scoped to the OOB daemon).
const HealMarkerPath = "/run/felhom-sshd.healed"

// HealAndCheck restores felhom-sshd if it is down AND its config is valid — ONE restart per cooldown
// [SF-5 / spike §6], so a persistently-broken instance is reported (Status), not restart-stormed. It
// NEVER restarts onto a broken config [trap 8]: if `sshd -t` fails, it hands off (Status reports
// config_invalid; the hub warns). A healthy/active instance is untouched.
func (m *Manager) HealAndCheck(ctx context.Context, port int) {
	if m.isActive(ctx) {
		return // healthy — nothing to heal
	}
	// Down. Only restart if the config is VALID — never convert degraded into dead.
	if _, errOut, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil {
		m.logger.Warn("felhomsshd: down AND config invalid — NOT restarting (report-only)",
			"stderr", strings.TrimSpace(string(errOut)))
		return
	}
	// Cooldown: at most one deliberate restart per window.
	now := m.now()
	if !m.lastRestartAt.IsZero() && now.Sub(m.lastRestartAt) < restartCooldown {
		return
	}
	m.lastRestartAt = now
	// reset-failed BEFORE restart [SF-5]: a start-limit lockout otherwise refuses the restart.
	if m.isFailed(ctx) {
		_, _, _ = m.runner.Run(ctx, "systemctl", "reset-failed", Unit)
	}
	if err := m.systemctl(ctx, "restart"); err != nil {
		return
	}
	// record the heal (best-effort; a failed marker write never fails the heal)
	_ = os.WriteFile(HealMarkerPath, []byte(now.UTC().Format(time.RFC3339)+"\n"), 0o644)
	m.logger.Warn("felhomsshd: was down with a valid config — restarted (heal)", "port", port)
}

// Status builds the OOB heartbeat stanza (Part 4). Read-only. Discovers the effective port(s) via
// `sshd -T` (authoritative — catches a non-default/multi-Port config), dials the port locally to
// prove reachability, reads the wg-felhom handshake age, and reflects operator-peer/key config from
// the desired-state block. NEVER `wg show dump` (the S1 ban) — `latest-handshakes` only.
func (m *Manager) Status(ctx context.Context, block *hub.WireWireguard) *hub.OOBStatus {
	st := &hub.OOBStatus{
		FelhomSshdActive: m.isActive(ctx),
		FelhomSshdPort:   m.port,
	}
	// Authoritative port(s) from `sshd -T` (may differ from m.port if the config was hand-edited).
	if ports := m.sshdEffectivePorts(ctx); len(ports) > 0 {
		st.FelhomSshdPort = ports[0]
	}
	// config validity
	if _, _, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil {
		st.ConfigInvalid = true
	}
	// local reachability (a TCP dial to the OOB port)
	if st.FelhomSshdPort > 0 {
		st.Reachable = dialLocal(ctx, st.FelhomSshdPort)
	}
	// wg-felhom handshake age (the OOB path rides the tunnel)
	if age, ok := m.wgHandshakeAge(ctx); ok {
		st.WGHandshakeAgeS = &age
	}
	// desired-state config reflection
	if block != nil {
		st.OperatorPeerConfigured = block.OOBPeerIP != ""
		st.OperatorKeyConfigured = strings.TrimSpace(block.OOBOperatorSSHKey) != ""
	}
	// last auto-heal
	if raw, err := os.ReadFile(HealMarkerPath); err == nil {
		st.HealedAt = strings.TrimSpace(string(raw))
	}
	return st
}

// sshdEffectivePorts parses `sshd -T -f <conf>` for the effective Port line(s) (handles multi-Port).
func (m *Manager) sshdEffectivePorts(ctx context.Context) []int {
	out, _, err := m.runner.Run(ctx, "sshd", "-T", "-f", ConfPath)
	if err != nil {
		return nil
	}
	var ports []int
	for _, line := range strings.Split(string(out), "\n") {
		f := strings.Fields(strings.ToLower(line))
		if len(f) == 2 && f[0] == "port" {
			if p, err := strconv.Atoi(f[1]); err == nil && p > 0 && p <= 65535 {
				ports = append(ports, p)
			}
		}
	}
	return ports
}

// wgHandshakeAge reads wg-felhom's latest-handshake age in seconds (latest-handshakes ONLY — the
// dump ban). ok=false when the tunnel is down or unreadable.
func (m *Manager) wgHandshakeAge(ctx context.Context) (int64, bool) {
	out, _, err := m.runner.Run(ctx, "wg", "show", "wg-felhom", "latest-handshakes")
	if err != nil {
		return 0, false
	}
	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		f := strings.Fields(line)
		if len(f) != 2 {
			continue
		}
		epoch, err := strconv.ParseInt(f[1], 10, 64)
		if err != nil || epoch <= 0 {
			continue
		}
		age := m.now().Unix() - epoch
		if age < 0 {
			age = 0
		}
		return age, true
	}
	return 0, false
}

// dialLocal reports whether a TCP connect to 127.0.0.1:port succeeds within a short timeout.
func dialLocal(ctx context.Context, port int) bool {
	d := net.Dialer{Timeout: 2 * time.Second}
	conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
	if err != nil {
		return false
	}
	_ = conn.Close()
	return true
}