felhom-agent/internal/selfheal/selfheal.go

// Package selfheal is a minimal check/remedy registry for node-level self-healing, gated hard on
// appliance deployment mode (CAMPAIGN-3 Part 6).
//
// Only ONE heal ships now: host networking recovery — F12-class defense in depth. The F12 fix (the
// automount template no longer creates the boot ordering cycle) is the CURE for that specific instance;
// this watchdog is the belt for the CLASS: any boot that leaves networking down for ANY reason (a future
// ordering bug, a flaky NIC bring-up, a botched netplan) is detected and — on a Felhom-managed appliance
// only — remedied with the exact command the morning recovery ran by hand (`systemctl start
// networking.service`). The registry shape is the deliverable so future appliance heals slot in behind
// the SAME gate; resist growing a zoo of heals.
//
// THE GATE (never touch a host we do not own): a BYO host runs the CHECK and WARNs, but the Manager
// refuses to call any Remediate before the appliance test — the remedy is structurally unreachable on
// byo (unit-tested: byo + unhealthy → zero privileged invocations).
package selfheal

import (
	"context"
	"fmt"
	"log/slog"
	"os/exec"
	"strings"
	"time"
)

// Heal is one node-level check + remedy. Healthy reports the current state (and a human detail for the
// unhealthy log); Remediate attempts recovery and returns a terminal error if it gives up.
type Heal interface {
	Name() string
	Healthy(ctx context.Context) (ok bool, detail string)
	Remediate(ctx context.Context) error
}

// Manager runs the registered heals at boot and on a periodic watchdog. The appliance gate lives HERE,
// before any Remediate — a byo node never reaches a remedy.
type Manager struct {
	Appliance bool
	Interval  time.Duration // watchdog cadence (≈60s); <=0 disables the periodic loop
	Heals     []Heal
	Logger    *slog.Logger
}

// RunOnce evaluates every heal once (the boot pass). For each unhealthy heal it WARNs (both facts in
// the detail); on an appliance it then remediates, on byo it stops at the WARN — the remedy exec is
// never reached off-appliance.
func (m *Manager) RunOnce(ctx context.Context) {
	for _, h := range m.Heals {
		ok, detail := h.Healthy(ctx)
		if ok {
			m.Logger.Debug("selfheal: node check healthy", "heal", h.Name())
			continue
		}
		m.Logger.Warn("selfheal: node check FAILED", "heal", h.Name(), "detail", detail, "mode", m.mode())
		if !m.Appliance {
			// GATE: byo hosts are the customer's to manage — never remediate. (Red-proof: byo +
			// unhealthy must record ZERO privileged invocations; the remedy is unreachable here.)
			m.Logger.Warn("selfheal: byo host — remedy skipped (not ours to touch)", "heal", h.Name())
			continue
		}
		if err := h.Remediate(ctx); err != nil {
			m.Logger.Error("selfheal: remedy GAVE UP — host needs attention", "heal", h.Name(), "err", err)
		}
	}
}

// Watch runs RunOnce once immediately (the boot pass) then on every Interval tick until ctx is done.
func (m *Manager) Watch(ctx context.Context) {
	m.RunOnce(ctx)
	if m.Interval <= 0 {
		return
	}
	t := time.NewTicker(m.Interval)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			m.RunOnce(ctx)
		}
	}
}

func (m *Manager) mode() string {
	if m.Appliance {
		return "appliance"
	}
	return "byo"
}

// NetworkingHeal is the one shipped heal: host networking (F12-class). Healthy ⇔ networking.service is
// active AND a default route exists. All three probes/actions are seams (unit-tested with fakes); Start
// is the ONLY privileged one (`systemctl start networking.service`, appliance-gated by the Manager).
type NetworkingHeal struct {
	IsActive func(ctx context.Context) bool  // `systemctl is-active networking.service` (unprivileged)
	HasRoute func(ctx context.Context) bool  // a default route exists (unprivileged)
	Start    func(ctx context.Context) error // PRIVILEGED `systemctl start networking.service`
	Sleep    func(ctx context.Context, d time.Duration)
	Logger   *slog.Logger
}

func (n *NetworkingHeal) Name() string { return "networking" }

// Healthy reports both facts so the WARN detail names exactly what is down.
func (n *NetworkingHeal) Healthy(ctx context.Context) (bool, string) {
	active := n.IsActive(ctx)
	route := n.HasRoute(ctx)
	if active && route {
		return true, ""
	}
	return false, fmt.Sprintf("networking.service active=%t, default route present=%t", active, route)
}

// networkingBackoffs is the wait after each start attempt before re-checking — 10s/30s/60s gives a slow
// NIC/DHCP time to come up. Package var so tests can shrink it.
var networkingBackoffs = []time.Duration{10 * time.Second, 30 * time.Second, 60 * time.Second}

// Remediate runs up to 3 `systemctl start networking.service` attempts, each followed by its backoff and
// a health re-check. ERROR per failed attempt; INFO on recovery; a terminal give-up error if all fail.
func (n *NetworkingHeal) Remediate(ctx context.Context) error {
	for attempt := 1; attempt <= len(networkingBackoffs); attempt++ {
		if err := n.Start(ctx); err != nil {
			n.Logger.Error("node self-heal: networking start attempt failed", "attempt", attempt, "err", err)
		}
		n.Sleep(ctx, networkingBackoffs[attempt-1])
		if ok, _ := n.Healthy(ctx); ok {
			n.Logger.Info("node self-heal: networking recovered", "attempt", attempt)
			return nil
		}
		n.Logger.Error("node self-heal: networking still down after start attempt", "attempt", attempt)
	}
	return fmt.Errorf("networking still down after %d attempts (host needs physical/console attention)", len(networkingBackoffs))
}

// --- production seams ------------------------------------------------------------------------------

// SystemctlIsActive is the default IsActive: `systemctl is-active <unit>` prints "active" when up.
// Unprivileged (unit state is world-readable) — NOT routed through sudo.
func SystemctlIsActive(unit string) func(context.Context) bool {
	return func(ctx context.Context) bool {
		out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "--", unit).Output()
		return strings.TrimSpace(string(out)) == "active"
	}
}

// HasDefaultRoute is the default HasRoute: `ip route show default` is non-empty when a default route
// exists. Unprivileged read.
func HasDefaultRoute(ctx context.Context) bool {
	out, err := exec.CommandContext(ctx, "ip", "route", "show", "default").Output()
	return err == nil && strings.TrimSpace(string(out)) != ""
}

// RealSleep is the default Sleep: a context-cancellable time.Sleep.
func RealSleep(ctx context.Context, d time.Duration) {
	t := time.NewTimer(d)
	defer t.Stop()
	select {
	case <-ctx.Done():
	case <-t.C:
	}
}