wgtunnel: v4-pin + re-resolve watchdog; FELHOM_WG Critical flips (S4 agent half)

v4-pin (doc 06 §4.2): renderConf takes a pre-resolved IPv4 literal and writes Endpoint=<ip>:<port> — never the DNS name, never AAAA. Resolver seam (A records only, LookupNetIP "ip4"); multiple A → lowest (deterministic fleet-wide); renderConf stays pure. Resolved IP cached: steady-state Apply = zero DNS + zero execs. DNS failure keeps the last conf (never a teardown). Watchdog (loop-only, so Apply's zero-exec steady state is untouched): handshake age > wg_tunnel.stale_after_seconds (default 180) → re-resolve; IP changed → re-render + restart (endpoint re-IP recovery); IP same → no churn (throttled warn). Staleness read reuses wg show latest-handshakes (never dump). Capability: wg-conf-install/enable/restart/handshake-read flipped Critical=true (backups ride the tunnel from S4); apt-install + disable stay non-critical. TestWGCapabilityCriticality pins the set. Tests + red-proofs a/b/d all fire. No new sudoers grant; no wire/JSON change. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
2026-07-04 15:47:17 +02:00
parent c618fc69f7
commit 734f45c422
8 changed files with 436 additions and 23 deletions
@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log/slog"
+	"net"
 	"net/netip"
 	"os"
 	"os/exec"
@@ -44,6 +45,32 @@ type Registrar interface {
 	RegisterWG(ctx context.Context, pubkey string) (*hub.WGRegisterResponse, error)
 }

+// Resolver resolves the endpoint's DNS name to IPv4 addresses (doc 06 §4.2 v4-pin). The seam lets
+// tests drive resolution deterministically; the production impl asks ONLY for A records ("ip4"),
+// so the AAAA is never returned — the tunnel can never silently ride un-NATed IPv6.
+type Resolver interface {
+	LookupIPv4(ctx context.Context, host string) ([]netip.Addr, error)
+}
+
+// netResolver is the production Resolver — the system resolver, A records only.
+type netResolver struct{}
+
+func (netResolver) LookupIPv4(ctx context.Context, host string) ([]netip.Addr, error) {
+	return net.DefaultResolver.LookupNetIP(ctx, "ip4", host)
+}
+
+// lowestAddr picks the numerically lowest address — a deterministic, fleet-wide-stable choice when
+// the endpoint publishes multiple A records (every box picks the same one; no per-box drift).
+func lowestAddr(addrs []netip.Addr) netip.Addr {
+	best := addrs[0]
+	for _, a := range addrs[1:] {
+		if a.Compare(best) < 0 {
+			best = a
+		}
+	}
+	return best
+}
+
 // marker is the local registration record (<StateDir>/wg/registered.json). Its EXISTENCE is the
 // registration gate: present → never register again except the pubkey-mismatch re-key path.
 // KEPT on revocation (revoked stays revoked — doc 06 §3.5 completion).
@@ -66,12 +93,20 @@ type Manager struct {
 	isActive  func(ctx context.Context) bool
 	wgPresent func() bool
 	now       func() time.Time
+	resolver  Resolver
+
+	staleAfter time.Duration // handshake-age threshold for the re-resolve watchdog (doc 06 §4.2)

 	mu              sync.Mutex
 	nextRegAt       time.Time
 	regBackoff      time.Duration
 	keyBroken       bool // corrupt key / partial state — loop idles until operator resolves
 	brokenAnnounced bool
+
+	// v4-pin cache + watchdog throttles (all guarded by mu).
+	lastResolvedIP    netip.Addr // last A-record we rendered into the conf; invalid → resolve on next apply
+	resolveFailLogged bool       // DNS-failure log throttle (reset on the next success)
+	staleSameIPLogged bool       // "stale but IP unchanged" log throttle (endpoint-down, not re-IP)
 }

 // NewManager builds a Manager. stateDir is the agent state dir (default /var/lib/felhom-agent —
@@ -93,7 +128,19 @@ func NewManager(runner proxmox.Runner, registrar Registrar, stateDir string, log
 			_, err := os.Stat("/usr/bin/wg")
 			return err == nil
 		},
-		now: time.Now,
+		now:        time.Now,
+		resolver:   netResolver{},
+		staleAfter: 180 * time.Second,
+	}
+}
+
+// SetStaleAfter overrides the re-resolve staleness threshold (from wg_tunnel.stale_after_seconds);
+// non-positive values keep the 180s default.
+func (m *Manager) SetStaleAfter(d time.Duration) {
+	if d > 0 {
+		m.mu.Lock()
+		m.staleAfter = d
+		m.mu.Unlock()
 	}
 }

@@ -148,17 +195,23 @@ func validKeyB64(s string) error {
 // (silently black-holed sub-~1480 paths — CGNAT smoke test 2026-07-04). See doc 06 §4.3.
 const clientMTU = 1280

-// renderConf builds the wg-felhom.conf content from the hub block + the local private key.
-// Client-side constants per doc 06 §4: MTU 1280, AllowedIPs = pbs_tunnel_ip/32 (the tunnel
+// renderConf builds the wg-felhom.conf content from the hub block + the local private key + the
+// PRE-RESOLVED endpoint IPv4 (the caller resolves the DNS name; renderConf stays pure — no DNS).
+// The Endpoint line carries the v4 LITERAL (doc 06 §4.2 v4-pin: deterministic family, never the
+// AAAA). Client-side constants per doc 06 §4: MTU 1280, AllowedIPs = pbs_tunnel_ip/32 (the tunnel
 // carries ONLY box→PBS traffic), PersistentKeepalive 25. All inputs validated — nothing
 // user-controlled is interpolatable (strict charsets, netip parses).
-func renderConf(block *hub.WireWireguard, privB64 string) (string, error) {
+func renderConf(block *hub.WireWireguard, privB64, endpointIPv4 string) (string, error) {
 	if err := validKeyB64(privB64); err != nil {
 		return "", fmt.Errorf("wgtunnel: private key: %w", err)
 	}
 	if err := validKeyB64(block.Endpoint.ServerPubkey); err != nil {
 		return "", fmt.Errorf("wgtunnel: server_pubkey: %w", err)
 	}
+	epIP, err := netip.ParseAddr(endpointIPv4)
+	if err != nil || !epIP.Is4() {
+		return "", fmt.Errorf("wgtunnel: endpoint ip %q is not an IPv4 literal", endpointIPv4)
+	}
 	addr, err := netip.ParsePrefix(block.AssignedIP)
 	if err != nil || addr.Bits() != 32 {
 		return "", fmt.Errorf("wgtunnel: assigned_ip %q is not an ip/32", block.AssignedIP)
@@ -182,7 +235,7 @@ func renderConf(block *hub.WireWireguard, privB64 string) (string, error) {
 	fmt.Fprintf(&b, "MTU = %d\n\n", clientMTU)
 	b.WriteString("[Peer]\n")
 	fmt.Fprintf(&b, "PublicKey = %s\n", block.Endpoint.ServerPubkey)
-	fmt.Fprintf(&b, "Endpoint = %s:%d\n", block.Endpoint.DNSName, block.Endpoint.WGPort)
+	fmt.Fprintf(&b, "Endpoint = %s:%d\n", epIP, block.Endpoint.WGPort)
 	fmt.Fprintf(&b, "AllowedIPs = %s/32\n", pbsIP)
 	b.WriteString("PersistentKeepalive = 25\n")
 	return b.String(), nil
@@ -320,7 +373,11 @@ func (m *Manager) ensureTunnelLocked(ctx context.Context, block *hub.WireWiregua
 		m.announceBroken("wgtunnel: " + err.Error())
 		return
 	}
-	conf, err := renderConf(block, priv)
+	epIP, ok := m.resolveEndpointLocked(ctx, block)
+	if !ok {
+		return // resolveEndpointLocked logged; conf UNTOUCHED — DNS failure is never a teardown
+	}
+	conf, err := renderConf(block, priv, epIP)
 	if err != nil {
 		m.logger.Error("wgtunnel: refusing to apply invalid desired block", "err", err)
 		return
@@ -375,7 +432,98 @@ func (m *Manager) ensureTunnelLocked(ctx context.Context, block *hub.WireWiregua
 		return
 	}
 	m.logger.Info("wgtunnel: tunnel conf applied", "endpoint",
-		fmt.Sprintf("%s:%d", block.Endpoint.DNSName, block.Endpoint.WGPort), "assigned_ip", block.AssignedIP, "action", verb[0])
+		fmt.Sprintf("%s:%d", epIP, block.Endpoint.WGPort), "dns_name", block.Endpoint.DNSName,
+		"assigned_ip", block.AssignedIP, "action", verb[0])
+}
+
+// resolveEndpointLocked returns the endpoint IPv4 to render. Steady state uses the CACHED IP — no
+// DNS call while the tunnel is healthy (the watchdog owns re-resolution, doc 06 §4.2). The cache is
+// empty on first apply and after a process restart → one resolve then. On resolver failure with a
+// cache present the caller keeps the last conf (never a teardown); with no cache yet it simply
+// can't apply and retries next tick.
+func (m *Manager) resolveEndpointLocked(ctx context.Context, block *hub.WireWireguard) (string, bool) {
+	if m.lastResolvedIP.IsValid() {
+		return m.lastResolvedIP.String(), true
+	}
+	pick, ok := m.resolveNowLocked(ctx, block)
+	if !ok {
+		return "", false
+	}
+	m.lastResolvedIP = pick
+	return pick.String(), true
+}
+
+// resolveNowLocked forces a fresh A-record lookup, picks the lowest, and throttles failure logging.
+// Never returns an AAAA (the resolver asks for "ip4"). Returns ok=false on a bad name or lookup
+// failure — the caller decides whether that means "keep last conf" or "can't apply yet".
+func (m *Manager) resolveNowLocked(ctx context.Context, block *hub.WireWireguard) (netip.Addr, bool) {
+	name := block.Endpoint.DNSName
+	if !dnsNameRe.MatchString(name) {
+		m.logger.Error("wgtunnel: endpoint dns_name has invalid characters — not resolving", "dns_name", name)
+		return netip.Addr{}, false
+	}
+	addrs, err := m.resolver.LookupIPv4(ctx, name)
+	if err != nil || len(addrs) == 0 {
+		if !m.resolveFailLogged {
+			m.logger.Error("wgtunnel: endpoint A-record resolution failed — keeping last-applied conf (DNS failure is NEVER a teardown)",
+				"err", err, "dns_name", name)
+			m.resolveFailLogged = true
+		}
+		return netip.Addr{}, false
+	}
+	m.resolveFailLogged = false
+	return lowestAddr(addrs), true
+}
+
+// handshakeStaleLocked reports whether the live tunnel's last handshake is older than staleAfter —
+// the watchdog's re-resolve trigger. Inactive tunnel or no-handshake-yet → NOT stale (the ensure
+// path handles those). This is the ONLY place a per-tick `wg show` runs outside Status; it lives in
+// Watchdog (loop-only) so Apply's steady state stays zero-exec.
+func (m *Manager) handshakeStaleLocked(ctx context.Context) bool {
+	if !m.isActive(ctx) {
+		return false
+	}
+	out, _, err := m.runner.Run(ctx, "wg", "show", Iface, "latest-handshakes")
+	if err != nil {
+		return false
+	}
+	age, ok := parseHandshakeAge(string(out), m.now())
+	if !ok {
+		return false
+	}
+	return age > int64(m.staleAfter.Seconds())
+}
+
+// Watchdog re-resolves the endpoint when the tunnel's handshake has gone stale and re-applies on an
+// IP change — recovery from an endpoint re-IP or a family flap (doc 06 §4.2). Loop-driven only:
+// Apply never calls it, so Apply's steady state stays DNS-free and exec-free. A stale handshake with
+// an UNCHANGED IP (endpoint merely down) causes no churn — one throttled log, no restart.
+func (m *Manager) Watchdog(ctx context.Context, fetched bool, block *hub.WireWireguard) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if !fetched || block == nil || !m.lastResolvedIP.IsValid() {
+		return // nothing applied to watch, or no desired block
+	}
+	if !m.handshakeStaleLocked(ctx) {
+		return // healthy → no DNS, no action (the load-bearing negative)
+	}
+	pick, ok := m.resolveNowLocked(ctx, block)
+	if !ok {
+		return // resolver failed while stale → keep last conf, retry next tick
+	}
+	if pick == m.lastResolvedIP {
+		if !m.staleSameIPLogged {
+			m.logger.Warn("wgtunnel: tunnel handshake stale but endpoint IP unchanged — endpoint may be down; not restarting",
+				"endpoint_ip", pick.String())
+			m.staleSameIPLogged = true
+		}
+		return
+	}
+	m.logger.Info("wgtunnel: endpoint re-IP detected — re-resolving and re-applying",
+		"old", m.lastResolvedIP.String(), "new", pick.String(), "dns_name", block.Endpoint.DNSName)
+	m.lastResolvedIP = pick
+	m.staleSameIPLogged = false
+	m.ensureTunnelLocked(ctx, block) // re-renders with the new cached IP → restart
 }

 // ensureToolsLocked installs wireguard-tools once when absent (dnsmasq-install precedent).