v0.21.0: agent-managed split-horizon LAN resolver (internal/lanresolver)

Host-side dnsmasq the agent manages so LAN clients reach their guest directly
(same hostname + real wildcard cert, no Cloudflare hairpin). Renders local=/
+address=/ per customer (AAAA->NODATA via authoritative zone, wildcard A ->
live guest IP), forwards everything else. Manager ensures dnsmasq+base config,
discovers guest IP (pct exec ip) + domain (controller.yaml), write-if-changed +
reload. Loop (7th daemon goroutine) tracks DHCP IP changes per provisioned
guest. --selftest=lanresolver. FELHOM_DNSMASQ sudoers. Spiked live on felhom-pve.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-11 18:24:58 +02:00
parent 621a09a1c5
commit a43e9813ad
7 changed files with 642 additions and 10 deletions
+38 -3
View File
@@ -28,9 +28,44 @@ type Config struct {
Hub HubConfig `json:"hub"`
Storage StorageConfig `json:"storage"`
Backup BackupConfig `json:"backup"`
Escrow EscrowConfig `json:"escrow"`
LocalAPI LocalAPIConfig `json:"local_api"`
LogLevel string `json:"log_level"` // debug|info|warn|error (default info)
Escrow EscrowConfig `json:"escrow"`
LocalAPI LocalAPIConfig `json:"local_api"`
LANResolver LANResolverConfig `json:"lan_resolver"`
LogLevel string `json:"log_level"` // debug|info|warn|error (default info)
}
// LANResolverConfig configures the host-level split-horizon DNS resolver (internal/lanresolver): a
// dnsmasq the agent manages so LAN clients reach their guest DIRECTLY at the same hostname + real cert.
// Disabled unless Enable is set. HostIP defaults to the local-API bridge IP (the host LAN anchor);
// Upstreams default to public resolvers; the loop re-checks the guest's live IP every interval.
type LANResolverConfig struct {
Enable bool `json:"enable"`
HostIP string `json:"host_ip"` // dnsmasq listen-address; default = LocalAPI bridge IP
Upstreams []string `json:"upstreams"` // forward targets for non-customer names
IntervalSeconds int `json:"interval_seconds"` // IP-freshness re-check cadence; default 300
StateDir string `json:"state_dir"` // provisioned guests under <StateDir>/guests/; default /var/lib/felhom-agent
}
// Enabled reports whether the split-horizon resolver should run.
func (l LANResolverConfig) Enabled() bool { return l.Enable }
// WithDefaults fills upstreams/interval/state-dir and derives HostIP from the local-API bind addr.
func (l LANResolverConfig) WithDefaults(localAPIListen string) LANResolverConfig {
if len(l.Upstreams) == 0 {
l.Upstreams = []string{"1.1.1.1", "8.8.8.8"}
}
if l.IntervalSeconds == 0 {
l.IntervalSeconds = 300
}
if l.StateDir == "" {
l.StateDir = "/var/lib/felhom-agent"
}
if strings.TrimSpace(l.HostIP) == "" && localAPIListen != "" {
if h, _, err := net.SplitHostPort(localAPIListen); err == nil && h != "" && h != "0.0.0.0" {
l.HostIP = h
}
}
return l
}
// LocalAPIConfig configures the per-guest local API server (doc 03 §6, slice 8A). The
+263
View File
@@ -0,0 +1,263 @@
// Package lanresolver manages a host-level split-horizon DNS resolver (dnsmasq) so LAN clients reach
// the customer's guest DIRECTLY (not hairpinning through Cloudflare) at the SAME public hostname with
// the SAME real wildcard cert. The agent is the natural owner: it runs on the host (the stable LAN
// anchor — static IP, unlike the DHCP/ephemeral guest), and it can read the guest's live IP + domain.
//
// Mechanism (proven in the slice spike): dnsmasq answers `*.<customer-domain>` with the guest's live
// LAN IP and returns NODATA for AAAA (the guest has only link-local v6 — no Cloudflare-AAAA leak),
// forwarding everything else upstream. Two pieces of config:
// - a BASE drop-in (felhom-resolver-base.conf): bind to the host LAN IP, no-resolv, upstreams.
// - a per-customer drop-in (felhom-<customer-id>.conf): `local=/<domain>/` + `address=/<domain>/<ip>`.
//
// `local=/<domain>/` (trailing slash, no server) makes dnsmasq AUTHORITATIVE for the zone → AAAA
// returns NODATA instead of forwarding; `address=/<domain>/<ip>` is the wildcard A. Non-customer names
// (and their AAAA) still forward normally, so the resolver can serve as the LAN's general resolver.
package lanresolver
import (
"context"
"fmt"
"log/slog"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)
const (
// DropinDir is where dnsmasq reads conf-dir drop-ins (Debian default).
DropinDir = "/etc/dnsmasq.d"
// BaseDropin holds the host-wide listen/upstream config (one per host).
BaseDropin = "felhom-resolver-base.conf"
)
// RenderBase returns the host-wide dnsmasq drop-in: bind to the host LAN IP (+ loopback), no-resolv,
// and the explicit upstreams everything-else forwards to. Kept separate from the per-customer
// drop-ins so the global directives aren't duplicated across guests on a multi-guest host.
func RenderBase(hostIP string, upstreams []string) string {
var b strings.Builder
b.WriteString("# felhom split-horizon resolver — host base config (agent-managed; DO NOT EDIT)\n")
b.WriteString("# Bound to the host LAN IP so LAN clients (or the LAN's forwarder) reach it here.\n")
b.WriteString("bind-interfaces\n")
fmt.Fprintf(&b, "listen-address=%s\n", hostIP)
b.WriteString("listen-address=127.0.0.1\n")
b.WriteString("no-resolv\n")
for _, up := range upstreams {
fmt.Fprintf(&b, "server=%s\n", up)
}
return b.String()
}
// RenderGuestDropin returns the per-customer split-horizon drop-in. `local=` makes dnsmasq
// authoritative for the zone (→ AAAA NODATA, no upstream leak); `address=` is the wildcard A → guest IP.
func RenderGuestDropin(customerID, domain, guestIP string) string {
return fmt.Sprintf(`# felhom split-horizon DNS — customer %s (agent-managed; DO NOT EDIT)
# Answer *.%s with the guest's live LAN IP; AAAA → NODATA; everything else forwards upstream.
local=/%s/
address=/%s/%s
`, customerID, domain, domain, domain, guestIP)
}
// DropinName is the per-customer drop-in filename (sanitized customer id).
func DropinName(customerID string) string {
return "felhom-" + sanitize(customerID) + ".conf"
}
var unsafeName = regexp.MustCompile(`[^a-zA-Z0-9._-]+`)
func sanitize(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
s = unsafeName.ReplaceAllString(s, "-")
return strings.Trim(s, "-.") // also strip leading/trailing dots (no "..", no path-ish names)
}
// Manager renders + applies the resolver config and reloads dnsmasq. It uses the same fenced Runner as
// the rest of the agent (direct as root on the demo; sudo + a narrow allowlist in the hardened model).
type Manager struct {
runner proxmox.Runner
hostIP string
upstreams []string
logger *slog.Logger
mu sync.Mutex
lastIP map[int]string // vmid → last applied guest IP (for transition logging + change detection)
}
// NewManager builds a Manager. hostIP is the host LAN IP dnsmasq binds to (the stable anchor);
// upstreams are the forward targets for non-customer names (defaults applied by the caller).
func NewManager(runner proxmox.Runner, hostIP string, upstreams []string, logger *slog.Logger) *Manager {
return &Manager{
runner: runner,
hostIP: hostIP,
upstreams: upstreams,
logger: logger,
lastIP: map[int]string{},
}
}
// EnsureDnsmasq makes dnsmasq present + enabled and writes the host base config. Idempotent: it
// installs the package only when absent, and writes the base drop-in only when its content changes.
func (m *Manager) EnsureDnsmasq(ctx context.Context) error {
if _, err := os.Stat("/usr/sbin/dnsmasq"); err != nil { // metadata read, no privilege needed
m.logger.Info("lanresolver: dnsmasq absent — installing")
if out, errOut, ierr := m.runner.Run(ctx, "apt-get", "install", "-y", "-q", "dnsmasq"); ierr != nil {
return fmt.Errorf("install dnsmasq: %s: %w", strings.TrimSpace(string(errOut))+string(out), ierr)
}
}
base := RenderBase(m.hostIP, m.upstreams)
changed, err := m.writeFileIfChanged(ctx, filepath.Join(DropinDir, BaseDropin), base, "0644")
if err != nil {
return fmt.Errorf("write base config: %w", err)
}
// enable + start (idempotent); reload if the base changed.
if _, _, err := m.runner.Run(ctx, "systemctl", "enable", "--now", "dnsmasq"); err != nil {
return fmt.Errorf("enable dnsmasq: %w", err)
}
if changed {
return m.reload(ctx)
}
return nil
}
// ReconcileGuest discovers the guest's live IP + domain and applies the per-customer drop-in (writing
// + reloading only on change). It tolerates the early-boot pre-lease window: an empty IP is a no-op
// (logged), never a blank/zero `address=` record. Logs IP transitions.
func (m *Manager) ReconcileGuest(ctx context.Context, vmid int, customerID string) error {
ip, err := m.discoverGuestIP(ctx, vmid)
if err != nil {
return fmt.Errorf("discover guest %d IP: %w", vmid, err)
}
if ip == "" {
m.logger.Info("lanresolver: guest IP not yet leased — skipping (will retry)", "vmid", vmid)
return nil
}
domain, err := m.discoverDomain(ctx, vmid)
if err != nil {
return fmt.Errorf("discover guest %d domain: %w", vmid, err)
}
if domain == "" {
m.logger.Info("lanresolver: guest domain not available yet — skipping (will retry)", "vmid", vmid)
return nil
}
if customerID == "" {
customerID = "guest-" + strconv.Itoa(vmid)
}
m.mu.Lock()
prev := m.lastIP[vmid]
m.mu.Unlock()
dropin := RenderGuestDropin(customerID, domain, ip)
changed, err := m.writeFileIfChanged(ctx, filepath.Join(DropinDir, DropinName(customerID)), dropin, "0644")
if err != nil {
return fmt.Errorf("write guest %d drop-in: %w", vmid, err)
}
if changed {
if prev != "" && prev != ip {
m.logger.Info("lanresolver: guest IP changed — updating resolver", "vmid", vmid, "customer", customerID, "domain", domain, "from", prev, "to", ip)
} else {
m.logger.Info("lanresolver: applied split-horizon record", "vmid", vmid, "customer", customerID, "domain", domain, "ip", ip)
}
if err := m.reload(ctx); err != nil {
return err
}
}
m.mu.Lock()
m.lastIP[vmid] = ip
m.mu.Unlock()
return nil
}
// Remove deletes a customer's drop-in (decommission) and reloads.
func (m *Manager) Remove(ctx context.Context, customerID string) error {
path := filepath.Join(DropinDir, DropinName(customerID))
if _, _, err := m.runner.Run(ctx, "rm", "-f", path); err != nil {
return fmt.Errorf("remove drop-in: %w", err)
}
return m.reload(ctx)
}
// discoverGuestIP runs `pct exec <vmid> -- ip -4 -o addr show dev eth0` and parses the inet address.
// Returns "" (no error) when no IPv4 is leased yet — the agent's IP tracking must tolerate this.
func (m *Manager) discoverGuestIP(ctx context.Context, vmid int) (string, error) {
out, errOut, err := m.runner.Run(ctx, "pct", "exec", strconv.Itoa(vmid), "--", "ip", "-4", "-o", "addr", "show", "dev", "eth0")
if err != nil {
// A stopped/early-boot guest can fail here; treat as "not ready" rather than a hard error
// only if stderr looks like a transient state. Otherwise surface it.
es := strings.TrimSpace(string(errOut))
if strings.Contains(es, "not running") || strings.Contains(es, "Configuration file") {
return "", nil
}
return "", fmt.Errorf("pct exec ip: %s: %w", es, err)
}
return parseInet(string(out)), nil
}
var inetRe = regexp.MustCompile(`inet\s+(\d{1,3}(?:\.\d{1,3}){3})/\d+`)
func parseInet(s string) string {
if mtch := inetRe.FindStringSubmatch(s); mtch != nil {
return mtch[1]
}
return ""
}
// discoverDomain reads customer.domain from the guest controller's pulled controller.yaml.
// (The v2 bootstrap.json deliberately omits the domain — it comes from the hub pull — so the agent
// reads it back from the running controller; no new credential.)
func (m *Manager) discoverDomain(ctx context.Context, vmid int) (string, error) {
out, errOut, err := m.runner.Run(ctx, "pct", "exec", strconv.Itoa(vmid), "--",
"docker", "exec", "felhom-controller", "cat", "/opt/docker/felhom-controller/controller.yaml")
if err != nil {
es := strings.TrimSpace(string(errOut))
if strings.Contains(es, "not running") || strings.Contains(es, "No such container") || strings.Contains(es, "Configuration file") {
return "", nil // controller not up yet — retry later
}
return "", fmt.Errorf("pct exec cat controller.yaml: %s: %w", es, err)
}
return parseDomain(string(out)), nil
}
var domainRe = regexp.MustCompile(`(?m)^[ \t]*domain:[ \t]*"?([A-Za-z0-9.-]+)"?`)
func parseDomain(s string) string {
if mtch := domainRe.FindStringSubmatch(s); mtch != nil {
return mtch[1]
}
return ""
}
func (m *Manager) reload(ctx context.Context) error {
if _, errOut, err := m.runner.Run(ctx, "systemctl", "reload", "dnsmasq"); err != nil {
return fmt.Errorf("reload dnsmasq: %s: %w", strings.TrimSpace(string(errOut)), err)
}
return nil
}
// writeFileIfChanged writes content to path (mode) only when the current content differs, returning
// whether it changed. It writes via a temp file + `install` so it works both as root (direct) and via
// the sudo allowlist (install runs privileged); the file is world-readable so the compare reads directly.
func (m *Manager) writeFileIfChanged(ctx context.Context, path, content, mode string) (bool, error) {
if cur, err := os.ReadFile(path); err == nil && string(cur) == content {
return false, nil
}
tmp, err := os.CreateTemp("", "felhom-resolver-*.conf")
if err != nil {
return false, fmt.Errorf("temp file: %w", err)
}
tmpName := tmp.Name()
defer os.Remove(tmpName)
if _, err := tmp.WriteString(content); err != nil {
tmp.Close()
return false, fmt.Errorf("write temp: %w", err)
}
tmp.Close()
if _, errOut, err := m.runner.Run(ctx, "install", "-m", mode, tmpName, path); err != nil {
return false, fmt.Errorf("install %s: %s: %w", path, strings.TrimSpace(string(errOut)), err)
}
return true, nil
}
+80
View File
@@ -0,0 +1,80 @@
package lanresolver
import (
"strings"
"testing"
)
func TestRenderGuestDropin(t *testing.T) {
d := RenderGuestDropin("demo-felhom", "demo-felhom.eu", "192.168.0.151")
// The load-bearing pair: local= makes dnsmasq authoritative (AAAA→NODATA), address= is the wildcard A.
if !strings.Contains(d, "local=/demo-felhom.eu/\n") {
t.Errorf("missing authoritative local= line (AAAA suppression):\n%s", d)
}
if !strings.Contains(d, "address=/demo-felhom.eu/192.168.0.151\n") {
t.Errorf("missing wildcard address= line:\n%s", d)
}
}
func TestRenderBase(t *testing.T) {
b := RenderBase("192.168.0.162", []string{"1.1.1.1", "8.8.8.8"})
for _, want := range []string{
"bind-interfaces\n", "listen-address=192.168.0.162\n", "listen-address=127.0.0.1\n",
"no-resolv\n", "server=1.1.1.1\n", "server=8.8.8.8\n",
} {
if !strings.Contains(b, want) {
t.Errorf("base config missing %q:\n%s", want, b)
}
}
}
func TestParseInet(t *testing.T) {
// Exact shape of `pct exec <vmid> -- ip -4 -o addr show dev eth0` (verified live).
live := `2: eth0 inet 192.168.0.151/24 brd 192.168.0.255 scope global dynamic eth0\ valid_lft 4765sec preferred_lft 4765sec`
if got := parseInet(live); got != "192.168.0.151" {
t.Errorf("parseInet = %q, want 192.168.0.151", got)
}
// Pre-lease window: no inet line → empty (must NOT be treated as a real IP).
if got := parseInet(`2: eth0 inet6 fe80::be24:11ff:fec7:409/64 scope link`); got != "" {
t.Errorf("parseInet(no v4) = %q, want empty", got)
}
if got := parseInet(""); got != "" {
t.Errorf("parseInet(empty) = %q, want empty", got)
}
}
func TestParseDomain(t *testing.T) {
yaml := `customer:
domain: demo-felhom.eu
id: demo-felhom
email: admin@felhom.eu
`
if got := parseDomain(yaml); got != "demo-felhom.eu" {
t.Errorf("parseDomain = %q, want demo-felhom.eu", got)
}
if got := parseDomain(` domain: "quoted-domain.eu"`); got != "quoted-domain.eu" {
t.Errorf("parseDomain(quoted) = %q, want quoted-domain.eu", got)
}
if got := parseDomain("no domain here\n"); got != "" {
t.Errorf("parseDomain(absent) = %q, want empty", got)
}
}
func TestDropinNameSanitize(t *testing.T) {
if got := DropinName("demo-felhom"); got != "felhom-demo-felhom.conf" {
t.Errorf("DropinName = %q", got)
}
// Hostile/odd id collapses to a safe filename.
if got := DropinName("../evil id"); got != "felhom-evil-id.conf" {
t.Errorf("DropinName(hostile) = %q", got)
}
}
func TestDeriveHostIP(t *testing.T) {
if got := DeriveHostIP("192.168.0.162:8443"); got != "192.168.0.162" {
t.Errorf("DeriveHostIP = %q, want 192.168.0.162", got)
}
if got := DeriveHostIP("0.0.0.0:8443"); got != "" {
t.Errorf("DeriveHostIP(0.0.0.0) = %q, want empty (require explicit)", got)
}
}
+145
View File
@@ -0,0 +1,145 @@
package lanresolver
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"os"
"path/filepath"
"strconv"
"time"
)
// Loop periodically reconciles the split-horizon resolver for every guest the agent has provisioned.
// It is the IP-freshness mechanism: the guest stays DHCP/ephemeral, so its LAN IP can move (lease
// renewal, MAC reset); the loop re-discovers it each tick and updates the resolver only on change.
// Like the storage watchdog / PBS verify loop, it owns its own cadence and is not journaled/gated.
type Loop struct {
mgr *Manager
stateDir string // agent state dir; provisioned guests live under <stateDir>/guests/<vmid>/
interval time.Duration
logger *slog.Logger
}
// NewLoop builds the reconcile loop. stateDir is the agent state dir (default /var/lib/felhom-agent).
func NewLoop(mgr *Manager, stateDir string, interval time.Duration, logger *slog.Logger) *Loop {
if interval <= 0 {
interval = 5 * time.Minute
}
return &Loop{mgr: mgr, stateDir: stateDir, interval: interval, logger: logger}
}
// Run ensures dnsmasq + the base config once, then reconciles all guests immediately and every
// interval until ctx is cancelled.
func (l *Loop) Run(ctx context.Context) error {
if err := l.mgr.EnsureDnsmasq(ctx); err != nil {
// Non-fatal: log and keep trying on the ticker (a transient apt/systemd hiccup must not kill
// the loop). The host's own resolution is unaffected (we never touch /etc/resolv.conf).
l.logger.Warn("lanresolver: EnsureDnsmasq failed — will retry", "err", err)
}
l.reconcileAll(ctx)
t := time.NewTicker(l.interval)
defer t.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
l.reconcileAll(ctx)
}
}
}
func (l *Loop) reconcileAll(ctx context.Context) {
guests, err := l.listGuests()
if err != nil {
l.logger.Warn("lanresolver: cannot list provisioned guests", "err", err)
return
}
for _, g := range guests {
if err := l.mgr.ReconcileGuest(ctx, g.VMID, g.CustomerID); err != nil {
l.logger.Warn("lanresolver: reconcile failed", "vmid", g.VMID, "err", err)
}
}
}
type guestRef struct {
VMID int
CustomerID string
}
// listGuests enumerates the provisioned guests under <stateDir>/guests/<vmid>/bootstrap/bootstrap.json,
// reading customer.id from each bootstrap. Dirs without a readable bootstrap are skipped.
func (l *Loop) listGuests() ([]guestRef, error) {
root := filepath.Join(l.stateDir, "guests")
entries, err := os.ReadDir(root)
if err != nil {
if os.IsNotExist(err) {
return nil, nil // no guests provisioned yet
}
return nil, err
}
var out []guestRef
for _, e := range entries {
if !e.IsDir() {
continue
}
vmid, err := strconv.Atoi(e.Name())
if err != nil {
continue
}
cid := readCustomerID(filepath.Join(root, e.Name(), "bootstrap", "bootstrap.json"))
out = append(out, guestRef{VMID: vmid, CustomerID: cid})
}
return out, nil
}
// CustomerID returns a provisioned guest's customer id from its bootstrap (best-effort "").
func CustomerID(stateDir string, vmid int) string {
return readCustomerID(filepath.Join(stateDir, "guests", strconv.Itoa(vmid), "bootstrap", "bootstrap.json"))
}
func readCustomerID(path string) string {
data, err := os.ReadFile(path)
if err != nil {
return ""
}
var b struct {
Customer struct {
ID string `json:"id"`
} `json:"customer"`
}
if err := json.Unmarshal(data, &b); err != nil {
return ""
}
return b.Customer.ID
}
// DeriveHostIP extracts the host LAN IP from the local-API listen address (host:port). Returns "" if
// it can't be parsed (caller then requires an explicit config value).
func DeriveHostIP(listenAddr string) string {
h, _, err := splitHostPort(listenAddr)
if err != nil || h == "" || h == "0.0.0.0" {
return ""
}
return h
}
// splitHostPort is net.SplitHostPort wrapped to avoid importing net in tests that don't need it.
func splitHostPort(s string) (string, string, error) {
i := lastIndexByte(s, ':')
if i < 0 {
return "", "", fmt.Errorf("missing port in %q", s)
}
return s[:i], s[i+1:], nil
}
func lastIndexByte(s string, b byte) int {
for i := len(s) - 1; i >= 0; i-- {
if s[i] == b {
return i
}
}
return -1
}