felhom-agent/internal/hub/cloudflared.go

package hub

import (
	"context"
	"fmt"
	"strings"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// Tunnel states the agent reports (R-841, agent v0.141.0). THREE, never two: a probe that could not ask is
// `unknown`, which the hub never shows as up or down and never alarms on (R-96 rule 3).
const (
	TunnelRunning    = "running"     // the cloudflared container runs AND its readiness check says CONNECTED
	TunnelNotRunning = "not_running" // stopped / exited / absent, or running but NOT connected (Detail says which)
	TunnelUnknown    = "unknown"     // the probe could not ask (guest down, pct/sudo error, health still starting)
)

// CloudflaredProber reports the box's tunnel. Injectable so tests use a fake and never exec.
type CloudflaredProber interface {
	// Status returns one of the Tunnel* states and a short detail (why not_running / why unknown).
	Status(ctx context.Context) (status, detail string)
}

// GuestTunnelProber reads the REAL tunnel: the `cloudflared` container in the box's own customer guest.
//
// Before v0.141.0 the agent ran `systemctl is-active cloudflared` on the HOST — a unit that does not exist (cloudflared
// is a guest container, `11-os-updates.md` C8), so every box reported `inactive` (R-841).
//
// It uses ONLY the existing sudoers line `pct exec [0-9]* -- docker inspect -f *` (03 §3): the container's state, exit
// code and Docker health status. The health status comes from the compose health check controller v0.292.0 adds
// (`cloudflared tunnel --metrics localhost:20241 ready` → /ready: 200 only with ≥ 1 connection). Measured 2026-10-04:
// with a wrong token the container stays "running" while /ready answers 503 — so the container state alone would lie.
// A container with no health check (an older controller) is judged on its state alone, and Detail says so.
type GuestTunnelProber struct {
	Runner proxmox.Runner
	// Guests returns the box's customer guest vmids (running pool guests that bind /mnt/felhom-drives).
	Guests func(ctx context.Context) ([]int, error)
}

const tunnelInspect = `{{.State.Status}}|{{.State.ExitCode}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}`

// Status probes every customer guest and reports the worst state (normally there is exactly one guest).
func (p GuestTunnelProber) Status(ctx context.Context) (string, string) {
	if p.Runner == nil || p.Guests == nil {
		return TunnelUnknown, "no probe wired"
	}
	vmids, err := p.Guests(ctx)
	if err != nil {
		return TunnelUnknown, "could not list the customer guest: " + err.Error()
	}
	if len(vmids) == 0 {
		return TunnelUnknown, "no running customer guest"
	}
	worst, wdetail := "", ""
	rank := map[string]int{TunnelRunning: 0, TunnelUnknown: 1, TunnelNotRunning: 2}
	for _, v := range vmids {
		out, errOut, err := p.Runner.Run(ctx, "/usr/sbin/pct", "exec", fmt.Sprint(v), "--", "docker", "inspect", "-f", tunnelInspect, "cloudflared")
		st, d := ClassifyTunnel(string(out), string(errOut), err)
		if len(vmids) > 1 {
			d = fmt.Sprintf("guest %d: %s", v, d)
		}
		if worst == "" || rank[st] > rank[worst] {
			worst, wdetail = st, d
		}
	}
	return worst, wdetail
}

// ClassifyTunnel maps one `docker inspect` answer to a state. Pure; pinned by TestClassifyTunnel.
func ClassifyTunnel(stdout, stderr string, err error) (string, string) {
	out := strings.TrimSpace(stdout)
	if err != nil || out == "" {
		if strings.Contains(stderr, "No such object") || strings.Contains(stderr, "No such container") {
			return TunnelNotRunning, "no cloudflared container in the guest"
		}
		return TunnelUnknown, "could not ask the guest: " + firstLine(stderr, err)
	}
	parts := strings.Split(out, "|")
	if len(parts) != 3 {
		return TunnelUnknown, "unreadable docker answer: " + out
	}
	state, code, health := parts[0], parts[1], parts[2]
	if state != "running" {
		return TunnelNotRunning, fmt.Sprintf("container %s, exit code %s", state, code)
	}
	switch health {
	case "healthy":
		return TunnelRunning, "connected"
	case "unhealthy":
		return TunnelNotRunning, "container running but the tunnel is NOT connected (cloudflared /ready fails)"
	case "starting":
		return TunnelUnknown, "container running, readiness check still starting"
	case "none":
		return TunnelRunning, "container running (no readiness check on this controller — connection not checked)"
	}
	return TunnelUnknown, "unknown health state " + health
}

func firstLine(stderr string, err error) string {
	s := strings.TrimSpace(stderr)
	if i := strings.IndexByte(s, '\n'); i >= 0 {
		s = s[:i]
	}
	if s == "" && err != nil {
		s = err.Error()
	}
	if len(s) > 160 {
		s = s[:160]
	}
	return s
}