OS updates host fast lane + true tunnel status + fast leg: wrapper host layer (R12 appliance proof from the root-owned install record, R14 kernel/boot/firmware refused), select pending-fast, one call per layer, host-side version checks, restart scan only after an install, reboot-needed for PID 1/lxc-start; the leg runs the host step after a healthy guest step; GuestTunnelProber reads the cloudflared container + its readiness check (R-841)
gates / gates (push) Successful in 19s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-04 12:54:02 +02:00
parent a55eedcf2c
commit c3d08b4821
15 changed files with 929 additions and 403 deletions
+97 -31
View File
@@ -2,45 +2,111 @@ package hub
import (
"context"
"os/exec"
"fmt"
"strings"
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)
// CloudflaredProber reports the cloudflared tunnel service health. It is a
// READ-ONLY probe: the agent does NOT manage or restart cloudflared in this slice
// (that is the tunnel-management slice — this is the seam for it). Injectable so
// tests use a fake and never exec.
// Tunnel states the agent reports (R-841, agent v0.141.0). THREE, never two: a probe that could not ask is
// `unknown`, which the hub never shows as up or down and never alarms on (R-96 rule 3).
const (
TunnelRunning = "running" // the cloudflared container runs AND its readiness check says CONNECTED
TunnelNotRunning = "not_running" // stopped / exited / absent, or running but NOT connected (Detail says which)
TunnelUnknown = "unknown" // the probe could not ask (guest down, pct/sudo error, health still starting)
)
// CloudflaredProber reports the box's tunnel. Injectable so tests use a fake and never exec.
type CloudflaredProber interface {
// Status returns one of: "active" | "inactive" | "failed" | "unknown".
Status(ctx context.Context) (string, error)
// Status returns one of the Tunnel* states and a short detail (why not_running / why unknown).
Status(ctx context.Context) (status, detail string)
}
// SystemctlProber runs `systemctl is-active cloudflared`. This is NOT a Privileged
// (root-CLI) op — `is-active` is non-root readable and is not one of the three
// proven root exceptions, so it does not go through internal/proxmox.Privileged.
type SystemctlProber struct {
Unit string // defaults to "cloudflared"
// GuestTunnelProber reads the REAL tunnel: the `cloudflared` container in the box's own customer guest.
//
// Before v0.141.0 the agent ran `systemctl is-active cloudflared` on the HOST — a unit that does not exist (cloudflared
// is a guest container, `11-os-updates.md` C8), so every box reported `inactive` (R-841).
//
// It uses ONLY the existing sudoers line `pct exec [0-9]* -- docker inspect -f *` (03 §3): the container's state, exit
// code and Docker health status. The health status comes from the compose health check controller v0.292.0 adds
// (`cloudflared tunnel --metrics localhost:20241 ready` → /ready: 200 only with ≥ 1 connection). Measured 2026-10-04:
// with a wrong token the container stays "running" while /ready answers 503 — so the container state alone would lie.
// A container with no health check (an older controller) is judged on its state alone, and Detail says so.
type GuestTunnelProber struct {
Runner proxmox.Runner
// Guests returns the box's customer guest vmids (running pool guests that bind /mnt/felhom-drives).
Guests func(ctx context.Context) ([]int, error)
}
// Status maps `systemctl is-active` output to the report vocabulary. systemctl
// exits non-zero for inactive/failed, so the output string is authoritative over
// the exit code; any exec error (binary missing, etc.) maps to "unknown".
func (p SystemctlProber) Status(ctx context.Context) (string, error) {
unit := p.Unit
if unit == "" {
unit = "cloudflared"
const tunnelInspect = `{{.State.Status}}|{{.State.ExitCode}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}`
// Status probes every customer guest and reports the worst state (normally there is exactly one guest).
func (p GuestTunnelProber) Status(ctx context.Context) (string, string) {
if p.Runner == nil || p.Guests == nil {
return TunnelUnknown, "no probe wired"
}
out, _ := exec.CommandContext(ctx, "systemctl", "is-active", unit).Output()
switch strings.TrimSpace(string(out)) {
case "active":
return "active", nil
case "failed":
return "failed", nil
case "inactive", "deactivating", "activating":
return "inactive", nil
case "":
return "unknown", nil // no output → systemctl/exec problem
default:
return "unknown", nil
vmids, err := p.Guests(ctx)
if err != nil {
return TunnelUnknown, "could not list the customer guest: " + err.Error()
}
if len(vmids) == 0 {
return TunnelUnknown, "no running customer guest"
}
worst, wdetail := "", ""
rank := map[string]int{TunnelRunning: 0, TunnelUnknown: 1, TunnelNotRunning: 2}
for _, v := range vmids {
out, errOut, err := p.Runner.Run(ctx, "/usr/sbin/pct", "exec", fmt.Sprint(v), "--", "docker", "inspect", "-f", tunnelInspect, "cloudflared")
st, d := ClassifyTunnel(string(out), string(errOut), err)
if len(vmids) > 1 {
d = fmt.Sprintf("guest %d: %s", v, d)
}
if worst == "" || rank[st] > rank[worst] {
worst, wdetail = st, d
}
}
return worst, wdetail
}
// ClassifyTunnel maps one `docker inspect` answer to a state. Pure; pinned by TestClassifyTunnel.
func ClassifyTunnel(stdout, stderr string, err error) (string, string) {
out := strings.TrimSpace(stdout)
if err != nil || out == "" {
if strings.Contains(stderr, "No such object") || strings.Contains(stderr, "No such container") {
return TunnelNotRunning, "no cloudflared container in the guest"
}
return TunnelUnknown, "could not ask the guest: " + firstLine(stderr, err)
}
parts := strings.Split(out, "|")
if len(parts) != 3 {
return TunnelUnknown, "unreadable docker answer: " + out
}
state, code, health := parts[0], parts[1], parts[2]
if state != "running" {
return TunnelNotRunning, fmt.Sprintf("container %s, exit code %s", state, code)
}
switch health {
case "healthy":
return TunnelRunning, "connected"
case "unhealthy":
return TunnelNotRunning, "container running but the tunnel is NOT connected (cloudflared /ready fails)"
case "starting":
return TunnelUnknown, "container running, readiness check still starting"
case "none":
return TunnelRunning, "container running (no readiness check on this controller — connection not checked)"
}
return TunnelUnknown, "unknown health state " + health
}
func firstLine(stderr string, err error) string {
s := strings.TrimSpace(stderr)
if i := strings.IndexByte(s, '\n'); i >= 0 {
s = s[:i]
}
if s == "" && err != nil {
s = err.Error()
}
if len(s) > 160 {
s = s[:160]
}
return s
}
+67
View File
@@ -0,0 +1,67 @@
package hub
import (
"context"
"errors"
"io"
"strings"
"testing"
)
// R-841: the three states from one `docker inspect` answer. Red-proof: map "unhealthy" to running (the container
// state alone — what a plain "is it running" probe would say) and the "running but not connected" case fails.
func TestClassifyTunnel(t *testing.T) {
cases := []struct {
name, out, errOut string
err error
want string
detail string
}{
{"connected", "running|0|healthy\n", "", nil, TunnelRunning, "connected"},
{"running but not connected", "running|0|unhealthy\n", "", nil, TunnelNotRunning, "NOT connected"},
{"stopped", "exited|137|unhealthy\n", "", nil, TunnelNotRunning, "exit code 137"},
{"absent", "", "Error: No such object: cloudflared", errors.New("exit status 1"), TunnelNotRunning, "no cloudflared container"},
{"still starting", "running|0|starting\n", "", nil, TunnelUnknown, "starting"},
{"no health check (older controller)", "running|0|none\n", "", nil, TunnelRunning, "connection not checked"},
{"guest not running", "", "CT 9201 not running", errors.New("exit status 255"), TunnelUnknown, "could not ask"},
{"sudo refused", "", "sudo: a password is required", errors.New("exit status 1"), TunnelUnknown, "could not ask"},
}
for _, c := range cases {
st, d := ClassifyTunnel(c.out, c.errOut, c.err)
if st != c.want || !strings.Contains(d, c.detail) {
t.Errorf("%s: got %q (%s), want %q (…%s…)", c.name, st, d, c.want, c.detail)
}
}
}
type tunnelRunner struct {
calls []string
out map[string]string
}
func (r *tunnelRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
line := name + " " + strings.Join(args, " ")
r.calls = append(r.calls, line)
return []byte(r.out[args[1]]), nil, nil
}
func (r *tunnelRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
return r.Run(ctx, name, args...)
}
// The probe uses EXACTLY the existing sudoers shape `pct exec <vmid> -- docker inspect -f <tmpl> cloudflared`, and
// with no customer guest it is unknown, never down.
func TestGuestTunnelProber(t *testing.T) {
r := &tunnelRunner{out: map[string]string{"9201": "running|0|healthy"}}
p := GuestTunnelProber{Runner: r, Guests: func(context.Context) ([]int, error) { return []int{9201}, nil }}
if st, d := p.Status(context.Background()); st != TunnelRunning || d != "connected" {
t.Fatalf("got %q %q", st, d)
}
want := "/usr/sbin/pct exec 9201 -- docker inspect -f " + tunnelInspect + " cloudflared"
if len(r.calls) != 1 || r.calls[0] != want {
t.Fatalf("command = %q, want %q", r.calls, want)
}
none := GuestTunnelProber{Runner: r, Guests: func(context.Context) ([]int, error) { return nil, nil }}
if st, _ := none.Status(context.Background()); st != TunnelUnknown {
t.Fatalf("no guest → %q, want unknown", st)
}
}
+10 -8
View File
@@ -280,7 +280,7 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
PBSSnapshots: c.collectPBSSnapshots(ctx),
AuditTail: []AuditEntry{},
Cloudflared: Cloudflared{Status: c.cloudflaredStatus(ctx)},
Cloudflared: c.cloudflared(ctx),
Capabilities: c.capabilities(ctx),
LeafFingerprint: c.leafFP,
Addresses: c.collectAddresses(),
@@ -555,16 +555,18 @@ func (c *Collector) collectPBSSnapshots(ctx context.Context) []PBSSnapshot {
return []PBSSnapshot{}
}
func (c *Collector) cloudflaredStatus(ctx context.Context) string {
func (c *Collector) cloudflared(ctx context.Context) Cloudflared {
if c.cf == nil {
return "unknown"
return Cloudflared{Status: TunnelUnknown, Detail: "no probe wired"}
}
st, err := c.cf.Status(ctx)
if err != nil || st == "" {
c.logger.Warn("hub: cloudflared probe failed", "err", err)
return "unknown"
st, d := c.cf.Status(ctx)
if st == "" {
st = TunnelUnknown
}
return st
if st == TunnelUnknown {
c.logger.Debug("hub: tunnel probe could not decide", "detail", d)
}
return Cloudflared{Status: st, Detail: d}
}
func percent(used, total int64) float64 {
+2 -2
View File
@@ -16,7 +16,7 @@ func (f fakeGuestNet) GuestNetStatus(context.Context) *GuestNetStatus { return f
func TestCollect_GuestNetOmittedWhenReporterNil(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("Collect: %v", err)
@@ -38,7 +38,7 @@ func TestCollect_GuestNetOmittedWhenReporterNil(t *testing.T) {
func TestCollect_GuestNetPopulatedWhenWired(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
c.SetGuestNetReporter(fakeGuestNet{st: &GuestNetStatus{
CheckedAt: "2026-07-21T10:00:00Z",
Guests: []GuestNetGuest{{
+4 -4
View File
@@ -12,7 +12,7 @@ func (f fakeMgmtPlane) MgmtPlaneStatus(context.Context) *MgmtPlaneStatus { retur
func TestCollect_MgmtPlaneOmittedWhenReporterNil(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("Collect: %v", err)
@@ -24,7 +24,7 @@ func TestCollect_MgmtPlaneOmittedWhenReporterNil(t *testing.T) {
func TestCollect_MgmtPlanePopulatedWhenWired(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger())
c.SetMgmtPlaneReporter(fakeMgmtPlane{st: &MgmtPlaneStatus{
PrivsepDirOK: true, SshdReachable: true, HealedRecently: true, PrivsepHealedAt: "2026-07-05T16:42:17Z",
}})
@@ -47,7 +47,7 @@ func (f fakeOOB) OOBStatus(context.Context) *OOBStatus { return f.st }
func TestCollect_OOBOmittedWhenNil(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.72.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, fakeObserver{}, nil, nil, nil, "h", "0.72.0", quietLogger())
r, _ := c.Collect(context.Background())
if r.OOB != nil {
t.Fatalf("no reporter → oob omitted, got %+v", r.OOB)
@@ -56,7 +56,7 @@ func TestCollect_OOBOmittedWhenNil(t *testing.T) {
func TestCollect_OOBPopulatedWhenWired(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.72.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, fakeObserver{}, nil, nil, nil, "h", "0.72.0", quietLogger())
c.SetOOBReporter(fakeOOB{st: &OOBStatus{FelhomSshdActive: true, FelhomSshdPort: 8822, Reachable: true}})
r, _ := c.Collect(context.Background())
if r.OOB == nil || r.OOB.FelhomSshdPort != 8822 || !r.OOB.Reachable {
+9 -9
View File
@@ -33,7 +33,7 @@ func TestCollect_StorageTargetsFromObserver(t *testing.T) {
obs := fakeObserver{targets: []StorageTarget{
{Name: "local-lvm", Type: StorageTypeLVMThin, State: StorageStateAttached, Reachable: true},
}}
c := NewCollector(px, fakeProber{status: "active"}, obs, nil, nil, nil, "h", "0.5.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, obs, nil, nil, nil, "h", "0.5.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("Collect: %v", err)
@@ -45,7 +45,7 @@ func TestCollect_StorageTargetsFromObserver(t *testing.T) {
func TestCollect_StorageObserverErrorDegradesToEmpty(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{err: errors.New("proxmox down")}, nil, nil, nil, "h", "0.5.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, fakeObserver{err: errors.New("proxmox down")}, nil, nil, nil, "h", "0.5.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("a storage observe error must not sink the heartbeat: %v", err)
@@ -64,7 +64,7 @@ func TestCollect_HostAndGuests(t *testing.T) {
},
cfg: map[int]proxmox.GuestConfig{100: {Cores: 2, Memory: 2048}},
}
c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "demo-host-01", "0.3.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, nil, nil, nil, nil, "demo-host-01", "0.3.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("Collect: %v", err)
@@ -88,7 +88,7 @@ func TestCollect_HostAndGuests(t *testing.T) {
if g.Spec.Cores != 2 || g.Spec.MemoryBytes != 2147483648 || g.Spec.DiskBytes != 21474836480 {
t.Errorf("spec = %+v", g.Spec)
}
if r.Cloudflared.Status != "active" {
if r.Cloudflared.Status != "running" || r.Cloudflared.Detail != "connected" {
t.Errorf("cloudflared = %q", r.Cloudflared.Status)
}
}
@@ -104,7 +104,7 @@ func TestCollect_GuestConfigFailureKeepsStatusOmitsSpec(t *testing.T) {
cfg: map[int]proxmox.GuestConfig{100: {Cores: 2}},
cfgErr: map[int]error{200: errors.New("config read failed")},
}
c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.3.1", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, nil, nil, nil, nil, "h", "0.3.1", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("a per-guest failure must NOT fail the whole report: %v", err)
@@ -125,7 +125,7 @@ func TestCollect_GuestConfigFailureKeepsStatusOmitsSpec(t *testing.T) {
func TestCollect_NodeStatusFailureIsHardError(t *testing.T) {
px := &fakePx{node: "n", nsErr: errors.New("proxmox down")}
c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.3.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, nil, nil, nil, nil, "h", "0.3.0", quietLogger())
if _, err := c.Collect(context.Background()); err == nil {
t.Fatal("NodeStatus failure must be a hard error (no useful report)")
}
@@ -133,7 +133,7 @@ func TestCollect_NodeStatusFailureIsHardError(t *testing.T) {
func TestCollect_CloudflaredProbeErrorIsUnknown(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{err: errors.New("no systemctl")}, nil, nil, nil, nil, "h", "0.3.0", quietLogger())
c := NewCollector(px, fakeProber{status: "", detail: "could not ask"}, nil, nil, nil, nil, "h", "0.3.0", quietLogger())
r, err := c.Collect(context.Background())
if err != nil {
t.Fatalf("cloudflared failure must not be fatal: %v", err)
@@ -153,7 +153,7 @@ func TestCollect_LeafFingerprint(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
const fp = "60b5974d586f5f3c8ec41eb998d0f07406178219c36bf6d3ff377570279d8245"
c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.48.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, nil, nil, nil, nil, "h", "0.48.0", quietLogger())
c.SetLeafFingerprint(fp)
r, err := c.Collect(context.Background())
if err != nil {
@@ -164,7 +164,7 @@ func TestCollect_LeafFingerprint(t *testing.T) {
}
// Companion: no SetLeafFingerprint (local API disabled) → empty, never a fabricated value.
c2 := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.48.0", quietLogger())
c2 := NewCollector(px, fakeProber{status: "running", detail: "connected"}, nil, nil, nil, nil, "h", "0.48.0", quietLogger())
r2, _ := c2.Collect(context.Background())
if r2.LeafFingerprint != "" {
t.Fatalf("unset leaf_fingerprint = %q, want empty", r2.LeafFingerprint)
+2 -2
View File
@@ -384,7 +384,7 @@ func TestCollectDRRecipe_ProductionPath(t *testing.T) {
obs := fakeObserver{targets: capturedDemoFelhomTargets()}
pbsRep := fakePBSReporter{snaps: capturedDemoFelhomSnapshots()}
c := NewCollector(px, fakeProber{status: "active"}, obs, nil, nil, pbsRep, "h", "0.118.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, obs, nil, nil, pbsRep, "h", "0.118.0", quietLogger())
c.SetBackupTargetResolver(func() ConfiguredBackupTarget {
return ConfiguredBackupTarget{StorageID: "felhom-backup", Known: true}
})
@@ -409,7 +409,7 @@ func TestCollectDRRecipe_ProductionPath(t *testing.T) {
// test that would have caught shipping the seam without wiring it.
func TestCollectDRRecipe_UnwiredSeamReportsUnknown(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{targets: capturedDemoFelhomTargets()},
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, fakeObserver{targets: capturedDemoFelhomTargets()},
nil, nil, nil, "h", "0.118.0", quietLogger())
r, err := c.Collect(context.Background())
+4 -4
View File
@@ -16,7 +16,7 @@ func intp(v int) *int { return &v }
// HostMetricsNow returns a fresh host block with cpu% from NodeStatus and the temp from the reader.
func TestHostMetricsNow_PopulatesTemp(t *testing.T) {
px := &fakePx{node: "demo-felhom", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()).
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()).
SetTempReader(fakeTemp{c: intp(46)})
h, err := c.HostMetricsNow(context.Background())
if err != nil {
@@ -36,7 +36,7 @@ func TestHostMetricsNow_PopulatesTemp(t *testing.T) {
// A missing temp sensor gracefully nulls cpu_temp_c without failing the host read.
func TestHostMetricsNow_GracefulNullTemp(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()).
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()).
SetTempReader(fakeTemp{c: nil})
h, err := c.HostMetricsNow(context.Background())
if err != nil {
@@ -50,7 +50,7 @@ func TestHostMetricsNow_GracefulNullTemp(t *testing.T) {
// A NodeStatus failure is a hard error (no useful host view).
func TestHostMetricsNow_NodeStatusErrorIsHard(t *testing.T) {
px := &fakePx{node: "n", nsErr: errors.New("proxmox down")}
c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger())
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger())
if _, err := c.HostMetricsNow(context.Background()); err == nil {
t.Fatal("NodeStatus failure must be a hard error")
}
@@ -59,7 +59,7 @@ func TestHostMetricsNow_NodeStatusErrorIsHard(t *testing.T) {
// Collect() (the hub report) also carries the temp now — the operator freebie.
func TestCollect_HostReportCarriesTemp(t *testing.T) {
px := &fakePx{node: "n", ns: newTestNodeStatus()}
c := NewCollector(px, fakeProber{status: "active"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()).
c := NewCollector(px, fakeProber{status: "running", detail: "connected"}, nil, nil, nil, nil, "h", "0.14.0", quietLogger()).
SetTempReader(fakeTemp{c: intp(51)})
r, err := c.Collect(context.Background())
if err != nil {
+2 -2
View File
@@ -56,7 +56,7 @@ func (f *fakePx) GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig
// fakeProber is a fake CloudflaredProber.
type fakeProber struct {
status string
err error
detail string
}
func (p fakeProber) Status(ctx context.Context) (string, error) { return p.status, p.err }
func (p fakeProber) Status(ctx context.Context) (string, string) { return p.status, p.detail }
+6 -2
View File
@@ -289,9 +289,10 @@ type GuestSpec struct {
DiskBytes int64 `json:"disk_bytes"`
}
// Cloudflared is the tunnel service health (read-only probe this slice).
// Cloudflared is the box's tunnel (R-841, agent v0.141.0): the cloudflared container in the customer guest.
type Cloudflared struct {
Status string `json:"status"` // active | inactive | failed | unknown
Status string `json:"status"` // running | not_running | unknown (TunnelRunning …)
Detail string `json:"detail,omitempty"` // why not_running / unknown, or "connected"
}
// The following element types are declared now so the empty collections above are
@@ -559,6 +560,9 @@ type WireOSUpdate struct {
Ring int `json:"ring"`
Enabled bool `json:"enabled"`
Release *WireOSRelease `json:"release,omitempty"`
// HostRelease is the newest approved HOST release (hub v0.131.0, `11` §8 step 3) — a separate set: a version
// approved for the guest is not approved for the host by that fact alone.
HostRelease *WireOSRelease `json:"host_release,omitempty"`
}
// WireOSRelease is an approved version set; Snapshot is the approval time (YYYYMMDDTHHMMSSZ) the wrapper uses
+164 -130
View File
@@ -1,14 +1,14 @@
// Package osupdate is the agent's OS-update leg for the customer GUEST (`11-os-updates.md` §8 step 2, agent v0.140.0).
// Package osupdate is the agent's OS-update leg (`11-os-updates.md` §8 steps 2–3): the customer GUEST's Debian fast
// lane (agent v0.140.0) and, after it in the same pass, the HOST's (agent v0.141.0).
//
// It runs right after the night's successful whole-guest backup, while the backup goroutine still holds the host-wide
// heavy-op gate (so it never overlaps a backup or a restore-test, `11` C10), at most once per night. All root work is
// the wrapper `felhom-os-apply` (configs/, its own tests); this package only builds plans, calls the wrapper through
// sudo, judges health and reports to the hub.
// sudo, judges health and reports to the hub — one report per layer.
//
// NO AUTOMATIC UNDO in this release (R-837, measured 2026-10-04): a customer guest cannot be snapshotted — PVE
// refuses any snapshot not named `vzdump` when the guest has host-path binds (mp8, mp9). A failed health check
// therefore stops, reports `health_failed` and the hub mails the operator; the whole-guest backup taken minutes
// earlier is the undo, by hand.
// NO AUTOMATIC UNDO (R-837 measured; `09` §3 decision 81): a failed health check stops, reports `health_failed` and the
// hub mails the operator; the whole-guest backup taken minutes earlier is the guest's undo, by hand; a host package is
// put back by hand from the previous release's snapshot (runbook). The host is NEVER rebooted by this package.
package osupdate
import (
@@ -33,6 +33,12 @@ const WrapperPath = "/usr/local/sbin/felhom-os-apply"
// DefaultPlanDir is where plans are written (the sudoers glob names it).
const DefaultPlanDir = "/var/lib/felhom-agent/os"
// Layers.
const (
LayerGuest = "guest"
LayerHost = "host"
)
// Package is one name=version with its origin.
type Package struct {
Name string `json:"name"`
@@ -40,7 +46,7 @@ type Package struct {
Origin string `json:"origin"`
}
// Pending is one update the guest's sources offer (origin as apt names it, possibly several).
// Pending is one update the sources offer (origin as apt names it, possibly several).
type Pending struct {
Name string `json:"name"`
From string `json:"from"`
@@ -54,17 +60,22 @@ type Container struct {
Health string `json:"health"` // healthy | unhealthy | starting | none
}
// Health is the guest's health snapshot (the wrapper's `health` object).
// Health is one health reading. Guest layer: DockerOK..Containers. Host layer: HostServices, GuestRunning and the
// guest's own reading in Guest.
type Health struct {
DockerOK bool `json:"docker_ok"`
NetworkOK bool `json:"network_ok"`
Controller string `json:"controller"`
Containers map[string]Container `json:"containers"`
DockerOK bool `json:"docker_ok"`
NetworkOK bool `json:"network_ok"`
Controller string `json:"controller"`
Containers map[string]Container `json:"containers"`
HostServices map[string]string `json:"host_services,omitempty"`
GuestRunning *bool `json:"guest_running,omitempty"`
Guest *Health `json:"guest,omitempty"`
}
// WrapperReport is the wrapper's OSAPPLY-REPORT object.
type WrapperReport struct {
Mode string `json:"mode"`
Layer string `json:"layer"`
Refused json.RawMessage `json:"refused"`
Failed json.RawMessage `json:"failed"`
Upgraded []Package `json:"upgraded"`
@@ -76,14 +87,16 @@ type WrapperReport struct {
HealthBefore *Health `json:"health_before"`
HealthAfter *Health `json:"health_after"`
Health *Health `json:"health"`
PassSeconds float64 `json:"pass_seconds"`
}
func (w WrapperReport) refused() bool { return len(w.Refused) > 0 && string(w.Refused) != "null" }
func (w WrapperReport) failed() bool { return len(w.Failed) > 0 && string(w.Failed) != "null" }
// Report is what the hub receives (hub osupdates.Report — field-exact).
// Report is what the hub receives per layer (hub osupdates.Report — field-exact).
type Report struct {
RunID string `json:"run_id"`
Layer string `json:"layer"`
Trigger string `json:"trigger"`
Mode string `json:"mode"`
Ring int `json:"ring"`
@@ -100,6 +113,7 @@ type Report struct {
DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"`
RebootNeeded bool `json:"reboot_needed,omitempty"`
Refused json.RawMessage `json:"refused,omitempty"`
PassSeconds float64 `json:"pass_seconds,omitempty"`
}
// Reporter posts a report to the hub (*hub.Client).
@@ -107,10 +121,12 @@ type Reporter interface {
PostOSReport(ctx context.Context, body []byte) error
}
// Leg runs one OS-update pass for the customer guest.
// Leg runs one OS-update pass for the customer guest and then the host.
type Leg struct {
Runner proxmox.Runner
Hub Reporter
Tunnel hub.CloudflaredProber // the host health rule needs the tunnel `running` (R-841)
Appliance bool // agent.json deployment_mode; the wrapper re-checks the ROOT-owned record (R12)
Logger *slog.Logger
PlanDir string
StatePath string // last night run (once per night)
@@ -122,7 +138,6 @@ type Leg struct {
mu sync.Mutex
block *hub.WireOSUpdate
have bool
}
// OnDesiredState stores the hub's os_update block (desired.RawConsumer — store only, never block).
@@ -132,7 +147,7 @@ func (l *Leg) OnDesiredState(_ context.Context, resp *hub.DesiredStateResponse)
}
l.mu.Lock()
defer l.mu.Unlock()
l.block, l.have = resp.DesiredState.OSUpdate, true
l.block = resp.DesiredState.OSUpdate
}
// Block returns the newest os_update block. No block (an older hub, or nothing fetched yet) = ring 1, ON, no
@@ -150,7 +165,7 @@ func (l *Leg) Block() hub.WireOSUpdate {
func (l *Leg) SetBlock(b *hub.WireOSUpdate) {
l.mu.Lock()
defer l.mu.Unlock()
l.block, l.have = b, true
l.block = b
}
func (l *Leg) now() time.Time {
@@ -191,18 +206,9 @@ func IsFast(origins []string) bool {
return true
}
func fastOrigin(origins []string) string {
for _, o := range origins {
if o == "Debian-Security" {
return o
}
}
return "Debian"
}
// HealthVerdict is THE health rule (`11` §5.4.1; pinned by TestHealthVerdict_*): after the run, docker answers, the
// guest's network resolves, the controller's own health check is `healthy`, and every container that was running
// before is running again — and healthy again if it was healthy before. "starting" is not yet healthy.
// HealthVerdict is THE guest health rule (`11` §8.1; pinned by TestHealthVerdict*): docker answers, the guest's
// network resolves, the controller's own health check is `healthy`, and every container that was running at the
// start of the pass runs again — and healthy again if it was. "starting" is not yet healthy.
func HealthVerdict(before, after *Health) (bool, string) {
if after == nil {
return false, "no health reading"
@@ -240,34 +246,44 @@ func HealthVerdict(before, after *Health) (bool, string) {
return true, ""
}
// MergeBaseline is the health baseline from two readings: a container counts as running (and healthy) if EITHER
// reading saw it so. nil-safe. Pinned by TestHealth_BaselineIsTheStartOfTheLeg.
func MergeBaseline(a, b *Health) *Health {
if a == nil {
return b
// HostHealthVerdict is THE host health rule (`11` §8.2; pinned by TestHostHealthVerdict): the Proxmox daemons and the
// agent are active, the customer guest still runs, the guest's own rule still passes against the start of the pass,
// and the tunnel is `running` (R-841).
func HostHealthVerdict(before, after *Health, tunnel string) (bool, string) {
if after == nil {
return false, "no health reading"
}
if b == nil {
return a
svcs := make([]string, 0, len(after.HostServices))
for s := range after.HostServices {
svcs = append(svcs, s)
}
out := &Health{DockerOK: a.DockerOK && b.DockerOK, NetworkOK: a.NetworkOK && b.NetworkOK, Controller: b.Controller,
Containers: map[string]Container{}}
for _, h := range []*Health{a, b} {
for n, c := range h.Containers {
cur, seen := out.Containers[n]
if !seen || (cur.State != "running" && c.State == "running") {
out.Containers[n] = c
continue
}
if c.State == "running" && c.Health == "healthy" {
out.Containers[n] = c
}
sort.Strings(svcs)
if len(svcs) == 0 {
return false, "no host service reading"
}
for _, s := range svcs {
if after.HostServices[s] != "active" {
return false, s + " is " + after.HostServices[s]
}
}
return out
if after.GuestRunning == nil || !*after.GuestRunning {
return false, "the customer guest is not running"
}
var gb *Health
if before != nil {
gb = before.Guest
}
if ok, why := HealthVerdict(gb, after.Guest); !ok {
return false, "guest: " + why
}
if tunnel != hub.TunnelRunning {
return false, "the tunnel is " + tunnel
}
return true, ""
}
// call writes the plan and runs the wrapper once.
func (l *Leg) call(ctx context.Context, runID, mode string, vmid int, rel hub.WireOSRelease, pkgs []Package) (WrapperReport, error) {
func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (WrapperReport, error) {
dir := l.PlanDir
if dir == "" {
dir = DefaultPlanDir
@@ -275,13 +291,8 @@ func (l *Leg) call(ctx context.Context, runID, mode string, vmid int, rel hub.Wi
if err := os.MkdirAll(dir, 0o700); err != nil {
return WrapperReport{}, fmt.Errorf("osupdate: plan dir: %w", err)
}
plan := map[string]any{"release_id": rel.ID, "layer": "guest", "lane": "fast", "vmid": vmid, "mode": mode,
"snapshot": rel.Snapshot, "packages": pkgs}
if pkgs == nil {
plan["packages"] = []Package{}
}
b, _ := json.Marshal(plan)
path := filepath.Join(dir, "plan-"+runID+"-"+mode+".json")
path := filepath.Join(dir, fmt.Sprintf("plan-%s-%s-%s.json", runID, plan["layer"], plan["mode"]))
if err := os.WriteFile(path, b, 0o600); err != nil {
return WrapperReport{}, fmt.Errorf("osupdate: write plan: %w", err)
}
@@ -307,20 +318,11 @@ func (l *Leg) call(ctx context.Context, runID, mode string, vmid int, rel hub.Wi
return rep, nil // a refusal / failure is IN the report (exit 2 / 3), not an error here
}
// Run is one pass: inventory → (switch, ring, plan) → apply → health → report. trigger is "night" or "debug".
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer. Returns both
// reports (host empty when skipped). trigger is "night" or "debug".
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report, host Report) {
runID := l.now().UTC().Format("20060102T150405Z")
blk := l.Block()
rel := hub.WireOSRelease{ID: "ring0-" + runID}
if blk.Ring == 1 {
rel = hub.WireOSRelease{}
if blk.Release != nil {
rel = *blk.Release
}
}
rep := Report{RunID: runID, Trigger: trigger, Ring: blk.Ring, VMID: vmid, ReleaseID: rel.ID, Mode: "inventory"}
lg := l.log().With("run", runID, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
lg := l.log().With("run", runID, "vmid", vmid, "trigger", trigger)
if trigger == "night" && l.StatePath != "" {
gap := l.MinGap
if gap == 0 {
@@ -329,68 +331,107 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
if b, err := os.ReadFile(l.StatePath); err == nil {
if last, perr := time.Parse(time.RFC3339, strings.TrimSpace(string(b))); perr == nil && l.now().Sub(last) < gap {
lg.Info("osupdate: skipped — already ran tonight", "last", last.UTC().Format(time.RFC3339))
rep.Outcome = "skipped"
return rep
return Report{RunID: runID, Layer: LayerGuest, Outcome: "skipped"}, Report{}
}
}
}
blk := l.Block()
guest = l.runLayer(ctx, runID, LayerGuest, vmid, trigger, blk)
if trigger == "night" && l.StatePath != "" {
_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
}
switch {
case !l.Appliance:
lg.Info("osupdate: host step skipped — not an appliance install (a BYO host belongs to its owner, `11` §1)")
case !(guest.Outcome == "applied" || guest.Outcome == "nothing" || guest.Outcome == "inventory") || !guest.Healthy:
lg.Warn("osupdate: host step skipped — the guest step did not end healthy", "guest_outcome", guest.Outcome, "reason", guest.HealthReason)
default:
host = l.runLayer(ctx, runID, LayerHost, vmid, trigger, blk)
}
return guest, host
}
func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
rel := hub.WireOSRelease{ID: "ring0-" + runID}
var wire *hub.WireOSRelease
if layer == LayerGuest {
wire = blk.Release
} else {
wire = blk.HostRelease
}
if blk.Ring == 1 {
rel = hub.WireOSRelease{}
if wire != nil {
rel = *wire
}
}
rep := Report{RunID: runID, Layer: layer, Trigger: trigger, Ring: blk.Ring, VMID: vmid, ReleaseID: rel.ID}
lg := l.log().With("run", runID, "layer", layer, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
lg.Info("osupdate: START", "enabled", blk.Enabled, "release", rel.ID)
inv, err := l.call(ctx, runID, "inventory", vmid, rel, nil)
if err != nil || inv.refused() || inv.failed() {
rep.Outcome, rep.Refused = "refused", inv.Refused
if err != nil {
rep.Outcome, rep.HealthReason = "failed", err.Error()
}
return l.finish(ctx, lg, rep)
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": "fast", "vmid": vmid, "snapshot": rel.Snapshot,
"packages": []Package{}, "mode": "apply", "select": "listed"}
if rel.ID == "" {
plan["release_id"] = "none"
}
var plan []Package
planned := map[string]bool{}
switch {
case !blk.Enabled:
rep.Outcome = "inventory"
plan["mode"] = "inventory"
lg.Info("osupdate: switched OFF for this box — reporting only")
case blk.Ring == 0:
for _, p := range inv.Pending {
if IsFast(p.Origin) {
plan = append(plan, Package{Name: p.Name, Version: p.To, Origin: fastOrigin(p.Origin)})
}
}
plan["select"] = "pending-fast" // the wrapper picks every pending Debian / Debian-Security upgrade
case len(rel.Packages) == 0:
plan["mode"] = "inventory" // ring 1 with no approved release for this layer: nothing to install
default:
var pk []Package
for _, p := range rel.Packages {
plan = append(plan, Package{Name: p.Name, Version: p.Version, Origin: p.Origin})
pk = append(pk, Package{Name: p.Name, Version: p.Version, Origin: p.Origin})
planned[p.Name] = true
}
plan["packages"] = pk
}
pkgNames := map[string]bool{}
for _, p := range plan {
pkgNames[p.Name] = true
rep.Mode = plan["mode"].(string)
wr, err := l.call(ctx, runID, plan)
switch {
case err != nil:
rep.Outcome, rep.HealthReason = "failed", err.Error()
return l.finish(ctx, lg, rep)
case wr.refused():
rep.Outcome, rep.Refused = "refused", wr.Refused
return l.finish(ctx, lg, rep)
case wr.failed():
rep.Outcome, rep.Refused = "failed", wr.Failed
}
if blk.Enabled && len(plan) == 0 {
rep.Outcome = "nothing"
}
final := inv
if blk.Enabled && len(plan) > 0 {
rep.Mode = "apply"
ap, err := l.call(ctx, runID, "apply", vmid, rel, plan)
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
if rep.Outcome == "" {
switch {
case err != nil:
rep.Outcome, rep.HealthReason = "failed", err.Error()
return l.finish(ctx, lg, rep)
case ap.refused():
rep.Outcome, rep.Refused = "refused", ap.Refused
return l.finish(ctx, lg, rep)
case ap.failed():
rep.Outcome, rep.Refused = "failed", ap.Failed
case rep.Mode == "inventory" && !blk.Enabled:
rep.Outcome = "inventory"
case len(wr.Upgraded) == 0:
rep.Outcome = "nothing"
default:
rep.Outcome = "applied"
}
final = ap
rep.Upgraded = ap.Upgraded
if rep.Outcome == "" {
if len(ap.Upgraded) == 0 {
rep.Outcome = "nothing"
} else {
rep.Outcome = "applied"
}
if blk.Ring == 0 {
for _, u := range wr.Upgraded {
planned[u.Name] = true
}
}
// Health: compare with the start of the pass; give restarted services time (only after an install).
cur := wr.HealthAfter
verdict := func(h *Health) (bool, string) {
if layer == LayerHost {
t := hub.TunnelUnknown
if l.Tunnel != nil {
t, _ = l.Tunnel.Status(ctx)
}
return HostHealthVerdict(wr.HealthBefore, h, t)
}
// Health: compare with what the guest looked like BEFORE the run; give restarted services time.
return HealthVerdict(wr.HealthBefore, h)
}
if len(wr.Upgraded) > 0 {
wait, poll := l.HealthWait, l.HealthPoll
if wait == 0 {
wait = 5 * time.Minute
@@ -399,19 +440,15 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
poll = 15 * time.Second
}
deadline := l.now().Add(wait)
cur := ap.HealthAfter
// The baseline is the guest as it was at the START of the leg (the inventory's reading) merged with the
// apply's own "before": an app that stops at any point during the run counts. Found live 2026-10-04: an app
// stopped between the inventory and the apply's own reading was taken as "stopped before" and ignored.
base := MergeBaseline(inv.HealthAfter, ap.HealthBefore)
for {
ok, why := HealthVerdict(base, cur)
ok, why := verdict(cur)
rep.Healthy, rep.HealthReason = ok, why
if ok || !l.now().Before(deadline) || ctx.Err() != nil {
break
}
l.sleep(ctx, poll)
hr, herr := l.call(ctx, runID, "health", vmid, rel, nil)
hp := map[string]any{"release_id": plan["release_id"], "layer": layer, "lane": "fast", "vmid": vmid, "mode": "health", "packages": []Package{}}
hr, herr := l.call(ctx, runID, hp)
if herr == nil && hr.Health != nil {
cur = hr.Health
}
@@ -420,20 +457,16 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
rep.Outcome = "health_failed"
}
} else {
ok, why := HealthVerdict(nil, inv.HealthAfter)
rep.Healthy, rep.HealthReason = ok, why
}
rep.Installed, rep.Pending = final.Installed, final.Pending
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = final.RestartNeeded, final.DockerRestartNeeded, final.RebootNeeded
rep.NotCovered = notCovered(final.Pending, blk.Ring, pkgNames)
if trigger == "night" && l.StatePath != "" {
_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
rep.Healthy, rep.HealthReason = verdict(cur)
}
rep.Installed, rep.Pending = wr.Installed, wr.Pending
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = wr.RestartNeeded, wr.DockerRestartNeeded, wr.RebootNeeded
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
return l.finish(ctx, lg, rep)
}
// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in
// ring 1 also every fast-lane update the release did not name.
// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in ring 1
// also every fast-lane update the release did not name.
func notCovered(pending []Pending, ring int, planned map[string]bool) []string {
var out []string
for _, p := range pending {
@@ -446,7 +479,8 @@ func notCovered(pending []Pending, ring int, planned map[string]bool) []string {
func (l *Leg) finish(ctx context.Context, lg *slog.Logger, rep Report) Report {
lg.Info("osupdate: DONE", "outcome", rep.Outcome, "healthy", rep.Healthy, "reason", rep.HealthReason,
"upgraded", len(rep.Upgraded), "pending", len(rep.Pending), "not_covered", len(rep.NotCovered), "restart_needed", len(rep.RestartNeeded))
"upgraded", len(rep.Upgraded), "pending", len(rep.Pending), "not_covered", len(rep.NotCovered),
"restart_needed", len(rep.RestartNeeded), "reboot_needed", rep.RebootNeeded, "wrapper_seconds", rep.PassSeconds)
if l.Hub != nil {
body, _ := json.Marshal(rep)
rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
+172 -98
View File
@@ -14,15 +14,27 @@ import (
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)
// fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per mode.
// fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per layer and mode.
type fakeWrapper struct {
t *testing.T
pending []Pending
applyRep WrapperReport
healthSeq []*Health // answers to successive "health" calls
applyRep map[string]WrapperReport // per layer
healthSeq map[string][]*Health // per layer: answers to successive "health" calls
plans []map[string]any
}
func yes() *bool { b := true; return &b }
func guestOK() *Health {
return &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "running", Health: "healthy"}}}
}
func hostOK() *Health {
return &Health{HostServices: map[string]string{"pveproxy": "active", "pvedaemon": "active", "pvestatd": "active",
"pve-cluster": "active", "felhom-agent": "active"}, GuestRunning: yes(), Guest: guestOK()}
}
func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
if name != WrapperPath || len(args) != 2 || args[0] != "--plan" {
f.t.Fatalf("unexpected command %s %v", name, args)
@@ -34,27 +46,30 @@ func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byt
var plan map[string]any
json.Unmarshal(b, &plan)
f.plans = append(f.plans, plan)
layer := plan["layer"].(string)
ok := guestOK()
if layer == LayerHost {
ok = hostOK()
}
var rep WrapperReport
healthy := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "running", Health: "healthy"}}}
switch plan["mode"] {
case "inventory":
rep = WrapperReport{Mode: "inventory", Pending: f.pending, HealthAfter: healthy,
rep = WrapperReport{Mode: "inventory", Pending: f.pending, HealthBefore: ok, HealthAfter: ok,
Installed: []Package{{Name: "libc6", Version: "u3", Origin: "Debian"}}}
case "apply":
rep = f.applyRep
rep = f.applyRep[layer]
rep.Mode = "apply"
if rep.HealthBefore == nil {
rep.HealthBefore = healthy
rep.HealthBefore = ok
}
if rep.HealthAfter == nil {
rep.HealthAfter = healthy
rep.HealthAfter = ok
}
case "health":
if len(f.healthSeq) > 0 {
rep.Health, f.healthSeq = f.healthSeq[0], f.healthSeq[1:]
if seq := f.healthSeq[layer]; len(seq) > 0 {
rep.Health, f.healthSeq[layer] = seq[0], seq[1:]
} else {
rep.Health = healthy
rep.Health = ok
}
}
out, _ := json.Marshal(rep)
@@ -74,11 +89,21 @@ func (h *fakeHub) PostOSReport(_ context.Context, body []byte) error {
return nil
}
type fakeTunnel struct{ st string }
func (t fakeTunnel) Status(context.Context) (string, string) { return t.st, "" }
func newLeg(t *testing.T, w *fakeWrapper, blk *hub.WireOSUpdate) (*Leg, *fakeHub) {
h := &fakeHub{}
now := time.Date(2026, 10, 4, 4, 0, 0, 0, time.UTC)
if w.applyRep == nil {
w.applyRep = map[string]WrapperReport{}
}
if w.healthSeq == nil {
w.healthSeq = map[string][]*Health{}
}
l := &Leg{Runner: w, Hub: h, PlanDir: t.TempDir(), StatePath: filepath.Join(t.TempDir(), "last"),
HealthWait: time.Minute, HealthPoll: 10 * time.Second,
HealthWait: time.Minute, HealthPoll: 10 * time.Second, Appliance: true, Tunnel: fakeTunnel{hub.TunnelRunning},
Now: func() time.Time { return now },
Sleep: func(_ context.Context, d time.Duration) { now = now.Add(d) }}
if blk != nil {
@@ -93,66 +118,73 @@ var pend = []Pending{
{Name: "docker-ce", From: "29.7", To: "29.8", Origin: []string{"Docker CE"}},
}
func modes(w *fakeWrapper) string {
func calls(w *fakeWrapper) string {
var m []string
for _, p := range w.plans {
m = append(m, p["mode"].(string))
m = append(m, p["layer"].(string)+":"+p["mode"].(string))
}
return strings.Join(m, ",")
}
// Ring 0 plans every pending FAST-LANE update (Docker excluded, `11` C3) and reports what it now runs.
func TestRing0_PlansTheFastLaneOnly(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6", Version: "u4"}, {Name: "openssl", Version: "u3"}}, Pending: pend[2:]}}
// Ring 0: ONE wrapper call per layer (R-845), select pending-fast (the wrapper picks every Debian / Debian-Security
// upgrade), from live sources; the guest step first, then the host step.
func TestRing0_OneCallPerLayer(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "u4"}, {Name: "openssl", Version: "u3"}}, Pending: pend[2:]},
LayerHost: {Upgraded: []Package{{Name: "openssl", Version: "u3"}}},
}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "applied" || !rep.Healthy {
t.Fatalf("rep = %+v", rep)
g, ho := l.Run(context.Background(), 9201, "night")
if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy {
t.Fatalf("guest %+v\nhost %+v", g, ho)
}
pk := w.plans[1]["packages"].([]any)
if len(pk) != 2 {
t.Fatalf("ring-0 plan = %v, want libc6 + openssl only", pk)
if calls(w) != "guest:apply,host:apply" {
t.Fatalf("calls = %s, want one apply per layer, guest first", calls(w))
}
if o := pk[1].(map[string]any)["origin"]; o != "Debian-Security" {
t.Fatalf("openssl origin = %v", o)
for _, p := range w.plans {
if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 {
t.Fatalf("ring-0 plan = %v", p)
}
}
if w.plans[1]["snapshot"] != "" {
t.Fatalf("ring 0 installs from live sources, snapshot = %v", w.plans[1]["snapshot"])
if len(g.NotCovered) != 1 || g.NotCovered[0] != "docker-ce" {
t.Fatalf("not covered = %v", g.NotCovered)
}
if len(rep.NotCovered) != 1 || rep.NotCovered[0] != "docker-ce" {
t.Fatalf("not covered = %v", rep.NotCovered)
}
if len(h.reports) != 1 || h.reports[0].Outcome != "applied" {
if len(h.reports) != 2 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost {
t.Fatalf("hub got %+v", h.reports)
}
}
// Ring 1 installs EXACTLY the approved release (its versions, its snapshot), nothing it computed itself.
func TestRing1_InstallsExactlyTheRelease(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6", Version: "u4-approved"}}, Pending: pend[1:]}}
rel := &hub.WireOSRelease{ID: "os-1", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "u4-approved", Origin: "Debian"}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: rel})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "applied" || rep.ReleaseID != "os-1" {
t.Fatalf("rep = %+v", rep)
// Ring 1 installs EXACTLY each layer's own approved release (a guest release is not a host release).
func TestRing1_EachLayerItsOwnRelease(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "g-u4"}}, Pending: pend[1:]},
LayerHost: {Upgraded: []Package{{Name: "openssl", Version: "h-u3"}}},
}}
gr := &hub.WireOSRelease{ID: "os-g", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "g-u4", Origin: "Debian"}}}
hr := &hub.WireOSRelease{ID: "os-h", Snapshot: "20261004T090000Z", Packages: []hub.WireOSPackage{{Name: "openssl", Version: "h-u3", Origin: "Debian-Security"}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: gr, HostRelease: hr})
g, ho := l.Run(context.Background(), 9201, "night")
if g.ReleaseID != "os-g" || ho.ReleaseID != "os-h" {
t.Fatalf("release ids %q %q", g.ReleaseID, ho.ReleaseID)
}
ap := w.plans[1]
pk := ap["packages"].([]any)
if len(pk) != 1 || pk[0].(map[string]any)["version"] != "u4-approved" || ap["snapshot"] != "20261004T080000Z" || ap["release_id"] != "os-1" {
t.Fatalf("ring-1 plan = %v", ap)
gp, hp := w.plans[0], w.plans[1]
if gp["snapshot"] != "20261004T080000Z" || gp["packages"].([]any)[0].(map[string]any)["version"] != "g-u4" {
t.Fatalf("guest plan %v", gp)
}
// openssl is pending and fast-lane but NOT in the release → not covered; docker-ce is never covered.
if strings.Join(rep.NotCovered, ",") != "openssl,docker-ce" {
t.Fatalf("not covered = %v", rep.NotCovered)
if hp["layer"] != LayerHost || hp["snapshot"] != "20261004T090000Z" || hp["packages"].([]any)[0].(map[string]any)["name"] != "openssl" {
t.Fatalf("host plan %v", hp)
}
if strings.Join(g.NotCovered, ",") != "openssl,docker-ce" {
t.Fatalf("guest not covered = %v", g.NotCovered)
}
}
func TestRing1_NoReleaseInstallsNothing(t *testing.T) {
func TestRing1_NoReleaseIsInventory(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "nothing" || modes(w) != "inventory" {
t.Fatalf("rep=%+v modes=%s", rep, modes(w))
g, ho := l.Run(context.Background(), 9201, "night")
if g.Outcome != "nothing" || ho.Outcome != "nothing" || calls(w) != "guest:inventory,host:inventory" {
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
}
}
@@ -160,31 +192,53 @@ func TestRing1_NoReleaseInstallsNothing(t *testing.T) {
func TestNoBlock_IsRing1Nothing(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend}
l, _ := newLeg(t, w, nil)
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "nothing" || rep.Ring != 1 || modes(w) != "inventory" {
t.Fatalf("rep=%+v modes=%s", rep, modes(w))
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "nothing" || g.Ring != 1 {
t.Fatalf("g=%+v calls=%s", g, calls(w))
}
}
// Switched OFF: the box reports but installs nothing (the brief, Part D 2).
// Switched OFF: the box reports but installs nothing, on both layers.
func TestSwitchOff_ReportsOnly(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: false})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "inventory" || modes(w) != "inventory" || len(h.reports) != 1 || len(rep.Pending) != 3 {
t.Fatalf("rep=%+v modes=%s", rep, modes(w))
g, ho := l.Run(context.Background(), 9201, "night")
if g.Outcome != "inventory" || ho.Outcome != "inventory" || calls(w) != "guest:inventory,host:inventory" || len(h.reports) != 2 {
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
}
}
// Unhealthy after the run, and still unhealthy at the end of the wait → health_failed (the hub mails the operator).
// Not an appliance (BYO, `11` §1): the host step never runs — no host plan at all.
func TestBYO_NoHostPlan(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Appliance = false
_, ho := l.Run(context.Background(), 9201, "night")
if ho.Outcome != "" || calls(w) != "guest:apply" || len(h.reports) != 1 {
t.Fatalf("a BYO box got a host step: host=%+v calls=%s", ho, calls(w))
}
}
// A failed guest step skips the host step that night.
func TestGuestFailure_SkipsTheHost(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
LayerGuest: {Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
g, ho := l.Run(context.Background(), 9201, "night")
if g.Outcome != "refused" || ho.Outcome != "" || calls(w) != "guest:apply" {
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
}
}
// Unhealthy after the run, and still unhealthy at the end of the wait → health_failed; the host step is skipped.
func TestHealth_FailsAfterTheWait(t *testing.T) {
bad := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "exited"}}}
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad},
healthSeq: []*Health{bad, bad, bad, bad, bad, bad, bad, bad}}
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}},
healthSeq: map[string][]*Health{LayerGuest: {bad, bad, bad, bad, bad, bad, bad, bad}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "health_failed" || rep.Healthy || !strings.Contains(rep.HealthReason, "app was running") {
t.Fatalf("rep = %+v", rep)
g, ho := l.Run(context.Background(), 9201, "night")
if g.Outcome != "health_failed" || g.Healthy || !strings.Contains(g.HealthReason, "app was running") || ho.Outcome != "" {
t.Fatalf("g=%+v h=%+v", g, ho)
}
if h.reports[0].Outcome != "health_failed" {
t.Fatal("the hub was not told")
@@ -194,11 +248,22 @@ func TestHealth_FailsAfterTheWait(t *testing.T) {
// A service that takes a moment to come back is not a failure: the poll sees it recover inside the wait.
func TestHealth_RecoversInsideTheWait(t *testing.T) {
starting := &Health{DockerOK: true, NetworkOK: true, Controller: "starting"}
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting},
healthSeq: []*Health{starting}}
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting}},
healthSeq: map[string][]*Health{LayerGuest: {starting}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "applied" || !rep.Healthy {
t.Fatalf("rep = %+v", rep)
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "applied" || !g.Healthy {
t.Fatalf("g = %+v", g)
}
}
// The host step judged unhealthy when the tunnel is down after it.
func TestHost_TunnelDownFailsTheHostStep(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerHost: {Upgraded: []Package{{Name: "openssl"}}}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Tunnel = fakeTunnel{hub.TunnelNotRunning}
_, ho := l.Run(context.Background(), 9201, "night")
if ho.Outcome != "health_failed" || !strings.Contains(ho.HealthReason, "tunnel") {
t.Fatalf("host = %+v", ho)
}
}
@@ -227,24 +292,49 @@ func TestHealthVerdict(t *testing.T) {
}
}
func TestOncePerNight(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Run(context.Background(), 9201, "night")
n := len(w.plans)
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "skipped" || len(w.plans) != n {
t.Fatalf("a second night run in the same night ran: %+v", rep)
// The host rule: every listed daemon active, the guest running and passing its own rule, the tunnel running.
// Red-proofs: drop any one check and its case fails.
func TestHostHealthVerdict(t *testing.T) {
no := false
svcDown := hostOK()
svcDown.HostServices["pveproxy"] = "failed"
guestDown := hostOK()
guestDown.GuestRunning = &no
guestApp := hostOK()
guestApp.Guest = &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"felhom-controller": {State: "running", Health: "healthy"}}}
cases := []struct {
name string
after *Health
tunnel string
want bool
why string
}{
{"all good", hostOK(), hub.TunnelRunning, true, ""},
{"a daemon down", svcDown, hub.TunnelRunning, false, "pveproxy"},
{"the guest stopped", guestDown, hub.TunnelRunning, false, "guest is not running"},
{"an app in the guest gone", guestApp, hub.TunnelRunning, false, "app was running"},
{"the tunnel down", hostOK(), hub.TunnelNotRunning, false, "tunnel"},
{"the tunnel unknown", hostOK(), hub.TunnelUnknown, false, "tunnel"},
{"no services read", &Health{GuestRunning: yes(), Guest: guestOK()}, hub.TunnelRunning, false, "no host service"},
}
if rep := l.Run(context.Background(), 9201, "debug"); rep.Outcome == "skipped" {
t.Fatal("the debug action must not be throttled")
for _, c := range cases {
got, why := HostHealthVerdict(hostOK(), c.after, c.tunnel)
if got != c.want || !strings.Contains(why, c.why) {
t.Errorf("%s: got %v (%s), want %v (…%s…)", c.name, got, why, c.want, c.why)
}
}
}
func TestRefusedIsReported(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "refused" || h.reports[0].Outcome != "refused" {
t.Fatalf("rep = %+v", rep)
func TestOncePerNight(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Run(context.Background(), 9201, "night")
n := len(w.plans)
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "skipped" || len(w.plans) != n {
t.Fatalf("a second night run in the same night ran: %+v", g)
}
if g, _ := l.Run(context.Background(), 9201, "debug"); g.Outcome == "skipped" {
t.Fatal("the debug action must not be throttled")
}
}
@@ -254,7 +344,7 @@ func TestWrapperSuite(t *testing.T) {
if err != nil {
t.Skip("python3 not available")
}
cmd := exec.Command(py, "../../configs/test_felhom_os_apply.py")
cmd := exec.Command(py, "-B", "../../configs/test_felhom_os_apply.py")
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("wrapper suite failed: %v\n%s", err, out)
@@ -263,19 +353,3 @@ func TestWrapperSuite(t *testing.T) {
t.Fatalf("wrapper suite did not report OK:\n%s", out)
}
}
// An app that stops BETWEEN the start of the leg and the apply's own "before" reading still fails the run.
// Measured live 2026-10-04 on demo-hp (privatebin stopped 1 s after the apply plan was written: the old rule
// passed). Red-proof: use ap.HealthBefore alone as the baseline and this fails.
func TestHealth_BaselineIsTheStartOfTheLeg(t *testing.T) {
stoppedEarly := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "exited"}}}
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}},
HealthBefore: stoppedEarly, HealthAfter: stoppedEarly},
healthSeq: []*Health{stoppedEarly, stoppedEarly, stoppedEarly, stoppedEarly, stoppedEarly, stoppedEarly, stoppedEarly}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "health_failed" || !strings.Contains(rep.HealthReason, "app was running") {
t.Fatalf("an app that stopped during the run passed: %+v", rep)
}
}