felhom-agent/internal/osupdate/leg_test.go

package osupdate

import (
	"context"
	"encoding/json"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)

// fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per layer and mode.
type fakeWrapper struct {
	t         *testing.T
	pending   []Pending
	applyRep  map[string]WrapperReport // per layer
	healthSeq map[string][]*Health     // per layer: answers to successive "health" calls
	plans     []map[string]any
}

func yes() *bool { b := true; return &b }

func guestOK() *Health {
	return &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
		"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "running", Health: "healthy"}}}
}

func hostOK() *Health {
	return &Health{HostServices: map[string]string{"pveproxy": "active", "pvedaemon": "active", "pvestatd": "active",
		"pve-cluster": "active", "felhom-agent": "active"}, GuestRunning: yes(), Guest: guestOK()}
}

func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
	if name != WrapperPath || len(args) != 2 || args[0] != "--plan" {
		f.t.Fatalf("unexpected command %s %v", name, args)
	}
	b, err := os.ReadFile(args[1])
	if err != nil {
		f.t.Fatal(err)
	}
	var plan map[string]any
	json.Unmarshal(b, &plan)
	f.plans = append(f.plans, plan)
	layer := plan["layer"].(string)
	ok := guestOK()
	if layer == LayerHost {
		ok = hostOK()
	}
	var rep WrapperReport
	switch plan["mode"] {
	case "inventory":
		rep = WrapperReport{Mode: "inventory", Pending: f.pending, HealthBefore: ok, HealthAfter: ok,
			Installed: []Package{{Name: "libc6", Version: "u3", Origin: "Debian"}}}
	case "apply":
		rep = f.applyRep[layer]
		rep.Mode = "apply"
		if rep.HealthBefore == nil {
			rep.HealthBefore = ok
		}
		if rep.HealthAfter == nil {
			rep.HealthAfter = ok
		}
	case "health":
		if seq := f.healthSeq[layer]; len(seq) > 0 {
			rep.Health, f.healthSeq[layer] = seq[0], seq[1:]
		} else {
			rep.Health = ok
		}
	}
	out, _ := json.Marshal(rep)
	return []byte("OSAPPLY-REPORT " + string(out) + "\n"), []byte("os-apply: DONE rc=0\n"), nil
}

func (f *fakeWrapper) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
	return f.Run(ctx, name, args...)
}

type fakeHub struct{ reports []Report }

func (h *fakeHub) PostOSReport(_ context.Context, body []byte) error {
	var r Report
	json.Unmarshal(body, &r)
	h.reports = append(h.reports, r)
	return nil
}

type fakeTunnel struct{ st string }

func (t fakeTunnel) Status(context.Context) (string, string) { return t.st, "" }

func newLeg(t *testing.T, w *fakeWrapper, blk *hub.WireOSUpdate) (*Leg, *fakeHub) {
	h := &fakeHub{}
	now := time.Date(2026, 10, 4, 4, 0, 0, 0, time.UTC)
	if w.applyRep == nil {
		w.applyRep = map[string]WrapperReport{}
	}
	if w.healthSeq == nil {
		w.healthSeq = map[string][]*Health{}
	}
	l := &Leg{Runner: w, Hub: h, PlanDir: t.TempDir(), StatePath: filepath.Join(t.TempDir(), "last"),
		HealthWait: time.Minute, HealthPoll: 10 * time.Second, Appliance: true, Tunnel: fakeTunnel{hub.TunnelRunning},
		Now:   func() time.Time { return now },
		Sleep: func(_ context.Context, d time.Duration) { now = now.Add(d) }}
	if blk != nil {
		l.SetBlock(blk)
	}
	return l, h
}

var pend = []Pending{
	{Name: "libc6", From: "u3", To: "u4", Origin: []string{"Debian"}},
	{Name: "openssl", From: "u1", To: "u3", Origin: []string{"Debian-Security", "Debian"}},
	{Name: "docker-ce", From: "29.7", To: "29.8", Origin: []string{"Docker CE"}},
}

func calls(w *fakeWrapper) string {
	var m []string
	for _, p := range w.plans {
		m = append(m, p["layer"].(string)+":"+p["mode"].(string))
	}
	return strings.Join(m, ",")
}

// Ring 0: ONE wrapper call per layer (R-845), select pending-fast (the wrapper picks every Debian / Debian-Security
// upgrade), from live sources; the guest step first, then the host step.
func TestRing0_OneCallPerLayer(t *testing.T) {
	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
		LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "u4"}, {Name: "openssl", Version: "u3"}}, Pending: pend[2:]},
		LayerHost:  {Upgraded: []Package{{Name: "openssl", Version: "u3"}}},
	}}
	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
	g, ho := l.Run(context.Background(), 9201, "night")
	if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy {
		t.Fatalf("guest %+v\nhost %+v", g, ho)
	}
	if calls(w) != "guest:apply,host:apply" {
		t.Fatalf("calls = %s, want one apply per layer, guest first", calls(w))
	}
	for _, p := range w.plans {
		if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 {
			t.Fatalf("ring-0 plan = %v", p)
		}
	}
	if len(g.NotCovered) != 1 || g.NotCovered[0] != "docker-ce" {
		t.Fatalf("not covered = %v", g.NotCovered)
	}
	if len(h.reports) != 2 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost {
		t.Fatalf("hub got %+v", h.reports)
	}
}

// Ring 1 installs EXACTLY each layer's own approved release (a guest release is not a host release).
func TestRing1_EachLayerItsOwnRelease(t *testing.T) {
	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
		LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "g-u4"}}, Pending: pend[1:]},
		LayerHost:  {Upgraded: []Package{{Name: "openssl", Version: "h-u3"}}},
	}}
	gr := &hub.WireOSRelease{ID: "os-g", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "g-u4", Origin: "Debian"}}}
	hr := &hub.WireOSRelease{ID: "os-h", Snapshot: "20261004T090000Z", Packages: []hub.WireOSPackage{{Name: "openssl", Version: "h-u3", Origin: "Debian-Security"}}}
	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: gr, HostRelease: hr})
	g, ho := l.Run(context.Background(), 9201, "night")
	if g.ReleaseID != "os-g" || ho.ReleaseID != "os-h" {
		t.Fatalf("release ids %q %q", g.ReleaseID, ho.ReleaseID)
	}
	gp, hp := w.plans[0], w.plans[1]
	if gp["snapshot"] != "20261004T080000Z" || gp["packages"].([]any)[0].(map[string]any)["version"] != "g-u4" {
		t.Fatalf("guest plan %v", gp)
	}
	if hp["layer"] != LayerHost || hp["snapshot"] != "20261004T090000Z" || hp["packages"].([]any)[0].(map[string]any)["name"] != "openssl" {
		t.Fatalf("host plan %v", hp)
	}
	if strings.Join(g.NotCovered, ",") != "openssl,docker-ce" {
		t.Fatalf("guest not covered = %v", g.NotCovered)
	}
}

func TestRing1_NoReleaseIsInventory(t *testing.T) {
	w := &fakeWrapper{t: t, pending: pend}
	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
	g, ho := l.Run(context.Background(), 9201, "night")
	if g.Outcome != "nothing" || ho.Outcome != "nothing" || calls(w) != "guest:inventory,host:inventory" {
		t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
	}
}

// No block from the hub (an older hub): ring 1, ON, no release → reports, installs nothing.
func TestNoBlock_IsRing1Nothing(t *testing.T) {
	w := &fakeWrapper{t: t, pending: pend}
	l, _ := newLeg(t, w, nil)
	if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "nothing" || g.Ring != 1 {
		t.Fatalf("g=%+v calls=%s", g, calls(w))
	}
}

// Switched OFF: the box reports but installs nothing, on both layers.
func TestSwitchOff_ReportsOnly(t *testing.T) {
	w := &fakeWrapper{t: t, pending: pend}
	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: false})
	g, ho := l.Run(context.Background(), 9201, "night")
	if g.Outcome != "inventory" || ho.Outcome != "inventory" || calls(w) != "guest:inventory,host:inventory" || len(h.reports) != 2 {
		t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
	}
}

// Not an appliance (BYO, `11` §1): the host step never runs — no host plan at all.
func TestBYO_NoHostPlan(t *testing.T) {
	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
	l.Appliance = false
	_, ho := l.Run(context.Background(), 9201, "night")
	if ho.Outcome != "" || calls(w) != "guest:apply" || len(h.reports) != 1 {
		t.Fatalf("a BYO box got a host step: host=%+v calls=%s", ho, calls(w))
	}
}

// A failed guest step skips the host step that night.
func TestGuestFailure_SkipsTheHost(t *testing.T) {
	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
		LayerGuest: {Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}}
	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
	g, ho := l.Run(context.Background(), 9201, "night")
	if g.Outcome != "refused" || ho.Outcome != "" || calls(w) != "guest:apply" {
		t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
	}
}

// Unhealthy after the run, and still unhealthy at the end of the wait → health_failed; the host step is skipped.
func TestHealth_FailsAfterTheWait(t *testing.T) {
	bad := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
		"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "exited"}}}
	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}},
		healthSeq: map[string][]*Health{LayerGuest: {bad, bad, bad, bad, bad, bad, bad, bad}}}
	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
	g, ho := l.Run(context.Background(), 9201, "night")
	if g.Outcome != "health_failed" || g.Healthy || !strings.Contains(g.HealthReason, "app was running") || ho.Outcome != "" {
		t.Fatalf("g=%+v h=%+v", g, ho)
	}
	if h.reports[0].Outcome != "health_failed" {
		t.Fatal("the hub was not told")
	}
}

// A service that takes a moment to come back is not a failure: the poll sees it recover inside the wait.
func TestHealth_RecoversInsideTheWait(t *testing.T) {
	starting := &Health{DockerOK: true, NetworkOK: true, Controller: "starting"}
	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting}},
		healthSeq: map[string][]*Health{LayerGuest: {starting}}}
	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
	if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "applied" || !g.Healthy {
		t.Fatalf("g = %+v", g)
	}
}

// The host step judged unhealthy when the tunnel is down after it.
func TestHost_TunnelDownFailsTheHostStep(t *testing.T) {
	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerHost: {Upgraded: []Package{{Name: "openssl"}}}}}
	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
	l.Tunnel = fakeTunnel{hub.TunnelNotRunning}
	_, ho := l.Run(context.Background(), 9201, "night")
	if ho.Outcome != "health_failed" || !strings.Contains(ho.HealthReason, "tunnel") {
		t.Fatalf("host = %+v", ho)
	}
}

func TestHealthVerdict(t *testing.T) {
	ok := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"a": {State: "running", Health: "healthy"}}}
	cases := []struct {
		name   string
		before *Health
		after  *Health
		want   bool
	}{
		{"all good", ok, ok, true},
		{"no reading", ok, nil, false},
		{"docker down", ok, &Health{NetworkOK: true, Controller: "healthy"}, false},
		{"no network", ok, &Health{DockerOK: true, Controller: "healthy"}, false},
		{"controller starting", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "starting"}, false},
		{"only the controller differs", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "unhealthy", Containers: map[string]Container{"a": {State: "running", Health: "healthy"}}}, false},
		{"app gone", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{}}, false},
		{"app unhealthy", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"a": {State: "running", Health: "unhealthy"}}}, false},
		{"stopped before stays stopped", &Health{Containers: map[string]Container{"x": {State: "exited"}}}, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy"}, true},
	}
	for _, c := range cases {
		if got, why := HealthVerdict(c.before, c.after); got != c.want {
			t.Errorf("%s: got %v (%s), want %v", c.name, got, why, c.want)
		}
	}
}

// The host rule: every listed daemon active, the guest running and passing its own rule, the tunnel running.
// Red-proofs: drop any one check and its case fails.
func TestHostHealthVerdict(t *testing.T) {
	no := false
	svcDown := hostOK()
	svcDown.HostServices["pveproxy"] = "failed"
	guestDown := hostOK()
	guestDown.GuestRunning = &no
	guestApp := hostOK()
	guestApp.Guest = &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"felhom-controller": {State: "running", Health: "healthy"}}}
	cases := []struct {
		name   string
		after  *Health
		tunnel string
		want   bool
		why    string
	}{
		{"all good", hostOK(), hub.TunnelRunning, true, ""},
		{"a daemon down", svcDown, hub.TunnelRunning, false, "pveproxy"},
		{"the guest stopped", guestDown, hub.TunnelRunning, false, "guest is not running"},
		{"an app in the guest gone", guestApp, hub.TunnelRunning, false, "app was running"},
		{"the tunnel down", hostOK(), hub.TunnelNotRunning, false, "tunnel"},
		{"the tunnel unknown", hostOK(), hub.TunnelUnknown, false, "tunnel"},
		{"no services read", &Health{GuestRunning: yes(), Guest: guestOK()}, hub.TunnelRunning, false, "no host service"},
	}
	for _, c := range cases {
		got, why := HostHealthVerdict(hostOK(), c.after, c.tunnel)
		if got != c.want || !strings.Contains(why, c.why) {
			t.Errorf("%s: got %v (%s), want %v (…%s…)", c.name, got, why, c.want, c.why)
		}
	}
}

func TestOncePerNight(t *testing.T) {
	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
	l.Run(context.Background(), 9201, "night")
	n := len(w.plans)
	if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "skipped" || len(w.plans) != n {
		t.Fatalf("a second night run in the same night ran: %+v", g)
	}
	if g, _ := l.Run(context.Background(), 9201, "debug"); g.Outcome == "skipped" {
		t.Fatal("the debug action must not be throttled")
	}
}

// The root wrapper's own suite (configs/test_felhom_os_apply.py) runs with `go test ./...` so CI covers it.
func TestWrapperSuite(t *testing.T) {
	py, err := exec.LookPath("python3")
	if err != nil {
		t.Skip("python3 not available")
	}
	cmd := exec.Command(py, "-B", "../../configs/test_felhom_os_apply.py")
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("wrapper suite failed: %v\n%s", err, out)
	}
	if !strings.Contains(string(out), "OK") {
		t.Fatalf("wrapper suite did not report OK:\n%s", out)
	}
}