package osupdate import ( "context" "encoding/json" "io" "os" "os/exec" "path/filepath" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" ) // fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per layer and mode. type fakeWrapper struct { t *testing.T pending []Pending applyRep map[string]WrapperReport // per layer healthSeq map[string][]*Health // per layer: answers to successive "health" calls plans []map[string]any } func yes() *bool { b := true; return &b } func guestOK() *Health { return &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{ "felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "running", Health: "healthy"}}} } func hostOK() *Health { return &Health{HostServices: map[string]string{"pveproxy": "active", "pvedaemon": "active", "pvestatd": "active", "pve-cluster": "active", "felhom-agent": "active"}, GuestRunning: yes(), Guest: guestOK()} } func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) { if name != WrapperPath || len(args) != 2 || args[0] != "--plan" { f.t.Fatalf("unexpected command %s %v", name, args) } b, err := os.ReadFile(args[1]) if err != nil { f.t.Fatal(err) } var plan map[string]any json.Unmarshal(b, &plan) f.plans = append(f.plans, plan) layer := plan["layer"].(string) ok := guestOK() if layer == LayerHost { ok = hostOK() } var rep WrapperReport switch plan["mode"] { case "inventory": rep = WrapperReport{Mode: "inventory", Pending: f.pending, HealthBefore: ok, HealthAfter: ok, Installed: []Package{{Name: "libc6", Version: "u3", Origin: "Debian"}}} case "apply": rep = f.applyRep[layer] rep.Mode = "apply" if rep.HealthBefore == nil { rep.HealthBefore = ok } if rep.HealthAfter == nil { rep.HealthAfter = ok } case "health": if seq := f.healthSeq[layer]; len(seq) > 0 { rep.Health, f.healthSeq[layer] = seq[0], seq[1:] } else { rep.Health = ok } } out, _ := json.Marshal(rep) return []byte("OSAPPLY-REPORT " + string(out) + "\n"), []byte("os-apply: DONE rc=0\n"), nil } func (f *fakeWrapper) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) { return f.Run(ctx, name, args...) } type fakeHub struct{ reports []Report } func (h *fakeHub) PostOSReport(_ context.Context, body []byte) error { var r Report json.Unmarshal(body, &r) h.reports = append(h.reports, r) return nil } type fakeTunnel struct{ st string } func (t fakeTunnel) Status(context.Context) (string, string) { return t.st, "" } func newLeg(t *testing.T, w *fakeWrapper, blk *hub.WireOSUpdate) (*Leg, *fakeHub) { h := &fakeHub{} now := time.Date(2026, 10, 4, 4, 0, 0, 0, time.UTC) if w.applyRep == nil { w.applyRep = map[string]WrapperReport{} } if w.healthSeq == nil { w.healthSeq = map[string][]*Health{} } l := &Leg{Runner: w, Hub: h, PlanDir: t.TempDir(), StatePath: filepath.Join(t.TempDir(), "last"), HealthWait: time.Minute, HealthPoll: 10 * time.Second, Appliance: true, Tunnel: fakeTunnel{hub.TunnelRunning}, Now: func() time.Time { return now }, Sleep: func(_ context.Context, d time.Duration) { now = now.Add(d) }} if blk != nil { l.SetBlock(blk) } return l, h } var pend = []Pending{ {Name: "libc6", From: "u3", To: "u4", Origin: []string{"Debian"}}, {Name: "openssl", From: "u1", To: "u3", Origin: []string{"Debian-Security", "Debian"}}, {Name: "docker-ce", From: "29.7", To: "29.8", Origin: []string{"Docker CE"}}, } func calls(w *fakeWrapper) string { var m []string for _, p := range w.plans { m = append(m, p["layer"].(string)+":"+p["mode"].(string)) } return strings.Join(m, ",") } // Ring 0: ONE wrapper call per layer (R-845), select pending-fast (the wrapper picks every Debian / Debian-Security // upgrade), from live sources; the guest step first, then the host step. func TestRing0_OneCallPerLayer(t *testing.T) { w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{ LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "u4"}, {Name: "openssl", Version: "u3"}}, Pending: pend[2:]}, LayerHost: {Upgraded: []Package{{Name: "openssl", Version: "u3"}}}, }} l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) g, ho := run2(l, "night") if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy { t.Fatalf("guest %+v\nhost %+v", g, ho) } if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply" { t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore and the ring-0 docker step", calls(w)) } for _, p := range w.plans[:2] { if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 { t.Fatalf("ring-0 plan = %v", p) } } if len(g.NotCovered) != 1 || g.NotCovered[0] != "docker-ce" { t.Fatalf("not covered = %v", g.NotCovered) } if len(h.reports) != 3 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost || h.reports[2].Layer != LayerDocker { t.Fatalf("hub got %+v", h.reports) } } // Ring 1 installs EXACTLY each layer's own approved release (a guest release is not a host release). func TestRing1_EachLayerItsOwnRelease(t *testing.T) { w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{ LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "g-u4"}}, Pending: pend[1:]}, LayerHost: {Upgraded: []Package{{Name: "openssl", Version: "h-u3"}}}, }} gr := &hub.WireOSRelease{ID: "os-g", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "g-u4", Origin: "Debian"}}} hr := &hub.WireOSRelease{ID: "os-h", Snapshot: "20261004T090000Z", Packages: []hub.WireOSPackage{{Name: "openssl", Version: "h-u3", Origin: "Debian-Security"}}} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: gr, HostRelease: hr}) g, ho := run2(l, "night") if g.ReleaseID != "os-g" || ho.ReleaseID != "os-h" { t.Fatalf("release ids %q %q", g.ReleaseID, ho.ReleaseID) } gp, hp := w.plans[0], w.plans[1] if gp["snapshot"] != "20261004T080000Z" || gp["packages"].([]any)[0].(map[string]any)["version"] != "g-u4" { t.Fatalf("guest plan %v", gp) } if hp["layer"] != LayerHost || hp["snapshot"] != "20261004T090000Z" || hp["packages"].([]any)[0].(map[string]any)["name"] != "openssl" { t.Fatalf("host plan %v", hp) } if strings.Join(g.NotCovered, ",") != "openssl,docker-ce" { t.Fatalf("guest not covered = %v", g.NotCovered) } } func TestRing1_NoReleaseIsInventory(t *testing.T) { w := &fakeWrapper{t: t, pending: pend} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true}) g, ho := run2(l, "night") if g.Outcome != "nothing" || ho.Outcome != "nothing" || calls(w) != "guest:inventory,host:inventory" { t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w)) } } // No block from the hub (an older hub): ring 1, ON, no release → reports, installs nothing. func TestNoBlock_IsRing1Nothing(t *testing.T) { w := &fakeWrapper{t: t, pending: pend} l, _ := newLeg(t, w, nil) if g, _ := run2(l, "night"); g.Outcome != "nothing" || g.Ring != 1 { t.Fatalf("g=%+v calls=%s", g, calls(w)) } } // Switched OFF: the box reports but installs nothing, on both layers. func TestSwitchOff_ReportsOnly(t *testing.T) { w := &fakeWrapper{t: t, pending: pend} l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: false}) g, ho := run2(l, "night") if g.Outcome != "inventory" || ho.Outcome != "inventory" || calls(w) != "guest:inventory,host:inventory" || len(h.reports) != 2 { t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w)) } } // Not an appliance (BYO, `11` §1): the host step never runs — no host plan at all. func TestBYO_NoHostPlan(t *testing.T) { w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}} l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) l.Appliance = false _, ho := run2(l, "night") // the guest (and so its Docker engine) is ours on a BYO box too: only the HOST is the owner's if ho.Outcome != "" || calls(w) != "guest:apply,guest:live-restore-on,docker:apply" || len(h.reports) != 2 { t.Fatalf("a BYO box got a host step: host=%+v calls=%s", ho, calls(w)) } } // A failed guest step skips the host step that night. func TestGuestFailure_SkipsTheHost(t *testing.T) { w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{ LayerGuest: {Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) g, ho := run2(l, "night") if g.Outcome != "refused" || ho.Outcome != "" || calls(w) != "guest:apply" { t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w)) } } // Unhealthy after the run, and still unhealthy at the end of the wait → health_failed; the host step is skipped. func TestHealth_FailsAfterTheWait(t *testing.T) { bad := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{ "felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "exited"}}} w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}}, healthSeq: map[string][]*Health{LayerGuest: {bad, bad, bad, bad, bad, bad, bad, bad}}} l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) g, ho := run2(l, "night") if g.Outcome != "health_failed" || g.Healthy || !strings.Contains(g.HealthReason, "app was running") || ho.Outcome != "" { t.Fatalf("g=%+v h=%+v", g, ho) } if h.reports[0].Outcome != "health_failed" { t.Fatal("the hub was not told") } } // A service that takes a moment to come back is not a failure: the poll sees it recover inside the wait. func TestHealth_RecoversInsideTheWait(t *testing.T) { starting := &Health{DockerOK: true, NetworkOK: true, Controller: "starting"} w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting}}, healthSeq: map[string][]*Health{LayerGuest: {starting}}} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) if g, _ := run2(l, "night"); g.Outcome != "applied" || !g.Healthy { t.Fatalf("g = %+v", g) } } // The host step judged unhealthy when the tunnel is down after it. func TestHost_TunnelDownFailsTheHostStep(t *testing.T) { w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerHost: {Upgraded: []Package{{Name: "openssl"}}}}} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) l.Tunnel = fakeTunnel{hub.TunnelNotRunning} _, ho := run2(l, "night") if ho.Outcome != "health_failed" || !strings.Contains(ho.HealthReason, "tunnel") { t.Fatalf("host = %+v", ho) } } func TestHealthVerdict(t *testing.T) { ok := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"a": {State: "running", Health: "healthy"}}} cases := []struct { name string before *Health after *Health want bool }{ {"all good", ok, ok, true}, {"no reading", ok, nil, false}, {"docker down", ok, &Health{NetworkOK: true, Controller: "healthy"}, false}, {"no network", ok, &Health{DockerOK: true, Controller: "healthy"}, false}, {"controller starting", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "starting"}, false}, {"only the controller differs", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "unhealthy", Containers: map[string]Container{"a": {State: "running", Health: "healthy"}}}, false}, {"app gone", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{}}, false}, {"app unhealthy", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"a": {State: "running", Health: "unhealthy"}}}, false}, {"stopped before stays stopped", &Health{Containers: map[string]Container{"x": {State: "exited"}}}, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy"}, true}, } for _, c := range cases { if got, why := HealthVerdict(c.before, c.after); got != c.want { t.Errorf("%s: got %v (%s), want %v", c.name, got, why, c.want) } } } // The host rule: every listed daemon active, the guest running and passing its own rule, the tunnel running. // Red-proofs: drop any one check and its case fails. func TestHostHealthVerdict(t *testing.T) { no := false svcDown := hostOK() svcDown.HostServices["pveproxy"] = "failed" guestDown := hostOK() guestDown.GuestRunning = &no guestApp := hostOK() guestApp.Guest = &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"felhom-controller": {State: "running", Health: "healthy"}}} cases := []struct { name string after *Health tunnel string want bool why string }{ {"all good", hostOK(), hub.TunnelRunning, true, ""}, {"a daemon down", svcDown, hub.TunnelRunning, false, "pveproxy"}, {"the guest stopped", guestDown, hub.TunnelRunning, false, "guest is not running"}, {"an app in the guest gone", guestApp, hub.TunnelRunning, false, "app was running"}, {"the tunnel down", hostOK(), hub.TunnelNotRunning, false, "tunnel"}, {"the tunnel unknown", hostOK(), hub.TunnelUnknown, false, "tunnel"}, {"no services read", &Health{GuestRunning: yes(), Guest: guestOK()}, hub.TunnelRunning, false, "no host service"}, } for _, c := range cases { got, why := HostHealthVerdict(hostOK(), c.after, c.tunnel) if got != c.want || !strings.Contains(why, c.why) { t.Errorf("%s: got %v (%s), want %v (…%s…)", c.name, got, why, c.want, c.why) } } } func TestOncePerNight(t *testing.T) { w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) run2(l, "night") n := len(w.plans) if g, _ := run2(l, "night"); g.Outcome != "skipped" || len(w.plans) != n { t.Fatalf("a second night run in the same night ran: %+v", g) } if g, _ := run2(l, "debug"); g.Outcome == "skipped" { t.Fatal("the debug action must not be throttled") } } // The root wrapper's own suite (configs/test_felhom_os_apply.py) runs with `go test ./...` so CI covers it. func TestWrapperSuite(t *testing.T) { py, err := exec.LookPath("python3") if err != nil { t.Skip("python3 not available") } // the OS wrapper and (agent v0.142.0) the crash guard — both root programs in configs/ with their own suites for _, suite := range []string{"../../configs/test_felhom_os_apply.py", "../../configs/test_felhom_crash_guard.py"} { cmd := exec.Command(py, "-B", suite) out, err := cmd.CombinedOutput() if err != nil { t.Fatalf("%s failed: %v\n%s", suite, err, out) } if !strings.Contains(string(out), "OK") { t.Fatalf("%s did not report OK:\n%s", suite, out) } } } // The host's restart scan result reaches the hub with reboot_scanned, so the hub can tell "looked: not needed" (a // reboot cleared it) from "did not look". Red-proof: drop the RebootScanned copy in runLayer and this fails. func TestHostReport_CarriesRebootScanned(t *testing.T) { w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{ LayerGuest: {}, LayerHost: {RebootScanned: true, RebootNeeded: true, RestartNeeded: []string{"lxc-start"}}, }} l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) run2(l, "night") if len(h.reports) != 3 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned { t.Fatalf("hub got %+v", h.reports) } } // run2 is the guest + host reports of one pass (the tests written before the docker step). func run2(l *Leg, trigger string) (Report, Report) { p := l.Run(context.Background(), 9201, trigger) return p.Guest, p.Host } // ---- the Docker step (`11` §5.8, agent v0.142.0) ---- // Ring 1 never takes an engine step in the night leg — only inside a signed operator job. Red-proof: drop the // `blk.Ring != 0` case in Run and the ring-1 pass makes a docker call. func TestDocker_Ring1NightLegNeverSteps(t *testing.T) { w := &fakeWrapper{t: t} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true}) p := l.Run(context.Background(), 9201, "night") if p.Docker.Layer != "" || strings.Contains(calls(w), "docker") || strings.Contains(calls(w), "live-restore") { t.Fatalf("ring 1 took a docker step: %s", calls(w)) } } // An unhealthy earlier step skips the docker step. func TestDocker_SkippedAfterAnUnhealthyStep(t *testing.T) { w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}, healthSeq: map[string][]*Health{}} bad := guestOK() bad.Controller = "unhealthy" w.applyRep[LayerGuest] = WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad} w.healthSeq[LayerGuest] = []*Health{bad, bad, bad, bad, bad, bad, bad, bad} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) p := l.Run(context.Background(), 9201, "night") if p.Docker.Layer != "" || strings.Contains(calls(w), "docker") { t.Fatalf("docker step ran after an unhealthy guest step: %s", calls(w)) } } // The docker plan is the slow lane, pending-docker for ring 0; the report carries only the engine set. func TestDocker_Ring0PlanAndReport(t *testing.T) { w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: { Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}}, Installed: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker"}, {Name: "libc6", Version: "u4", Origin: "Debian"}}, DockerEngine: "29.8.2", Authority: "ring0"}}} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) p := l.Run(context.Background(), 9201, "night") dp := w.plans[len(w.plans)-1] if dp["layer"] != "docker" || dp["lane"] != "slow" || dp["select"] != "pending-docker" { t.Fatalf("docker plan = %v", dp) } d := p.Docker if d.Outcome != "applied" || !d.Healthy || d.DockerEngine != "29.8.2" || len(d.Installed) != 1 || d.Installed[0].Name != "docker-ce" { t.Fatalf("docker report = %+v", d) } } // THE docker health rule. Red-proof: drop the id comparison (or the engine check) in DockerHealthVerdict and a case fails. func TestDockerHealthVerdict(t *testing.T) { before := guestOK() before.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"}, "app": {State: "running", Health: "healthy", ID: "b"}} same := guestOK() same.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"}, "app": {State: "running", Health: "healthy", ID: "b"}} moved := guestOK() moved.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"}, "app": {State: "running", Health: "healthy", ID: "c"}} if ok, why := DockerHealthVerdict(before, same, "29.8.2", "29.8.2"); !ok { t.Fatalf("same ids, right engine: %s", why) } if ok, _ := DockerHealthVerdict(before, moved, "29.8.2", "29.8.2"); ok { t.Fatal("a changed container id passed — live-restore failed and the apps restarted") } if ok, _ := DockerHealthVerdict(before, same, "29.8.2", "29.7.2"); ok { t.Fatal("the engine did not move and the step passed") } if EngineOf("5:29.8.2-1~debian.13~trixie") != "29.8.2" { t.Fatalf("EngineOf = %q", EngineOf("5:29.8.2-1~debian.13~trixie")) } } // A changed id after the step → health_failed (the consequence, not only the verdict). func TestDocker_ChangedIDIsHealthFailed(t *testing.T) { before := guestOK() before.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"}} after := guestOK() after.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "z"}} w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: { Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1"}}, DockerEngine: "29.8.2", HealthBefore: before, HealthAfter: after}}, healthSeq: map[string][]*Health{}} w.healthSeq[LayerDocker] = []*Health{after, after, after, after, after, after, after, after} l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) p := l.Run(context.Background(), 9201, "night") if p.Docker.Outcome != "health_failed" || !strings.Contains(p.Docker.HealthReason, "id changed") { t.Fatalf("docker = %+v", p.Docker) } }