OS updates host fast lane + true tunnel status + fast leg: wrapper host layer (R12 appliance proof from the root-owned install record, R14 kernel/boot/firmware refused), select pending-fast, one call per layer, host-side version checks, restart scan only after an install, reboot-needed for PID 1/lxc-start; the leg runs the host step after a healthy guest step; GuestTunnelProber reads the cloudflared container + its readiness check (R-841)
gates / gates (push) Successful in 19s
gates / gates (push) Successful in 19s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
+164
-130
@@ -1,14 +1,14 @@
|
||||
// Package osupdate is the agent's OS-update leg for the customer GUEST (`11-os-updates.md` §8 step 2, agent v0.140.0).
|
||||
// Package osupdate is the agent's OS-update leg (`11-os-updates.md` §8 steps 2–3): the customer GUEST's Debian fast
|
||||
// lane (agent v0.140.0) and, after it in the same pass, the HOST's (agent v0.141.0).
|
||||
//
|
||||
// It runs right after the night's successful whole-guest backup, while the backup goroutine still holds the host-wide
|
||||
// heavy-op gate (so it never overlaps a backup or a restore-test, `11` C10), at most once per night. All root work is
|
||||
// the wrapper `felhom-os-apply` (configs/, its own tests); this package only builds plans, calls the wrapper through
|
||||
// sudo, judges health and reports to the hub.
|
||||
// sudo, judges health and reports to the hub — one report per layer.
|
||||
//
|
||||
// NO AUTOMATIC UNDO in this release (R-837, measured 2026-10-04): a customer guest cannot be snapshotted — PVE
|
||||
// refuses any snapshot not named `vzdump` when the guest has host-path binds (mp8, mp9). A failed health check
|
||||
// therefore stops, reports `health_failed` and the hub mails the operator; the whole-guest backup taken minutes
|
||||
// earlier is the undo, by hand.
|
||||
// NO AUTOMATIC UNDO (R-837 measured; `09` §3 decision 81): a failed health check stops, reports `health_failed` and the
|
||||
// hub mails the operator; the whole-guest backup taken minutes earlier is the guest's undo, by hand; a host package is
|
||||
// put back by hand from the previous release's snapshot (runbook). The host is NEVER rebooted by this package.
|
||||
package osupdate
|
||||
|
||||
import (
|
||||
@@ -33,6 +33,12 @@ const WrapperPath = "/usr/local/sbin/felhom-os-apply"
|
||||
// DefaultPlanDir is where plans are written (the sudoers glob names it).
|
||||
const DefaultPlanDir = "/var/lib/felhom-agent/os"
|
||||
|
||||
// Layers.
|
||||
const (
|
||||
LayerGuest = "guest"
|
||||
LayerHost = "host"
|
||||
)
|
||||
|
||||
// Package is one name=version with its origin.
|
||||
type Package struct {
|
||||
Name string `json:"name"`
|
||||
@@ -40,7 +46,7 @@ type Package struct {
|
||||
Origin string `json:"origin"`
|
||||
}
|
||||
|
||||
// Pending is one update the guest's sources offer (origin as apt names it, possibly several).
|
||||
// Pending is one update the sources offer (origin as apt names it, possibly several).
|
||||
type Pending struct {
|
||||
Name string `json:"name"`
|
||||
From string `json:"from"`
|
||||
@@ -54,17 +60,22 @@ type Container struct {
|
||||
Health string `json:"health"` // healthy | unhealthy | starting | none
|
||||
}
|
||||
|
||||
// Health is the guest's health snapshot (the wrapper's `health` object).
|
||||
// Health is one health reading. Guest layer: DockerOK..Containers. Host layer: HostServices, GuestRunning and the
|
||||
// guest's own reading in Guest.
|
||||
type Health struct {
|
||||
DockerOK bool `json:"docker_ok"`
|
||||
NetworkOK bool `json:"network_ok"`
|
||||
Controller string `json:"controller"`
|
||||
Containers map[string]Container `json:"containers"`
|
||||
DockerOK bool `json:"docker_ok"`
|
||||
NetworkOK bool `json:"network_ok"`
|
||||
Controller string `json:"controller"`
|
||||
Containers map[string]Container `json:"containers"`
|
||||
HostServices map[string]string `json:"host_services,omitempty"`
|
||||
GuestRunning *bool `json:"guest_running,omitempty"`
|
||||
Guest *Health `json:"guest,omitempty"`
|
||||
}
|
||||
|
||||
// WrapperReport is the wrapper's OSAPPLY-REPORT object.
|
||||
type WrapperReport struct {
|
||||
Mode string `json:"mode"`
|
||||
Layer string `json:"layer"`
|
||||
Refused json.RawMessage `json:"refused"`
|
||||
Failed json.RawMessage `json:"failed"`
|
||||
Upgraded []Package `json:"upgraded"`
|
||||
@@ -76,14 +87,16 @@ type WrapperReport struct {
|
||||
HealthBefore *Health `json:"health_before"`
|
||||
HealthAfter *Health `json:"health_after"`
|
||||
Health *Health `json:"health"`
|
||||
PassSeconds float64 `json:"pass_seconds"`
|
||||
}
|
||||
|
||||
func (w WrapperReport) refused() bool { return len(w.Refused) > 0 && string(w.Refused) != "null" }
|
||||
func (w WrapperReport) failed() bool { return len(w.Failed) > 0 && string(w.Failed) != "null" }
|
||||
|
||||
// Report is what the hub receives (hub osupdates.Report — field-exact).
|
||||
// Report is what the hub receives per layer (hub osupdates.Report — field-exact).
|
||||
type Report struct {
|
||||
RunID string `json:"run_id"`
|
||||
Layer string `json:"layer"`
|
||||
Trigger string `json:"trigger"`
|
||||
Mode string `json:"mode"`
|
||||
Ring int `json:"ring"`
|
||||
@@ -100,6 +113,7 @@ type Report struct {
|
||||
DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"`
|
||||
RebootNeeded bool `json:"reboot_needed,omitempty"`
|
||||
Refused json.RawMessage `json:"refused,omitempty"`
|
||||
PassSeconds float64 `json:"pass_seconds,omitempty"`
|
||||
}
|
||||
|
||||
// Reporter posts a report to the hub (*hub.Client).
|
||||
@@ -107,10 +121,12 @@ type Reporter interface {
|
||||
PostOSReport(ctx context.Context, body []byte) error
|
||||
}
|
||||
|
||||
// Leg runs one OS-update pass for the customer guest.
|
||||
// Leg runs one OS-update pass for the customer guest and then the host.
|
||||
type Leg struct {
|
||||
Runner proxmox.Runner
|
||||
Hub Reporter
|
||||
Tunnel hub.CloudflaredProber // the host health rule needs the tunnel `running` (R-841)
|
||||
Appliance bool // agent.json deployment_mode; the wrapper re-checks the ROOT-owned record (R12)
|
||||
Logger *slog.Logger
|
||||
PlanDir string
|
||||
StatePath string // last night run (once per night)
|
||||
@@ -122,7 +138,6 @@ type Leg struct {
|
||||
|
||||
mu sync.Mutex
|
||||
block *hub.WireOSUpdate
|
||||
have bool
|
||||
}
|
||||
|
||||
// OnDesiredState stores the hub's os_update block (desired.RawConsumer — store only, never block).
|
||||
@@ -132,7 +147,7 @@ func (l *Leg) OnDesiredState(_ context.Context, resp *hub.DesiredStateResponse)
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.block, l.have = resp.DesiredState.OSUpdate, true
|
||||
l.block = resp.DesiredState.OSUpdate
|
||||
}
|
||||
|
||||
// Block returns the newest os_update block. No block (an older hub, or nothing fetched yet) = ring 1, ON, no
|
||||
@@ -150,7 +165,7 @@ func (l *Leg) Block() hub.WireOSUpdate {
|
||||
func (l *Leg) SetBlock(b *hub.WireOSUpdate) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.block, l.have = b, true
|
||||
l.block = b
|
||||
}
|
||||
|
||||
func (l *Leg) now() time.Time {
|
||||
@@ -191,18 +206,9 @@ func IsFast(origins []string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func fastOrigin(origins []string) string {
|
||||
for _, o := range origins {
|
||||
if o == "Debian-Security" {
|
||||
return o
|
||||
}
|
||||
}
|
||||
return "Debian"
|
||||
}
|
||||
|
||||
// HealthVerdict is THE health rule (`11` §5.4.1; pinned by TestHealthVerdict_*): after the run, docker answers, the
|
||||
// guest's network resolves, the controller's own health check is `healthy`, and every container that was running
|
||||
// before is running again — and healthy again if it was healthy before. "starting" is not yet healthy.
|
||||
// HealthVerdict is THE guest health rule (`11` §8.1; pinned by TestHealthVerdict*): docker answers, the guest's
|
||||
// network resolves, the controller's own health check is `healthy`, and every container that was running at the
|
||||
// start of the pass runs again — and healthy again if it was. "starting" is not yet healthy.
|
||||
func HealthVerdict(before, after *Health) (bool, string) {
|
||||
if after == nil {
|
||||
return false, "no health reading"
|
||||
@@ -240,34 +246,44 @@ func HealthVerdict(before, after *Health) (bool, string) {
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// MergeBaseline is the health baseline from two readings: a container counts as running (and healthy) if EITHER
|
||||
// reading saw it so. nil-safe. Pinned by TestHealth_BaselineIsTheStartOfTheLeg.
|
||||
func MergeBaseline(a, b *Health) *Health {
|
||||
if a == nil {
|
||||
return b
|
||||
// HostHealthVerdict is THE host health rule (`11` §8.2; pinned by TestHostHealthVerdict): the Proxmox daemons and the
|
||||
// agent are active, the customer guest still runs, the guest's own rule still passes against the start of the pass,
|
||||
// and the tunnel is `running` (R-841).
|
||||
func HostHealthVerdict(before, after *Health, tunnel string) (bool, string) {
|
||||
if after == nil {
|
||||
return false, "no health reading"
|
||||
}
|
||||
if b == nil {
|
||||
return a
|
||||
svcs := make([]string, 0, len(after.HostServices))
|
||||
for s := range after.HostServices {
|
||||
svcs = append(svcs, s)
|
||||
}
|
||||
out := &Health{DockerOK: a.DockerOK && b.DockerOK, NetworkOK: a.NetworkOK && b.NetworkOK, Controller: b.Controller,
|
||||
Containers: map[string]Container{}}
|
||||
for _, h := range []*Health{a, b} {
|
||||
for n, c := range h.Containers {
|
||||
cur, seen := out.Containers[n]
|
||||
if !seen || (cur.State != "running" && c.State == "running") {
|
||||
out.Containers[n] = c
|
||||
continue
|
||||
}
|
||||
if c.State == "running" && c.Health == "healthy" {
|
||||
out.Containers[n] = c
|
||||
}
|
||||
sort.Strings(svcs)
|
||||
if len(svcs) == 0 {
|
||||
return false, "no host service reading"
|
||||
}
|
||||
for _, s := range svcs {
|
||||
if after.HostServices[s] != "active" {
|
||||
return false, s + " is " + after.HostServices[s]
|
||||
}
|
||||
}
|
||||
return out
|
||||
if after.GuestRunning == nil || !*after.GuestRunning {
|
||||
return false, "the customer guest is not running"
|
||||
}
|
||||
var gb *Health
|
||||
if before != nil {
|
||||
gb = before.Guest
|
||||
}
|
||||
if ok, why := HealthVerdict(gb, after.Guest); !ok {
|
||||
return false, "guest: " + why
|
||||
}
|
||||
if tunnel != hub.TunnelRunning {
|
||||
return false, "the tunnel is " + tunnel
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// call writes the plan and runs the wrapper once.
|
||||
func (l *Leg) call(ctx context.Context, runID, mode string, vmid int, rel hub.WireOSRelease, pkgs []Package) (WrapperReport, error) {
|
||||
func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (WrapperReport, error) {
|
||||
dir := l.PlanDir
|
||||
if dir == "" {
|
||||
dir = DefaultPlanDir
|
||||
@@ -275,13 +291,8 @@ func (l *Leg) call(ctx context.Context, runID, mode string, vmid int, rel hub.Wi
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return WrapperReport{}, fmt.Errorf("osupdate: plan dir: %w", err)
|
||||
}
|
||||
plan := map[string]any{"release_id": rel.ID, "layer": "guest", "lane": "fast", "vmid": vmid, "mode": mode,
|
||||
"snapshot": rel.Snapshot, "packages": pkgs}
|
||||
if pkgs == nil {
|
||||
plan["packages"] = []Package{}
|
||||
}
|
||||
b, _ := json.Marshal(plan)
|
||||
path := filepath.Join(dir, "plan-"+runID+"-"+mode+".json")
|
||||
path := filepath.Join(dir, fmt.Sprintf("plan-%s-%s-%s.json", runID, plan["layer"], plan["mode"]))
|
||||
if err := os.WriteFile(path, b, 0o600); err != nil {
|
||||
return WrapperReport{}, fmt.Errorf("osupdate: write plan: %w", err)
|
||||
}
|
||||
@@ -307,20 +318,11 @@ func (l *Leg) call(ctx context.Context, runID, mode string, vmid int, rel hub.Wi
|
||||
return rep, nil // a refusal / failure is IN the report (exit 2 / 3), not an error here
|
||||
}
|
||||
|
||||
// Run is one pass: inventory → (switch, ring, plan) → apply → health → report. trigger is "night" or "debug".
|
||||
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
|
||||
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer. Returns both
|
||||
// reports (host empty when skipped). trigger is "night" or "debug".
|
||||
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report, host Report) {
|
||||
runID := l.now().UTC().Format("20060102T150405Z")
|
||||
blk := l.Block()
|
||||
rel := hub.WireOSRelease{ID: "ring0-" + runID}
|
||||
if blk.Ring == 1 {
|
||||
rel = hub.WireOSRelease{}
|
||||
if blk.Release != nil {
|
||||
rel = *blk.Release
|
||||
}
|
||||
}
|
||||
rep := Report{RunID: runID, Trigger: trigger, Ring: blk.Ring, VMID: vmid, ReleaseID: rel.ID, Mode: "inventory"}
|
||||
lg := l.log().With("run", runID, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
|
||||
|
||||
lg := l.log().With("run", runID, "vmid", vmid, "trigger", trigger)
|
||||
if trigger == "night" && l.StatePath != "" {
|
||||
gap := l.MinGap
|
||||
if gap == 0 {
|
||||
@@ -329,68 +331,107 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
|
||||
if b, err := os.ReadFile(l.StatePath); err == nil {
|
||||
if last, perr := time.Parse(time.RFC3339, strings.TrimSpace(string(b))); perr == nil && l.now().Sub(last) < gap {
|
||||
lg.Info("osupdate: skipped — already ran tonight", "last", last.UTC().Format(time.RFC3339))
|
||||
rep.Outcome = "skipped"
|
||||
return rep
|
||||
return Report{RunID: runID, Layer: LayerGuest, Outcome: "skipped"}, Report{}
|
||||
}
|
||||
}
|
||||
}
|
||||
blk := l.Block()
|
||||
guest = l.runLayer(ctx, runID, LayerGuest, vmid, trigger, blk)
|
||||
if trigger == "night" && l.StatePath != "" {
|
||||
_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
|
||||
}
|
||||
switch {
|
||||
case !l.Appliance:
|
||||
lg.Info("osupdate: host step skipped — not an appliance install (a BYO host belongs to its owner, `11` §1)")
|
||||
case !(guest.Outcome == "applied" || guest.Outcome == "nothing" || guest.Outcome == "inventory") || !guest.Healthy:
|
||||
lg.Warn("osupdate: host step skipped — the guest step did not end healthy", "guest_outcome", guest.Outcome, "reason", guest.HealthReason)
|
||||
default:
|
||||
host = l.runLayer(ctx, runID, LayerHost, vmid, trigger, blk)
|
||||
}
|
||||
return guest, host
|
||||
}
|
||||
|
||||
func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
|
||||
rel := hub.WireOSRelease{ID: "ring0-" + runID}
|
||||
var wire *hub.WireOSRelease
|
||||
if layer == LayerGuest {
|
||||
wire = blk.Release
|
||||
} else {
|
||||
wire = blk.HostRelease
|
||||
}
|
||||
if blk.Ring == 1 {
|
||||
rel = hub.WireOSRelease{}
|
||||
if wire != nil {
|
||||
rel = *wire
|
||||
}
|
||||
}
|
||||
rep := Report{RunID: runID, Layer: layer, Trigger: trigger, Ring: blk.Ring, VMID: vmid, ReleaseID: rel.ID}
|
||||
lg := l.log().With("run", runID, "layer", layer, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
|
||||
lg.Info("osupdate: START", "enabled", blk.Enabled, "release", rel.ID)
|
||||
|
||||
inv, err := l.call(ctx, runID, "inventory", vmid, rel, nil)
|
||||
if err != nil || inv.refused() || inv.failed() {
|
||||
rep.Outcome, rep.Refused = "refused", inv.Refused
|
||||
if err != nil {
|
||||
rep.Outcome, rep.HealthReason = "failed", err.Error()
|
||||
}
|
||||
return l.finish(ctx, lg, rep)
|
||||
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": "fast", "vmid": vmid, "snapshot": rel.Snapshot,
|
||||
"packages": []Package{}, "mode": "apply", "select": "listed"}
|
||||
if rel.ID == "" {
|
||||
plan["release_id"] = "none"
|
||||
}
|
||||
var plan []Package
|
||||
planned := map[string]bool{}
|
||||
switch {
|
||||
case !blk.Enabled:
|
||||
rep.Outcome = "inventory"
|
||||
plan["mode"] = "inventory"
|
||||
lg.Info("osupdate: switched OFF for this box — reporting only")
|
||||
case blk.Ring == 0:
|
||||
for _, p := range inv.Pending {
|
||||
if IsFast(p.Origin) {
|
||||
plan = append(plan, Package{Name: p.Name, Version: p.To, Origin: fastOrigin(p.Origin)})
|
||||
}
|
||||
}
|
||||
plan["select"] = "pending-fast" // the wrapper picks every pending Debian / Debian-Security upgrade
|
||||
case len(rel.Packages) == 0:
|
||||
plan["mode"] = "inventory" // ring 1 with no approved release for this layer: nothing to install
|
||||
default:
|
||||
var pk []Package
|
||||
for _, p := range rel.Packages {
|
||||
plan = append(plan, Package{Name: p.Name, Version: p.Version, Origin: p.Origin})
|
||||
pk = append(pk, Package{Name: p.Name, Version: p.Version, Origin: p.Origin})
|
||||
planned[p.Name] = true
|
||||
}
|
||||
plan["packages"] = pk
|
||||
}
|
||||
pkgNames := map[string]bool{}
|
||||
for _, p := range plan {
|
||||
pkgNames[p.Name] = true
|
||||
rep.Mode = plan["mode"].(string)
|
||||
wr, err := l.call(ctx, runID, plan)
|
||||
switch {
|
||||
case err != nil:
|
||||
rep.Outcome, rep.HealthReason = "failed", err.Error()
|
||||
return l.finish(ctx, lg, rep)
|
||||
case wr.refused():
|
||||
rep.Outcome, rep.Refused = "refused", wr.Refused
|
||||
return l.finish(ctx, lg, rep)
|
||||
case wr.failed():
|
||||
rep.Outcome, rep.Refused = "failed", wr.Failed
|
||||
}
|
||||
if blk.Enabled && len(plan) == 0 {
|
||||
rep.Outcome = "nothing"
|
||||
}
|
||||
final := inv
|
||||
if blk.Enabled && len(plan) > 0 {
|
||||
rep.Mode = "apply"
|
||||
ap, err := l.call(ctx, runID, "apply", vmid, rel, plan)
|
||||
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
|
||||
if rep.Outcome == "" {
|
||||
switch {
|
||||
case err != nil:
|
||||
rep.Outcome, rep.HealthReason = "failed", err.Error()
|
||||
return l.finish(ctx, lg, rep)
|
||||
case ap.refused():
|
||||
rep.Outcome, rep.Refused = "refused", ap.Refused
|
||||
return l.finish(ctx, lg, rep)
|
||||
case ap.failed():
|
||||
rep.Outcome, rep.Refused = "failed", ap.Failed
|
||||
case rep.Mode == "inventory" && !blk.Enabled:
|
||||
rep.Outcome = "inventory"
|
||||
case len(wr.Upgraded) == 0:
|
||||
rep.Outcome = "nothing"
|
||||
default:
|
||||
rep.Outcome = "applied"
|
||||
}
|
||||
final = ap
|
||||
rep.Upgraded = ap.Upgraded
|
||||
if rep.Outcome == "" {
|
||||
if len(ap.Upgraded) == 0 {
|
||||
rep.Outcome = "nothing"
|
||||
} else {
|
||||
rep.Outcome = "applied"
|
||||
}
|
||||
if blk.Ring == 0 {
|
||||
for _, u := range wr.Upgraded {
|
||||
planned[u.Name] = true
|
||||
}
|
||||
}
|
||||
// Health: compare with the start of the pass; give restarted services time (only after an install).
|
||||
cur := wr.HealthAfter
|
||||
verdict := func(h *Health) (bool, string) {
|
||||
if layer == LayerHost {
|
||||
t := hub.TunnelUnknown
|
||||
if l.Tunnel != nil {
|
||||
t, _ = l.Tunnel.Status(ctx)
|
||||
}
|
||||
return HostHealthVerdict(wr.HealthBefore, h, t)
|
||||
}
|
||||
// Health: compare with what the guest looked like BEFORE the run; give restarted services time.
|
||||
return HealthVerdict(wr.HealthBefore, h)
|
||||
}
|
||||
if len(wr.Upgraded) > 0 {
|
||||
wait, poll := l.HealthWait, l.HealthPoll
|
||||
if wait == 0 {
|
||||
wait = 5 * time.Minute
|
||||
@@ -399,19 +440,15 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
|
||||
poll = 15 * time.Second
|
||||
}
|
||||
deadline := l.now().Add(wait)
|
||||
cur := ap.HealthAfter
|
||||
// The baseline is the guest as it was at the START of the leg (the inventory's reading) merged with the
|
||||
// apply's own "before": an app that stops at any point during the run counts. Found live 2026-10-04: an app
|
||||
// stopped between the inventory and the apply's own reading was taken as "stopped before" and ignored.
|
||||
base := MergeBaseline(inv.HealthAfter, ap.HealthBefore)
|
||||
for {
|
||||
ok, why := HealthVerdict(base, cur)
|
||||
ok, why := verdict(cur)
|
||||
rep.Healthy, rep.HealthReason = ok, why
|
||||
if ok || !l.now().Before(deadline) || ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
l.sleep(ctx, poll)
|
||||
hr, herr := l.call(ctx, runID, "health", vmid, rel, nil)
|
||||
hp := map[string]any{"release_id": plan["release_id"], "layer": layer, "lane": "fast", "vmid": vmid, "mode": "health", "packages": []Package{}}
|
||||
hr, herr := l.call(ctx, runID, hp)
|
||||
if herr == nil && hr.Health != nil {
|
||||
cur = hr.Health
|
||||
}
|
||||
@@ -420,20 +457,16 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
|
||||
rep.Outcome = "health_failed"
|
||||
}
|
||||
} else {
|
||||
ok, why := HealthVerdict(nil, inv.HealthAfter)
|
||||
rep.Healthy, rep.HealthReason = ok, why
|
||||
}
|
||||
rep.Installed, rep.Pending = final.Installed, final.Pending
|
||||
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = final.RestartNeeded, final.DockerRestartNeeded, final.RebootNeeded
|
||||
rep.NotCovered = notCovered(final.Pending, blk.Ring, pkgNames)
|
||||
if trigger == "night" && l.StatePath != "" {
|
||||
_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
|
||||
rep.Healthy, rep.HealthReason = verdict(cur)
|
||||
}
|
||||
rep.Installed, rep.Pending = wr.Installed, wr.Pending
|
||||
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = wr.RestartNeeded, wr.DockerRestartNeeded, wr.RebootNeeded
|
||||
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
|
||||
return l.finish(ctx, lg, rep)
|
||||
}
|
||||
|
||||
// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in
|
||||
// ring 1 also every fast-lane update the release did not name.
|
||||
// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in ring 1
|
||||
// also every fast-lane update the release did not name.
|
||||
func notCovered(pending []Pending, ring int, planned map[string]bool) []string {
|
||||
var out []string
|
||||
for _, p := range pending {
|
||||
@@ -446,7 +479,8 @@ func notCovered(pending []Pending, ring int, planned map[string]bool) []string {
|
||||
|
||||
func (l *Leg) finish(ctx context.Context, lg *slog.Logger, rep Report) Report {
|
||||
lg.Info("osupdate: DONE", "outcome", rep.Outcome, "healthy", rep.Healthy, "reason", rep.HealthReason,
|
||||
"upgraded", len(rep.Upgraded), "pending", len(rep.Pending), "not_covered", len(rep.NotCovered), "restart_needed", len(rep.RestartNeeded))
|
||||
"upgraded", len(rep.Upgraded), "pending", len(rep.Pending), "not_covered", len(rep.NotCovered),
|
||||
"restart_needed", len(rep.RestartNeeded), "reboot_needed", rep.RebootNeeded, "wrapper_seconds", rep.PassSeconds)
|
||||
if l.Hub != nil {
|
||||
body, _ := json.Marshal(rep)
|
||||
rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||||
|
||||
+172
-98
@@ -14,15 +14,27 @@ import (
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per mode.
|
||||
// fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per layer and mode.
|
||||
type fakeWrapper struct {
|
||||
t *testing.T
|
||||
pending []Pending
|
||||
applyRep WrapperReport
|
||||
healthSeq []*Health // answers to successive "health" calls
|
||||
applyRep map[string]WrapperReport // per layer
|
||||
healthSeq map[string][]*Health // per layer: answers to successive "health" calls
|
||||
plans []map[string]any
|
||||
}
|
||||
|
||||
func yes() *bool { b := true; return &b }
|
||||
|
||||
func guestOK() *Health {
|
||||
return &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
|
||||
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "running", Health: "healthy"}}}
|
||||
}
|
||||
|
||||
func hostOK() *Health {
|
||||
return &Health{HostServices: map[string]string{"pveproxy": "active", "pvedaemon": "active", "pvestatd": "active",
|
||||
"pve-cluster": "active", "felhom-agent": "active"}, GuestRunning: yes(), Guest: guestOK()}
|
||||
}
|
||||
|
||||
func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
if name != WrapperPath || len(args) != 2 || args[0] != "--plan" {
|
||||
f.t.Fatalf("unexpected command %s %v", name, args)
|
||||
@@ -34,27 +46,30 @@ func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byt
|
||||
var plan map[string]any
|
||||
json.Unmarshal(b, &plan)
|
||||
f.plans = append(f.plans, plan)
|
||||
layer := plan["layer"].(string)
|
||||
ok := guestOK()
|
||||
if layer == LayerHost {
|
||||
ok = hostOK()
|
||||
}
|
||||
var rep WrapperReport
|
||||
healthy := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
|
||||
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "running", Health: "healthy"}}}
|
||||
switch plan["mode"] {
|
||||
case "inventory":
|
||||
rep = WrapperReport{Mode: "inventory", Pending: f.pending, HealthAfter: healthy,
|
||||
rep = WrapperReport{Mode: "inventory", Pending: f.pending, HealthBefore: ok, HealthAfter: ok,
|
||||
Installed: []Package{{Name: "libc6", Version: "u3", Origin: "Debian"}}}
|
||||
case "apply":
|
||||
rep = f.applyRep
|
||||
rep = f.applyRep[layer]
|
||||
rep.Mode = "apply"
|
||||
if rep.HealthBefore == nil {
|
||||
rep.HealthBefore = healthy
|
||||
rep.HealthBefore = ok
|
||||
}
|
||||
if rep.HealthAfter == nil {
|
||||
rep.HealthAfter = healthy
|
||||
rep.HealthAfter = ok
|
||||
}
|
||||
case "health":
|
||||
if len(f.healthSeq) > 0 {
|
||||
rep.Health, f.healthSeq = f.healthSeq[0], f.healthSeq[1:]
|
||||
if seq := f.healthSeq[layer]; len(seq) > 0 {
|
||||
rep.Health, f.healthSeq[layer] = seq[0], seq[1:]
|
||||
} else {
|
||||
rep.Health = healthy
|
||||
rep.Health = ok
|
||||
}
|
||||
}
|
||||
out, _ := json.Marshal(rep)
|
||||
@@ -74,11 +89,21 @@ func (h *fakeHub) PostOSReport(_ context.Context, body []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeTunnel struct{ st string }
|
||||
|
||||
func (t fakeTunnel) Status(context.Context) (string, string) { return t.st, "" }
|
||||
|
||||
func newLeg(t *testing.T, w *fakeWrapper, blk *hub.WireOSUpdate) (*Leg, *fakeHub) {
|
||||
h := &fakeHub{}
|
||||
now := time.Date(2026, 10, 4, 4, 0, 0, 0, time.UTC)
|
||||
if w.applyRep == nil {
|
||||
w.applyRep = map[string]WrapperReport{}
|
||||
}
|
||||
if w.healthSeq == nil {
|
||||
w.healthSeq = map[string][]*Health{}
|
||||
}
|
||||
l := &Leg{Runner: w, Hub: h, PlanDir: t.TempDir(), StatePath: filepath.Join(t.TempDir(), "last"),
|
||||
HealthWait: time.Minute, HealthPoll: 10 * time.Second,
|
||||
HealthWait: time.Minute, HealthPoll: 10 * time.Second, Appliance: true, Tunnel: fakeTunnel{hub.TunnelRunning},
|
||||
Now: func() time.Time { return now },
|
||||
Sleep: func(_ context.Context, d time.Duration) { now = now.Add(d) }}
|
||||
if blk != nil {
|
||||
@@ -93,66 +118,73 @@ var pend = []Pending{
|
||||
{Name: "docker-ce", From: "29.7", To: "29.8", Origin: []string{"Docker CE"}},
|
||||
}
|
||||
|
||||
func modes(w *fakeWrapper) string {
|
||||
func calls(w *fakeWrapper) string {
|
||||
var m []string
|
||||
for _, p := range w.plans {
|
||||
m = append(m, p["mode"].(string))
|
||||
m = append(m, p["layer"].(string)+":"+p["mode"].(string))
|
||||
}
|
||||
return strings.Join(m, ",")
|
||||
}
|
||||
|
||||
// Ring 0 plans every pending FAST-LANE update (Docker excluded, `11` C3) and reports what it now runs.
|
||||
func TestRing0_PlansTheFastLaneOnly(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6", Version: "u4"}, {Name: "openssl", Version: "u3"}}, Pending: pend[2:]}}
|
||||
// Ring 0: ONE wrapper call per layer (R-845), select pending-fast (the wrapper picks every Debian / Debian-Security
|
||||
// upgrade), from live sources; the guest step first, then the host step.
|
||||
func TestRing0_OneCallPerLayer(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
|
||||
LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "u4"}, {Name: "openssl", Version: "u3"}}, Pending: pend[2:]},
|
||||
LayerHost: {Upgraded: []Package{{Name: "openssl", Version: "u3"}}},
|
||||
}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
rep := l.Run(context.Background(), 9201, "night")
|
||||
if rep.Outcome != "applied" || !rep.Healthy {
|
||||
t.Fatalf("rep = %+v", rep)
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy {
|
||||
t.Fatalf("guest %+v\nhost %+v", g, ho)
|
||||
}
|
||||
pk := w.plans[1]["packages"].([]any)
|
||||
if len(pk) != 2 {
|
||||
t.Fatalf("ring-0 plan = %v, want libc6 + openssl only", pk)
|
||||
if calls(w) != "guest:apply,host:apply" {
|
||||
t.Fatalf("calls = %s, want one apply per layer, guest first", calls(w))
|
||||
}
|
||||
if o := pk[1].(map[string]any)["origin"]; o != "Debian-Security" {
|
||||
t.Fatalf("openssl origin = %v", o)
|
||||
for _, p := range w.plans {
|
||||
if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 {
|
||||
t.Fatalf("ring-0 plan = %v", p)
|
||||
}
|
||||
}
|
||||
if w.plans[1]["snapshot"] != "" {
|
||||
t.Fatalf("ring 0 installs from live sources, snapshot = %v", w.plans[1]["snapshot"])
|
||||
if len(g.NotCovered) != 1 || g.NotCovered[0] != "docker-ce" {
|
||||
t.Fatalf("not covered = %v", g.NotCovered)
|
||||
}
|
||||
if len(rep.NotCovered) != 1 || rep.NotCovered[0] != "docker-ce" {
|
||||
t.Fatalf("not covered = %v", rep.NotCovered)
|
||||
}
|
||||
if len(h.reports) != 1 || h.reports[0].Outcome != "applied" {
|
||||
if len(h.reports) != 2 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost {
|
||||
t.Fatalf("hub got %+v", h.reports)
|
||||
}
|
||||
}
|
||||
|
||||
// Ring 1 installs EXACTLY the approved release (its versions, its snapshot), nothing it computed itself.
|
||||
func TestRing1_InstallsExactlyTheRelease(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6", Version: "u4-approved"}}, Pending: pend[1:]}}
|
||||
rel := &hub.WireOSRelease{ID: "os-1", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "u4-approved", Origin: "Debian"}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: rel})
|
||||
rep := l.Run(context.Background(), 9201, "night")
|
||||
if rep.Outcome != "applied" || rep.ReleaseID != "os-1" {
|
||||
t.Fatalf("rep = %+v", rep)
|
||||
// Ring 1 installs EXACTLY each layer's own approved release (a guest release is not a host release).
|
||||
func TestRing1_EachLayerItsOwnRelease(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
|
||||
LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "g-u4"}}, Pending: pend[1:]},
|
||||
LayerHost: {Upgraded: []Package{{Name: "openssl", Version: "h-u3"}}},
|
||||
}}
|
||||
gr := &hub.WireOSRelease{ID: "os-g", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "g-u4", Origin: "Debian"}}}
|
||||
hr := &hub.WireOSRelease{ID: "os-h", Snapshot: "20261004T090000Z", Packages: []hub.WireOSPackage{{Name: "openssl", Version: "h-u3", Origin: "Debian-Security"}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: gr, HostRelease: hr})
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
if g.ReleaseID != "os-g" || ho.ReleaseID != "os-h" {
|
||||
t.Fatalf("release ids %q %q", g.ReleaseID, ho.ReleaseID)
|
||||
}
|
||||
ap := w.plans[1]
|
||||
pk := ap["packages"].([]any)
|
||||
if len(pk) != 1 || pk[0].(map[string]any)["version"] != "u4-approved" || ap["snapshot"] != "20261004T080000Z" || ap["release_id"] != "os-1" {
|
||||
t.Fatalf("ring-1 plan = %v", ap)
|
||||
gp, hp := w.plans[0], w.plans[1]
|
||||
if gp["snapshot"] != "20261004T080000Z" || gp["packages"].([]any)[0].(map[string]any)["version"] != "g-u4" {
|
||||
t.Fatalf("guest plan %v", gp)
|
||||
}
|
||||
// openssl is pending and fast-lane but NOT in the release → not covered; docker-ce is never covered.
|
||||
if strings.Join(rep.NotCovered, ",") != "openssl,docker-ce" {
|
||||
t.Fatalf("not covered = %v", rep.NotCovered)
|
||||
if hp["layer"] != LayerHost || hp["snapshot"] != "20261004T090000Z" || hp["packages"].([]any)[0].(map[string]any)["name"] != "openssl" {
|
||||
t.Fatalf("host plan %v", hp)
|
||||
}
|
||||
if strings.Join(g.NotCovered, ",") != "openssl,docker-ce" {
|
||||
t.Fatalf("guest not covered = %v", g.NotCovered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRing1_NoReleaseInstallsNothing(t *testing.T) {
|
||||
func TestRing1_NoReleaseIsInventory(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
||||
rep := l.Run(context.Background(), 9201, "night")
|
||||
if rep.Outcome != "nothing" || modes(w) != "inventory" {
|
||||
t.Fatalf("rep=%+v modes=%s", rep, modes(w))
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
if g.Outcome != "nothing" || ho.Outcome != "nothing" || calls(w) != "guest:inventory,host:inventory" {
|
||||
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,31 +192,53 @@ func TestRing1_NoReleaseInstallsNothing(t *testing.T) {
|
||||
func TestNoBlock_IsRing1Nothing(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend}
|
||||
l, _ := newLeg(t, w, nil)
|
||||
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "nothing" || rep.Ring != 1 || modes(w) != "inventory" {
|
||||
t.Fatalf("rep=%+v modes=%s", rep, modes(w))
|
||||
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "nothing" || g.Ring != 1 {
|
||||
t.Fatalf("g=%+v calls=%s", g, calls(w))
|
||||
}
|
||||
}
|
||||
|
||||
// Switched OFF: the box reports but installs nothing (the brief, Part D 2).
|
||||
// Switched OFF: the box reports but installs nothing, on both layers.
|
||||
func TestSwitchOff_ReportsOnly(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: false})
|
||||
rep := l.Run(context.Background(), 9201, "night")
|
||||
if rep.Outcome != "inventory" || modes(w) != "inventory" || len(h.reports) != 1 || len(rep.Pending) != 3 {
|
||||
t.Fatalf("rep=%+v modes=%s", rep, modes(w))
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
if g.Outcome != "inventory" || ho.Outcome != "inventory" || calls(w) != "guest:inventory,host:inventory" || len(h.reports) != 2 {
|
||||
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
|
||||
}
|
||||
}
|
||||
|
||||
// Unhealthy after the run, and still unhealthy at the end of the wait → health_failed (the hub mails the operator).
|
||||
// Not an appliance (BYO, `11` §1): the host step never runs — no host plan at all.
|
||||
func TestBYO_NoHostPlan(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l.Appliance = false
|
||||
_, ho := l.Run(context.Background(), 9201, "night")
|
||||
if ho.Outcome != "" || calls(w) != "guest:apply" || len(h.reports) != 1 {
|
||||
t.Fatalf("a BYO box got a host step: host=%+v calls=%s", ho, calls(w))
|
||||
}
|
||||
}
|
||||
|
||||
// A failed guest step skips the host step that night.
|
||||
func TestGuestFailure_SkipsTheHost(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
|
||||
LayerGuest: {Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
if g.Outcome != "refused" || ho.Outcome != "" || calls(w) != "guest:apply" {
|
||||
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
|
||||
}
|
||||
}
|
||||
|
||||
// Unhealthy after the run, and still unhealthy at the end of the wait → health_failed; the host step is skipped.
|
||||
func TestHealth_FailsAfterTheWait(t *testing.T) {
|
||||
bad := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
|
||||
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "exited"}}}
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad},
|
||||
healthSeq: []*Health{bad, bad, bad, bad, bad, bad, bad, bad}}
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}},
|
||||
healthSeq: map[string][]*Health{LayerGuest: {bad, bad, bad, bad, bad, bad, bad, bad}}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
rep := l.Run(context.Background(), 9201, "night")
|
||||
if rep.Outcome != "health_failed" || rep.Healthy || !strings.Contains(rep.HealthReason, "app was running") {
|
||||
t.Fatalf("rep = %+v", rep)
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
if g.Outcome != "health_failed" || g.Healthy || !strings.Contains(g.HealthReason, "app was running") || ho.Outcome != "" {
|
||||
t.Fatalf("g=%+v h=%+v", g, ho)
|
||||
}
|
||||
if h.reports[0].Outcome != "health_failed" {
|
||||
t.Fatal("the hub was not told")
|
||||
@@ -194,11 +248,22 @@ func TestHealth_FailsAfterTheWait(t *testing.T) {
|
||||
// A service that takes a moment to come back is not a failure: the poll sees it recover inside the wait.
|
||||
func TestHealth_RecoversInsideTheWait(t *testing.T) {
|
||||
starting := &Health{DockerOK: true, NetworkOK: true, Controller: "starting"}
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting},
|
||||
healthSeq: []*Health{starting}}
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting}},
|
||||
healthSeq: map[string][]*Health{LayerGuest: {starting}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "applied" || !rep.Healthy {
|
||||
t.Fatalf("rep = %+v", rep)
|
||||
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "applied" || !g.Healthy {
|
||||
t.Fatalf("g = %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// The host step judged unhealthy when the tunnel is down after it.
|
||||
func TestHost_TunnelDownFailsTheHostStep(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerHost: {Upgraded: []Package{{Name: "openssl"}}}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l.Tunnel = fakeTunnel{hub.TunnelNotRunning}
|
||||
_, ho := l.Run(context.Background(), 9201, "night")
|
||||
if ho.Outcome != "health_failed" || !strings.Contains(ho.HealthReason, "tunnel") {
|
||||
t.Fatalf("host = %+v", ho)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,24 +292,49 @@ func TestHealthVerdict(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOncePerNight(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l.Run(context.Background(), 9201, "night")
|
||||
n := len(w.plans)
|
||||
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "skipped" || len(w.plans) != n {
|
||||
t.Fatalf("a second night run in the same night ran: %+v", rep)
|
||||
// The host rule: every listed daemon active, the guest running and passing its own rule, the tunnel running.
|
||||
// Red-proofs: drop any one check and its case fails.
|
||||
func TestHostHealthVerdict(t *testing.T) {
|
||||
no := false
|
||||
svcDown := hostOK()
|
||||
svcDown.HostServices["pveproxy"] = "failed"
|
||||
guestDown := hostOK()
|
||||
guestDown.GuestRunning = &no
|
||||
guestApp := hostOK()
|
||||
guestApp.Guest = &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"felhom-controller": {State: "running", Health: "healthy"}}}
|
||||
cases := []struct {
|
||||
name string
|
||||
after *Health
|
||||
tunnel string
|
||||
want bool
|
||||
why string
|
||||
}{
|
||||
{"all good", hostOK(), hub.TunnelRunning, true, ""},
|
||||
{"a daemon down", svcDown, hub.TunnelRunning, false, "pveproxy"},
|
||||
{"the guest stopped", guestDown, hub.TunnelRunning, false, "guest is not running"},
|
||||
{"an app in the guest gone", guestApp, hub.TunnelRunning, false, "app was running"},
|
||||
{"the tunnel down", hostOK(), hub.TunnelNotRunning, false, "tunnel"},
|
||||
{"the tunnel unknown", hostOK(), hub.TunnelUnknown, false, "tunnel"},
|
||||
{"no services read", &Health{GuestRunning: yes(), Guest: guestOK()}, hub.TunnelRunning, false, "no host service"},
|
||||
}
|
||||
if rep := l.Run(context.Background(), 9201, "debug"); rep.Outcome == "skipped" {
|
||||
t.Fatal("the debug action must not be throttled")
|
||||
for _, c := range cases {
|
||||
got, why := HostHealthVerdict(hostOK(), c.after, c.tunnel)
|
||||
if got != c.want || !strings.Contains(why, c.why) {
|
||||
t.Errorf("%s: got %v (%s), want %v (…%s…)", c.name, got, why, c.want, c.why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefusedIsReported(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "refused" || h.reports[0].Outcome != "refused" {
|
||||
t.Fatalf("rep = %+v", rep)
|
||||
func TestOncePerNight(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l.Run(context.Background(), 9201, "night")
|
||||
n := len(w.plans)
|
||||
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "skipped" || len(w.plans) != n {
|
||||
t.Fatalf("a second night run in the same night ran: %+v", g)
|
||||
}
|
||||
if g, _ := l.Run(context.Background(), 9201, "debug"); g.Outcome == "skipped" {
|
||||
t.Fatal("the debug action must not be throttled")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,7 +344,7 @@ func TestWrapperSuite(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Skip("python3 not available")
|
||||
}
|
||||
cmd := exec.Command(py, "../../configs/test_felhom_os_apply.py")
|
||||
cmd := exec.Command(py, "-B", "../../configs/test_felhom_os_apply.py")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("wrapper suite failed: %v\n%s", err, out)
|
||||
@@ -263,19 +353,3 @@ func TestWrapperSuite(t *testing.T) {
|
||||
t.Fatalf("wrapper suite did not report OK:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// An app that stops BETWEEN the start of the leg and the apply's own "before" reading still fails the run.
|
||||
// Measured live 2026-10-04 on demo-hp (privatebin stopped 1 s after the apply plan was written: the old rule
|
||||
// passed). Red-proof: use ap.HealthBefore alone as the baseline and this fails.
|
||||
func TestHealth_BaselineIsTheStartOfTheLeg(t *testing.T) {
|
||||
stoppedEarly := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
|
||||
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "exited"}}}
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}},
|
||||
HealthBefore: stoppedEarly, HealthAfter: stoppedEarly},
|
||||
healthSeq: []*Health{stoppedEarly, stoppedEarly, stoppedEarly, stoppedEarly, stoppedEarly, stoppedEarly, stoppedEarly}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
rep := l.Run(context.Background(), 9201, "night")
|
||||
if rep.Outcome != "health_failed" || !strings.Contains(rep.HealthReason, "app was running") {
|
||||
t.Fatalf("an app that stopped during the run passed: %+v", rep)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user