OS updates, guest fast lane (11 §8 step 2): felhom-os-apply wrapper (R1-R13 refusals, repair first, snapshot.debian.org fallback), FELHOM_OSAPPLY sudoers, the OS leg after the primary backup, hub os_update block + os-report, --selftest=os-update
gates / gates (push) Successful in 18s

No automatic undo: a customer guest cannot be snapshotted (R-837, measured).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-04 10:44:29 +02:00
parent 596238cc2e
commit 23a8ef3de4
13 changed files with 1877 additions and 1 deletions
+3
View File
@@ -123,6 +123,9 @@ var manifest = []Capability{
{"dnsmasq-install", "dnsmasq package install", "/usr/bin/apt-get", []string{"install", "-y", "-q", "dnsmasq"}, false, ""},
{"dnsmasq-write", "dnsmasq drop-in write", "/usr/bin/install", []string{"-m", "0644", "/tmp/felhom-resolver-x.conf", "/etc/dnsmasq.d/felhom-x.conf"}, false, ""},
{"dnsmasq-enable", "dnsmasq enable", "/usr/bin/systemctl", []string{"enable", "--now", "dnsmasq"}, false, ""},
// ---- OS updates, guest fast lane (`11` §5.4.1; the wrapper holds every rule) ----
{"osapply-run", "OS update wrapper (guest fast lane)", "/usr/local/sbin/felhom-os-apply", []string{"--plan", "/var/lib/felhom-agent/os/plan-x.json"}, false, ""},
{"dnsmasq-reload", "dnsmasq reload", "/usr/bin/systemctl", []string{"reload", "dnsmasq"}, false, ""},
{"dnsmasq-restart", "dnsmasq restart (LAN-DNS self-heal)", "/usr/bin/systemctl", []string{"restart", "dnsmasq"}, false, ""},
{"dnsmasq-rm", "dnsmasq drop-in remove (decommission)", "/usr/bin/rm", []string{"-f", "/etc/dnsmasq.d/felhom-x.conf"}, false, ""},
+24
View File
@@ -425,3 +425,27 @@ func (c *Client) FetchRetainedIdentityEscrow(ctx context.Context) (*RetainedEscr
}
return &out, nil
}
// PostOSReport sends the OS-update leg's report after every run (hub v0.130.0): POST /api/v1/hosts/{id}/os-report.
// Per-host key, self-scoped on the hub. Errors are typed like RegisterWG's and never include the bearer.
func (c *Client) PostOSReport(ctx context.Context, body []byte) error {
if c.hostID == "" {
return fmt.Errorf("hub: PostOSReport requires a configured host_id")
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/api/v1/hosts/"+c.hostID+"/os-report", bytes.NewReader(body))
if err != nil {
return fmt.Errorf("hub: building os-report request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Content-Type", "application/json")
resp, err := c.hc.Do(req)
if err != nil {
return &TransportError{Err: err}
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
}
return nil
}
+30
View File
@@ -0,0 +1,30 @@
package hub
import (
"encoding/json"
"os"
"testing"
)
// The os_update block is a cross-repo contract: testdata/desired-state-osupdate.golden.json is byte-identical with
// felhom.eu/hub/internal/api/testdata (the hub's TestOSUpdate_DesiredBlockMatchesTheGolden proves the hub SERVES
// it). Here: the agent DECODES every field. A renamed json tag on either side fails one of the two tests.
func TestOSUpdateGolden_Decodes(t *testing.T) {
raw, err := os.ReadFile("testdata/desired-state-osupdate.golden.json")
if err != nil {
t.Fatal(err)
}
var resp DesiredStateResponse
if err := json.Unmarshal(raw, &resp); err != nil {
t.Fatal(err)
}
o := resp.DesiredState.OSUpdate
if o == nil || o.Ring != 1 || !o.Enabled || o.Release == nil {
t.Fatalf("os_update = %+v", o)
}
r := o.Release
if r.ID != "os-20261004-120000" || r.Snapshot != "20261004T120000Z" || len(r.Packages) != 2 ||
r.Packages[1].Name != "openssl" || r.Packages[1].Version != "3.5.7-1~deb13u3" || r.Packages[1].Origin != "Debian-Security" {
t.Fatalf("release = %+v", r)
}
}
+26
View File
@@ -548,6 +548,32 @@ type WireDesiredState struct {
RestoreDirective *WireRestoreDirective `json:"restore_directive,omitempty"` // slice 10D (forward-compat)
Wireguard *WireWireguard `json:"wireguard,omitempty"` // S3 (doc 06 §3.2; golden-pinned)
PBSDR *WirePBSDR `json:"pbs_dr,omitempty"` // PBS DR tier (slice 2 consumer)
OSUpdate *WireOSUpdate `json:"os_update,omitempty"` // OS updates, guest fast lane (agent v0.140.0)
}
// WireOSUpdate is the hub-OWNED OS-update block (hub v0.130.0, `11-os-updates.md` §5.3), merged into the served
// document at read time. Ring 0 installs every pending Debian / Debian-Security fix; ring 1 installs exactly the
// newest approved release. Absent (older hub) → the agent treats the box as ring 1, ON, no release: it reports
// and installs nothing. Golden: testdata/desired-state-osupdate.golden.json (byte-identical with the hub's).
type WireOSUpdate struct {
Ring int `json:"ring"`
Enabled bool `json:"enabled"`
Release *WireOSRelease `json:"release,omitempty"`
}
// WireOSRelease is an approved version set; Snapshot is the approval time (YYYYMMDDTHHMMSSZ) the wrapper uses
// for snapshot.debian.org when Debian has already replaced a version (decision 79).
type WireOSRelease struct {
ID string `json:"id"`
Snapshot string `json:"snapshot"`
Packages []WireOSPackage `json:"packages"`
}
// WireOSPackage is one approved name=version and its origin ("Debian" | "Debian-Security").
type WireOSPackage struct {
Name string `json:"name"`
Version string `json:"version"`
Origin string `json:"origin"`
}
// WirePBSDR is the hub's PBS-DR-tier descriptor (PBS DR slice 1, hub/internal/web/pbsdr.go
@@ -0,0 +1,17 @@
{
"generation": 1,
"desired_state": {
"os_update": {
"ring": 1,
"enabled": true,
"release": {
"id": "os-20261004-120000",
"snapshot": "20261004T120000Z",
"packages": [
{"name": "libc6", "version": "2.41-12+deb13u4", "origin": "Debian"},
{"name": "openssl", "version": "3.5.7-1~deb13u3", "origin": "Debian-Security"}
]
}
}
}
}
+63
View File
@@ -0,0 +1,63 @@
package localapi
import (
"context"
"net/http"
"sync"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
)
// The OS leg (agent v0.140.0) runs after a SUCCESSFUL primary backup, and only then; and it runs BEFORE the
// host-wide heavy-op gate is released, so a restore-test cannot start in the middle of it (`11` C10).
// Red-proof: drop the `b.Success &&` guard and the failed-backup sub-case fails; move the call after release()
// and the gate sub-case fails.
func TestAfterPrimaryBackup(t *testing.T) {
run := func(t *testing.T, failErr string) (calls []int, gateHeld bool) {
gate := &backup.InFlight{}
b := &fakeBackups{failErr: failErr}
srv := newTestServerS(t, &fakeGuests{}, b, &fakeStore{}, nil)
srv.inFlight = gate
var mu sync.Mutex
done := make(chan struct{}, 1)
srv.SetAfterPrimaryBackup(func(_ context.Context, vmid int) {
rel, _, ok := gate.TryAcquire("probe")
mu.Lock()
calls = append(calls, vmid)
gateHeld = !ok
mu.Unlock()
if ok {
rel()
}
done <- struct{}{}
})
h := srv.Handler()
if do(t, h, "POST", "/backup", "A", "").Code != http.StatusAccepted {
t.Fatal("POST /backup not accepted")
}
select {
case <-done:
case <-time.After(500 * time.Millisecond):
}
time.Sleep(20 * time.Millisecond)
mu.Lock()
defer mu.Unlock()
return calls, gateHeld
}
t.Run("success runs the leg under the gate", func(t *testing.T) {
calls, held := run(t, "")
if len(calls) != 1 {
t.Fatalf("the leg ran %d time(s), want 1", len(calls))
}
if !held {
t.Fatal("the heavy-op gate was free while the leg ran — a restore-test could overlap it")
}
})
t.Run("a failed backup runs nothing", func(t *testing.T) {
if calls, _ := run(t, "vzdump exploded"); len(calls) != 0 {
t.Fatalf("the leg ran after a FAILED backup: %v", calls)
}
})
}
+13
View File
@@ -160,6 +160,10 @@ type Options struct {
// NetStorage is the privileged network-mount (NAS) surface (Part A1). OPTIONAL — when nil, the
// /netstorage endpoints report "not configured". Satisfied by *storage.SudoHostOps.
NetStorage NetworkStorageOps
// AfterPrimaryBackup (agent v0.140.0, `11-os-updates.md` §8 step 2) runs right after a SUCCESSFUL backup on the
// PRIMARY tier, inside the backup goroutine and BEFORE the host-wide heavy-op gate is released — so the OS leg
// that it starts can never overlap another backup or a restore-test (`11` C10). OPTIONAL — nil → nothing runs.
AfterPrimaryBackup func(ctx context.Context, vmid int)
// Privileged runs the fenced root wrappers (E-2a: felhom-backup-target-apply). OPTIONAL — when
// nil, POST /backup/target reports "not configured". Satisfied by *proxmox.ExecRunner.
Privileged PrivilegedRunner
@@ -276,6 +280,7 @@ type Server struct {
tiers []BackupTier
// inFlight (R-85) is shared with the restore-test scheduler so the two never run together.
inFlight *backup.InFlight
afterPrimaryBackup func(ctx context.Context, vmid int) // the OS leg (agent v0.140.0); nil = none
logger *slog.Logger
now func() time.Time
@@ -469,6 +474,7 @@ func NewServer(o Options) (*Server, error) {
// the primary is always first, because that is what the untargeted endpoints act on.
s.tiers = normalizeBackupTiers(o.BackupTiers, o.Backups, cadence)
s.inFlight = o.InFlight
s.afterPrimaryBackup = o.AfterPrimaryBackup
if s.backups == nil && len(s.tiers) > 0 {
s.backups = s.tiers[0].Service
}
@@ -894,6 +900,10 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int)
}
s.store.RecordBackup(b)
s.finishJob(key, jobID, b)
// OS leg (agent v0.140.0): after the night's whole-guest copy exists, still holding the heavy-op gate.
if b.Success && tier.Primary && s.afterPrimaryBackup != nil {
s.afterPrimaryBackup(base, vmid)
}
}()
writeStatus(w, http.StatusAccepted, true, BackupResponse{VMID: vmid, JobID: jobID, Phase: PhaseRunning}, "")
}
@@ -1465,3 +1475,6 @@ func writeStatus(w http.ResponseWriter, code int, ok bool, data any, errMsg stri
w.WriteHeader(code)
_ = json.NewEncoder(w).Encode(apiResponse{OK: ok, Data: data, Error: errMsg})
}
// SetAfterPrimaryBackup wires the hook that runs after a successful primary-tier backup (the OS leg, agent v0.140.0).
func (s *Server) SetAfterPrimaryBackup(fn func(ctx context.Context, vmid int)) { s.afterPrimaryBackup = fn }
+429
View File
@@ -0,0 +1,429 @@
// Package osupdate is the agent's OS-update leg for the customer GUEST (`11-os-updates.md` §8 step 2, agent v0.140.0).
//
// It runs right after the night's successful whole-guest backup, while the backup goroutine still holds the host-wide
// heavy-op gate (so it never overlaps a backup or a restore-test, `11` C10), at most once per night. All root work is
// the wrapper `felhom-os-apply` (configs/, its own tests); this package only builds plans, calls the wrapper through
// sudo, judges health and reports to the hub.
//
// NO AUTOMATIC UNDO in this release (R-837, measured 2026-10-04): a customer guest cannot be snapshotted — PVE
// refuses any snapshot not named `vzdump` when the guest has host-path binds (mp8, mp9). A failed health check
// therefore stops, reports `health_failed` and the hub mails the operator; the whole-guest backup taken minutes
// earlier is the undo, by hand.
package osupdate
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)
// WrapperPath is the pinned sudoers vector (configs/felhom-agent.sudoers FELHOM_OSAPPLY).
const WrapperPath = "/usr/local/sbin/felhom-os-apply"
// DefaultPlanDir is where plans are written (the sudoers glob names it).
const DefaultPlanDir = "/var/lib/felhom-agent/os"
// Package is one name=version with its origin.
type Package struct {
Name string `json:"name"`
Version string `json:"version"`
Origin string `json:"origin"`
}
// Pending is one update the guest's sources offer (origin as apt names it, possibly several).
type Pending struct {
Name string `json:"name"`
From string `json:"from"`
To string `json:"to"`
Origin []string `json:"origin"`
}
// Container is one container's state as the wrapper saw it.
type Container struct {
State string `json:"state"`
Health string `json:"health"` // healthy | unhealthy | starting | none
}
// Health is the guest's health snapshot (the wrapper's `health` object).
type Health struct {
DockerOK bool `json:"docker_ok"`
NetworkOK bool `json:"network_ok"`
Controller string `json:"controller"`
Containers map[string]Container `json:"containers"`
}
// WrapperReport is the wrapper's OSAPPLY-REPORT object.
type WrapperReport struct {
Mode string `json:"mode"`
Refused json.RawMessage `json:"refused"`
Failed json.RawMessage `json:"failed"`
Upgraded []Package `json:"upgraded"`
Installed []Package `json:"installed"`
Pending []Pending `json:"pending"`
RestartNeeded []string `json:"restart_needed"`
DockerRestartNeeded bool `json:"docker_restart_needed"`
RebootNeeded bool `json:"reboot_needed"`
HealthBefore *Health `json:"health_before"`
HealthAfter *Health `json:"health_after"`
Health *Health `json:"health"`
}
func (w WrapperReport) refused() bool { return len(w.Refused) > 0 && string(w.Refused) != "null" }
func (w WrapperReport) failed() bool { return len(w.Failed) > 0 && string(w.Failed) != "null" }
// Report is what the hub receives (hub osupdates.Report — field-exact).
type Report struct {
RunID string `json:"run_id"`
Trigger string `json:"trigger"`
Mode string `json:"mode"`
Ring int `json:"ring"`
ReleaseID string `json:"release_id"`
Outcome string `json:"outcome"`
Healthy bool `json:"healthy"`
HealthReason string `json:"health_reason,omitempty"`
VMID int `json:"vmid"`
Upgraded []Package `json:"upgraded,omitempty"`
Installed []Package `json:"installed,omitempty"`
Pending []Pending `json:"pending,omitempty"`
NotCovered []string `json:"not_covered,omitempty"`
RestartNeeded []string `json:"restart_needed,omitempty"`
DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"`
RebootNeeded bool `json:"reboot_needed,omitempty"`
Refused json.RawMessage `json:"refused,omitempty"`
}
// Reporter posts a report to the hub (*hub.Client).
type Reporter interface {
PostOSReport(ctx context.Context, body []byte) error
}
// Leg runs one OS-update pass for the customer guest.
type Leg struct {
Runner proxmox.Runner
Hub Reporter
Logger *slog.Logger
PlanDir string
StatePath string // last night run (once per night)
HealthWait time.Duration // how long health may take to come back (default 5 min)
HealthPoll time.Duration // default 15 s
MinGap time.Duration // between night runs (default 20 h)
Now func() time.Time
Sleep func(context.Context, time.Duration)
mu sync.Mutex
block *hub.WireOSUpdate
have bool
}
// OnDesiredState stores the hub's os_update block (desired.RawConsumer — store only, never block).
func (l *Leg) OnDesiredState(_ context.Context, resp *hub.DesiredStateResponse) {
if resp == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
l.block, l.have = resp.DesiredState.OSUpdate, true
}
// Block returns the newest os_update block. No block (an older hub, or nothing fetched yet) = ring 1, ON, no
// release: the box reports and installs nothing.
func (l *Leg) Block() hub.WireOSUpdate {
l.mu.Lock()
defer l.mu.Unlock()
if l.block == nil {
return hub.WireOSUpdate{Ring: 1, Enabled: true}
}
return *l.block
}
// SetBlock sets the block directly (the selftest fetches the desired state itself).
func (l *Leg) SetBlock(b *hub.WireOSUpdate) {
l.mu.Lock()
defer l.mu.Unlock()
l.block, l.have = b, true
}
func (l *Leg) now() time.Time {
if l.Now != nil {
return l.Now()
}
return time.Now()
}
func (l *Leg) log() *slog.Logger {
if l.Logger != nil {
return l.Logger
}
return slog.Default()
}
func (l *Leg) sleep(ctx context.Context, d time.Duration) {
if l.Sleep != nil {
l.Sleep(ctx, d)
return
}
select {
case <-ctx.Done():
case <-time.After(d):
}
}
// IsFast reports whether every origin apt names is Debian / Debian-Security (the fast lane, `11` C3).
func IsFast(origins []string) bool {
if len(origins) == 0 {
return false
}
for _, o := range origins {
if o != "Debian" && o != "Debian-Security" {
return false
}
}
return true
}
func fastOrigin(origins []string) string {
for _, o := range origins {
if o == "Debian-Security" {
return o
}
}
return "Debian"
}
// HealthVerdict is THE health rule (`11` §5.4.1; pinned by TestHealthVerdict_*): after the run, docker answers, the
// guest's network resolves, the controller's own health check is `healthy`, and every container that was running
// before is running again — and healthy again if it was healthy before. "starting" is not yet healthy.
func HealthVerdict(before, after *Health) (bool, string) {
if after == nil {
return false, "no health reading"
}
if !after.DockerOK {
return false, "docker does not answer"
}
if !after.NetworkOK {
return false, "the guest cannot resolve deb.debian.org"
}
if after.Controller != "healthy" {
return false, "the controller is " + after.Controller
}
if before == nil {
return true, ""
}
names := make([]string, 0, len(before.Containers))
for n := range before.Containers {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
b := before.Containers[n]
if b.State != "running" {
continue
}
a, ok := after.Containers[n]
if !ok || a.State != "running" {
return false, n + " was running and is not"
}
if b.Health == "healthy" && a.Health != "healthy" {
return false, n + " was healthy and is " + a.Health
}
}
return true, ""
}
// call writes the plan and runs the wrapper once.
func (l *Leg) call(ctx context.Context, runID, mode string, vmid int, rel hub.WireOSRelease, pkgs []Package) (WrapperReport, error) {
dir := l.PlanDir
if dir == "" {
dir = DefaultPlanDir
}
if err := os.MkdirAll(dir, 0o700); err != nil {
return WrapperReport{}, fmt.Errorf("osupdate: plan dir: %w", err)
}
plan := map[string]any{"release_id": rel.ID, "layer": "guest", "lane": "fast", "vmid": vmid, "mode": mode,
"snapshot": rel.Snapshot, "packages": pkgs}
if pkgs == nil {
plan["packages"] = []Package{}
}
b, _ := json.Marshal(plan)
path := filepath.Join(dir, "plan-"+runID+"-"+mode+".json")
if err := os.WriteFile(path, b, 0o600); err != nil {
return WrapperReport{}, fmt.Errorf("osupdate: write plan: %w", err)
}
defer os.Remove(path)
stdout, stderr, err := l.Runner.Run(ctx, WrapperPath, "--plan", path)
for _, line := range strings.Split(strings.TrimSpace(string(stderr)), "\n") {
if strings.HasPrefix(line, "os-apply: ") {
l.log().Info("osupdate: wrapper", "line", line)
}
}
var rep WrapperReport
found := false
for _, line := range strings.Split(string(stdout), "\n") {
if strings.HasPrefix(line, "OSAPPLY-REPORT ") {
if jerr := json.Unmarshal([]byte(strings.TrimPrefix(line, "OSAPPLY-REPORT ")), &rep); jerr == nil {
found = true
}
}
}
if !found {
return rep, fmt.Errorf("osupdate: wrapper gave no report (err %v): %s", err, strings.TrimSpace(string(stderr)))
}
return rep, nil // a refusal / failure is IN the report (exit 2 / 3), not an error here
}
// Run is one pass: inventory → (switch, ring, plan) → apply → health → report. trigger is "night" or "debug".
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
runID := l.now().UTC().Format("20060102T150405Z")
blk := l.Block()
rel := hub.WireOSRelease{ID: "ring0-" + runID}
if blk.Ring == 1 {
rel = hub.WireOSRelease{}
if blk.Release != nil {
rel = *blk.Release
}
}
rep := Report{RunID: runID, Trigger: trigger, Ring: blk.Ring, VMID: vmid, ReleaseID: rel.ID, Mode: "inventory"}
lg := l.log().With("run", runID, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
if trigger == "night" && l.StatePath != "" {
gap := l.MinGap
if gap == 0 {
gap = 20 * time.Hour
}
if b, err := os.ReadFile(l.StatePath); err == nil {
if last, perr := time.Parse(time.RFC3339, strings.TrimSpace(string(b))); perr == nil && l.now().Sub(last) < gap {
lg.Info("osupdate: skipped — already ran tonight", "last", last.UTC().Format(time.RFC3339))
rep.Outcome = "skipped"
return rep
}
}
}
lg.Info("osupdate: START", "enabled", blk.Enabled, "release", rel.ID)
inv, err := l.call(ctx, runID, "inventory", vmid, rel, nil)
if err != nil || inv.refused() || inv.failed() {
rep.Outcome, rep.Refused = "refused", inv.Refused
if err != nil {
rep.Outcome, rep.HealthReason = "failed", err.Error()
}
return l.finish(ctx, lg, rep)
}
var plan []Package
switch {
case !blk.Enabled:
rep.Outcome = "inventory"
lg.Info("osupdate: switched OFF for this box — reporting only")
case blk.Ring == 0:
for _, p := range inv.Pending {
if IsFast(p.Origin) {
plan = append(plan, Package{Name: p.Name, Version: p.To, Origin: fastOrigin(p.Origin)})
}
}
default:
for _, p := range rel.Packages {
plan = append(plan, Package{Name: p.Name, Version: p.Version, Origin: p.Origin})
}
}
pkgNames := map[string]bool{}
for _, p := range plan {
pkgNames[p.Name] = true
}
if blk.Enabled && len(plan) == 0 {
rep.Outcome = "nothing"
}
final := inv
if blk.Enabled && len(plan) > 0 {
rep.Mode = "apply"
ap, err := l.call(ctx, runID, "apply", vmid, rel, plan)
switch {
case err != nil:
rep.Outcome, rep.HealthReason = "failed", err.Error()
return l.finish(ctx, lg, rep)
case ap.refused():
rep.Outcome, rep.Refused = "refused", ap.Refused
return l.finish(ctx, lg, rep)
case ap.failed():
rep.Outcome, rep.Refused = "failed", ap.Failed
}
final = ap
rep.Upgraded = ap.Upgraded
if rep.Outcome == "" {
if len(ap.Upgraded) == 0 {
rep.Outcome = "nothing"
} else {
rep.Outcome = "applied"
}
}
// Health: compare with what the guest looked like BEFORE the run; give restarted services time.
wait, poll := l.HealthWait, l.HealthPoll
if wait == 0 {
wait = 5 * time.Minute
}
if poll == 0 {
poll = 15 * time.Second
}
deadline := l.now().Add(wait)
cur := ap.HealthAfter
for {
ok, why := HealthVerdict(ap.HealthBefore, cur)
rep.Healthy, rep.HealthReason = ok, why
if ok || !l.now().Before(deadline) || ctx.Err() != nil {
break
}
l.sleep(ctx, poll)
hr, herr := l.call(ctx, runID, "health", vmid, rel, nil)
if herr == nil && hr.Health != nil {
cur = hr.Health
}
}
if !rep.Healthy && rep.Outcome == "applied" {
rep.Outcome = "health_failed"
}
} else {
ok, why := HealthVerdict(nil, inv.HealthAfter)
rep.Healthy, rep.HealthReason = ok, why
}
rep.Installed, rep.Pending = final.Installed, final.Pending
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = final.RestartNeeded, final.DockerRestartNeeded, final.RebootNeeded
rep.NotCovered = notCovered(final.Pending, blk.Ring, pkgNames)
if trigger == "night" && l.StatePath != "" {
_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
}
return l.finish(ctx, lg, rep)
}
// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in
// ring 1 also every fast-lane update the release did not name.
func notCovered(pending []Pending, ring int, planned map[string]bool) []string {
var out []string
for _, p := range pending {
if !IsFast(p.Origin) || (ring == 1 && !planned[p.Name]) {
out = append(out, p.Name)
}
}
return out
}
func (l *Leg) finish(ctx context.Context, lg *slog.Logger, rep Report) Report {
lg.Info("osupdate: DONE", "outcome", rep.Outcome, "healthy", rep.Healthy, "reason", rep.HealthReason,
"upgraded", len(rep.Upgraded), "pending", len(rep.Pending), "not_covered", len(rep.NotCovered), "restart_needed", len(rep.RestartNeeded))
if l.Hub != nil {
body, _ := json.Marshal(rep)
rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
defer cancel()
if err := l.Hub.PostOSReport(rctx, body); err != nil {
lg.Warn("osupdate: reporting to the hub failed (the run itself is done)", "err", err)
}
}
return rep
}
+265
View File
@@ -0,0 +1,265 @@
package osupdate
import (
"context"
"encoding/json"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)
// fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per mode.
type fakeWrapper struct {
t *testing.T
pending []Pending
applyRep WrapperReport
healthSeq []*Health // answers to successive "health" calls
plans []map[string]any
}
func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
if name != WrapperPath || len(args) != 2 || args[0] != "--plan" {
f.t.Fatalf("unexpected command %s %v", name, args)
}
b, err := os.ReadFile(args[1])
if err != nil {
f.t.Fatal(err)
}
var plan map[string]any
json.Unmarshal(b, &plan)
f.plans = append(f.plans, plan)
var rep WrapperReport
healthy := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "running", Health: "healthy"}}}
switch plan["mode"] {
case "inventory":
rep = WrapperReport{Mode: "inventory", Pending: f.pending, HealthAfter: healthy,
Installed: []Package{{Name: "libc6", Version: "u3", Origin: "Debian"}}}
case "apply":
rep = f.applyRep
rep.Mode = "apply"
if rep.HealthBefore == nil {
rep.HealthBefore = healthy
}
if rep.HealthAfter == nil {
rep.HealthAfter = healthy
}
case "health":
if len(f.healthSeq) > 0 {
rep.Health, f.healthSeq = f.healthSeq[0], f.healthSeq[1:]
} else {
rep.Health = healthy
}
}
out, _ := json.Marshal(rep)
return []byte("OSAPPLY-REPORT " + string(out) + "\n"), []byte("os-apply: DONE rc=0\n"), nil
}
func (f *fakeWrapper) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
return f.Run(ctx, name, args...)
}
type fakeHub struct{ reports []Report }
func (h *fakeHub) PostOSReport(_ context.Context, body []byte) error {
var r Report
json.Unmarshal(body, &r)
h.reports = append(h.reports, r)
return nil
}
func newLeg(t *testing.T, w *fakeWrapper, blk *hub.WireOSUpdate) (*Leg, *fakeHub) {
h := &fakeHub{}
now := time.Date(2026, 10, 4, 4, 0, 0, 0, time.UTC)
l := &Leg{Runner: w, Hub: h, PlanDir: t.TempDir(), StatePath: filepath.Join(t.TempDir(), "last"),
HealthWait: time.Minute, HealthPoll: 10 * time.Second,
Now: func() time.Time { return now },
Sleep: func(_ context.Context, d time.Duration) { now = now.Add(d) }}
if blk != nil {
l.SetBlock(blk)
}
return l, h
}
var pend = []Pending{
{Name: "libc6", From: "u3", To: "u4", Origin: []string{"Debian"}},
{Name: "openssl", From: "u1", To: "u3", Origin: []string{"Debian-Security", "Debian"}},
{Name: "docker-ce", From: "29.7", To: "29.8", Origin: []string{"Docker CE"}},
}
func modes(w *fakeWrapper) string {
var m []string
for _, p := range w.plans {
m = append(m, p["mode"].(string))
}
return strings.Join(m, ",")
}
// Ring 0 plans every pending FAST-LANE update (Docker excluded, `11` C3) and reports what it now runs.
func TestRing0_PlansTheFastLaneOnly(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6", Version: "u4"}, {Name: "openssl", Version: "u3"}}, Pending: pend[2:]}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "applied" || !rep.Healthy {
t.Fatalf("rep = %+v", rep)
}
pk := w.plans[1]["packages"].([]any)
if len(pk) != 2 {
t.Fatalf("ring-0 plan = %v, want libc6 + openssl only", pk)
}
if o := pk[1].(map[string]any)["origin"]; o != "Debian-Security" {
t.Fatalf("openssl origin = %v", o)
}
if w.plans[1]["snapshot"] != "" {
t.Fatalf("ring 0 installs from live sources, snapshot = %v", w.plans[1]["snapshot"])
}
if len(rep.NotCovered) != 1 || rep.NotCovered[0] != "docker-ce" {
t.Fatalf("not covered = %v", rep.NotCovered)
}
if len(h.reports) != 1 || h.reports[0].Outcome != "applied" {
t.Fatalf("hub got %+v", h.reports)
}
}
// Ring 1 installs EXACTLY the approved release (its versions, its snapshot), nothing it computed itself.
func TestRing1_InstallsExactlyTheRelease(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6", Version: "u4-approved"}}, Pending: pend[1:]}}
rel := &hub.WireOSRelease{ID: "os-1", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "u4-approved", Origin: "Debian"}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: rel})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "applied" || rep.ReleaseID != "os-1" {
t.Fatalf("rep = %+v", rep)
}
ap := w.plans[1]
pk := ap["packages"].([]any)
if len(pk) != 1 || pk[0].(map[string]any)["version"] != "u4-approved" || ap["snapshot"] != "20261004T080000Z" || ap["release_id"] != "os-1" {
t.Fatalf("ring-1 plan = %v", ap)
}
// openssl is pending and fast-lane but NOT in the release → not covered; docker-ce is never covered.
if strings.Join(rep.NotCovered, ",") != "openssl,docker-ce" {
t.Fatalf("not covered = %v", rep.NotCovered)
}
}
func TestRing1_NoReleaseInstallsNothing(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "nothing" || modes(w) != "inventory" {
t.Fatalf("rep=%+v modes=%s", rep, modes(w))
}
}
// No block from the hub (an older hub): ring 1, ON, no release → reports, installs nothing.
func TestNoBlock_IsRing1Nothing(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend}
l, _ := newLeg(t, w, nil)
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "nothing" || rep.Ring != 1 || modes(w) != "inventory" {
t.Fatalf("rep=%+v modes=%s", rep, modes(w))
}
}
// Switched OFF: the box reports but installs nothing (the brief, Part D 2).
func TestSwitchOff_ReportsOnly(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: false})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "inventory" || modes(w) != "inventory" || len(h.reports) != 1 || len(rep.Pending) != 3 {
t.Fatalf("rep=%+v modes=%s", rep, modes(w))
}
}
// Unhealthy after the run, and still unhealthy at the end of the wait → health_failed (the hub mails the operator).
func TestHealth_FailsAfterTheWait(t *testing.T) {
bad := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "exited"}}}
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad},
healthSeq: []*Health{bad, bad, bad, bad, bad, bad, bad, bad}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
rep := l.Run(context.Background(), 9201, "night")
if rep.Outcome != "health_failed" || rep.Healthy || !strings.Contains(rep.HealthReason, "app was running") {
t.Fatalf("rep = %+v", rep)
}
if h.reports[0].Outcome != "health_failed" {
t.Fatal("the hub was not told")
}
}
// A service that takes a moment to come back is not a failure: the poll sees it recover inside the wait.
func TestHealth_RecoversInsideTheWait(t *testing.T) {
starting := &Health{DockerOK: true, NetworkOK: true, Controller: "starting"}
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting},
healthSeq: []*Health{starting}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "applied" || !rep.Healthy {
t.Fatalf("rep = %+v", rep)
}
}
func TestHealthVerdict(t *testing.T) {
ok := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"a": {State: "running", Health: "healthy"}}}
cases := []struct {
name string
before *Health
after *Health
want bool
}{
{"all good", ok, ok, true},
{"no reading", ok, nil, false},
{"docker down", ok, &Health{NetworkOK: true, Controller: "healthy"}, false},
{"no network", ok, &Health{DockerOK: true, Controller: "healthy"}, false},
{"controller starting", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "starting"}, false},
{"only the controller differs", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "unhealthy", Containers: map[string]Container{"a": {State: "running", Health: "healthy"}}}, false},
{"app gone", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{}}, false},
{"app unhealthy", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"a": {State: "running", Health: "unhealthy"}}}, false},
{"stopped before stays stopped", &Health{Containers: map[string]Container{"x": {State: "exited"}}}, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy"}, true},
}
for _, c := range cases {
if got, why := HealthVerdict(c.before, c.after); got != c.want {
t.Errorf("%s: got %v (%s), want %v", c.name, got, why, c.want)
}
}
}
func TestOncePerNight(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Run(context.Background(), 9201, "night")
n := len(w.plans)
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "skipped" || len(w.plans) != n {
t.Fatalf("a second night run in the same night ran: %+v", rep)
}
if rep := l.Run(context.Background(), 9201, "debug"); rep.Outcome == "skipped" {
t.Fatal("the debug action must not be throttled")
}
}
func TestRefusedIsReported(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "refused" || h.reports[0].Outcome != "refused" {
t.Fatalf("rep = %+v", rep)
}
}
// The root wrapper's own suite (configs/test_felhom_os_apply.py) runs with `go test ./...` so CI covers it.
func TestWrapperSuite(t *testing.T) {
py, err := exec.LookPath("python3")
if err != nil {
t.Skip("python3 not available")
}
cmd := exec.Command(py, "../../configs/test_felhom_os_apply.py")
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("wrapper suite failed: %v\n%s", err, out)
}
if !strings.Contains(string(out), "OK") {
t.Fatalf("wrapper suite did not report OK:\n%s", out)
}
}