admin
c3d08b4821
gates / gates (push) Successful in 19s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
494 lines
17 KiB
Go
494 lines
17 KiB
Go
// Package osupdate is the agent's OS-update leg (`11-os-updates.md` §8 steps 2–3): the customer GUEST's Debian fast
|
||
// lane (agent v0.140.0) and, after it in the same pass, the HOST's (agent v0.141.0).
|
||
//
|
||
// It runs right after the night's successful whole-guest backup, while the backup goroutine still holds the host-wide
|
||
// heavy-op gate (so it never overlaps a backup or a restore-test, `11` C10), at most once per night. All root work is
|
||
// the wrapper `felhom-os-apply` (configs/, its own tests); this package only builds plans, calls the wrapper through
|
||
// sudo, judges health and reports to the hub — one report per layer.
|
||
//
|
||
// NO AUTOMATIC UNDO (R-837 measured; `09` §3 decision 81): a failed health check stops, reports `health_failed` and the
|
||
// hub mails the operator; the whole-guest backup taken minutes earlier is the guest's undo, by hand; a host package is
|
||
// put back by hand from the previous release's snapshot (runbook). The host is NEVER rebooted by this package.
|
||
package osupdate
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"log/slog"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||
)
|
||
|
||
// WrapperPath is the pinned sudoers vector (configs/felhom-agent.sudoers FELHOM_OSAPPLY).
|
||
const WrapperPath = "/usr/local/sbin/felhom-os-apply"
|
||
|
||
// DefaultPlanDir is where plans are written (the sudoers glob names it).
|
||
const DefaultPlanDir = "/var/lib/felhom-agent/os"
|
||
|
||
// Layers.
|
||
const (
|
||
LayerGuest = "guest"
|
||
LayerHost = "host"
|
||
)
|
||
|
||
// Package is one name=version with its origin.
|
||
type Package struct {
|
||
Name string `json:"name"`
|
||
Version string `json:"version"`
|
||
Origin string `json:"origin"`
|
||
}
|
||
|
||
// Pending is one update the sources offer (origin as apt names it, possibly several).
|
||
type Pending struct {
|
||
Name string `json:"name"`
|
||
From string `json:"from"`
|
||
To string `json:"to"`
|
||
Origin []string `json:"origin"`
|
||
}
|
||
|
||
// Container is one container's state as the wrapper saw it.
|
||
type Container struct {
|
||
State string `json:"state"`
|
||
Health string `json:"health"` // healthy | unhealthy | starting | none
|
||
}
|
||
|
||
// Health is one health reading. Guest layer: DockerOK..Containers. Host layer: HostServices, GuestRunning and the
|
||
// guest's own reading in Guest.
|
||
type Health struct {
|
||
DockerOK bool `json:"docker_ok"`
|
||
NetworkOK bool `json:"network_ok"`
|
||
Controller string `json:"controller"`
|
||
Containers map[string]Container `json:"containers"`
|
||
HostServices map[string]string `json:"host_services,omitempty"`
|
||
GuestRunning *bool `json:"guest_running,omitempty"`
|
||
Guest *Health `json:"guest,omitempty"`
|
||
}
|
||
|
||
// WrapperReport is the wrapper's OSAPPLY-REPORT object.
|
||
type WrapperReport struct {
|
||
Mode string `json:"mode"`
|
||
Layer string `json:"layer"`
|
||
Refused json.RawMessage `json:"refused"`
|
||
Failed json.RawMessage `json:"failed"`
|
||
Upgraded []Package `json:"upgraded"`
|
||
Installed []Package `json:"installed"`
|
||
Pending []Pending `json:"pending"`
|
||
RestartNeeded []string `json:"restart_needed"`
|
||
DockerRestartNeeded bool `json:"docker_restart_needed"`
|
||
RebootNeeded bool `json:"reboot_needed"`
|
||
HealthBefore *Health `json:"health_before"`
|
||
HealthAfter *Health `json:"health_after"`
|
||
Health *Health `json:"health"`
|
||
PassSeconds float64 `json:"pass_seconds"`
|
||
}
|
||
|
||
func (w WrapperReport) refused() bool { return len(w.Refused) > 0 && string(w.Refused) != "null" }
|
||
func (w WrapperReport) failed() bool { return len(w.Failed) > 0 && string(w.Failed) != "null" }
|
||
|
||
// Report is what the hub receives per layer (hub osupdates.Report — field-exact).
|
||
type Report struct {
|
||
RunID string `json:"run_id"`
|
||
Layer string `json:"layer"`
|
||
Trigger string `json:"trigger"`
|
||
Mode string `json:"mode"`
|
||
Ring int `json:"ring"`
|
||
ReleaseID string `json:"release_id"`
|
||
Outcome string `json:"outcome"`
|
||
Healthy bool `json:"healthy"`
|
||
HealthReason string `json:"health_reason,omitempty"`
|
||
VMID int `json:"vmid"`
|
||
Upgraded []Package `json:"upgraded,omitempty"`
|
||
Installed []Package `json:"installed,omitempty"`
|
||
Pending []Pending `json:"pending,omitempty"`
|
||
NotCovered []string `json:"not_covered,omitempty"`
|
||
RestartNeeded []string `json:"restart_needed,omitempty"`
|
||
DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"`
|
||
RebootNeeded bool `json:"reboot_needed,omitempty"`
|
||
Refused json.RawMessage `json:"refused,omitempty"`
|
||
PassSeconds float64 `json:"pass_seconds,omitempty"`
|
||
}
|
||
|
||
// Reporter posts a report to the hub (*hub.Client).
|
||
type Reporter interface {
|
||
PostOSReport(ctx context.Context, body []byte) error
|
||
}
|
||
|
||
// Leg runs one OS-update pass for the customer guest and then the host.
|
||
type Leg struct {
|
||
Runner proxmox.Runner
|
||
Hub Reporter
|
||
Tunnel hub.CloudflaredProber // the host health rule needs the tunnel `running` (R-841)
|
||
Appliance bool // agent.json deployment_mode; the wrapper re-checks the ROOT-owned record (R12)
|
||
Logger *slog.Logger
|
||
PlanDir string
|
||
StatePath string // last night run (once per night)
|
||
HealthWait time.Duration // how long health may take to come back (default 5 min)
|
||
HealthPoll time.Duration // default 15 s
|
||
MinGap time.Duration // between night runs (default 20 h)
|
||
Now func() time.Time
|
||
Sleep func(context.Context, time.Duration)
|
||
|
||
mu sync.Mutex
|
||
block *hub.WireOSUpdate
|
||
}
|
||
|
||
// OnDesiredState stores the hub's os_update block (desired.RawConsumer — store only, never block).
|
||
func (l *Leg) OnDesiredState(_ context.Context, resp *hub.DesiredStateResponse) {
|
||
if resp == nil {
|
||
return
|
||
}
|
||
l.mu.Lock()
|
||
defer l.mu.Unlock()
|
||
l.block = resp.DesiredState.OSUpdate
|
||
}
|
||
|
||
// Block returns the newest os_update block. No block (an older hub, or nothing fetched yet) = ring 1, ON, no
|
||
// release: the box reports and installs nothing.
|
||
func (l *Leg) Block() hub.WireOSUpdate {
|
||
l.mu.Lock()
|
||
defer l.mu.Unlock()
|
||
if l.block == nil {
|
||
return hub.WireOSUpdate{Ring: 1, Enabled: true}
|
||
}
|
||
return *l.block
|
||
}
|
||
|
||
// SetBlock sets the block directly (the selftest fetches the desired state itself).
|
||
func (l *Leg) SetBlock(b *hub.WireOSUpdate) {
|
||
l.mu.Lock()
|
||
defer l.mu.Unlock()
|
||
l.block = b
|
||
}
|
||
|
||
func (l *Leg) now() time.Time {
|
||
if l.Now != nil {
|
||
return l.Now()
|
||
}
|
||
return time.Now()
|
||
}
|
||
|
||
func (l *Leg) log() *slog.Logger {
|
||
if l.Logger != nil {
|
||
return l.Logger
|
||
}
|
||
return slog.Default()
|
||
}
|
||
|
||
func (l *Leg) sleep(ctx context.Context, d time.Duration) {
|
||
if l.Sleep != nil {
|
||
l.Sleep(ctx, d)
|
||
return
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
case <-time.After(d):
|
||
}
|
||
}
|
||
|
||
// IsFast reports whether every origin apt names is Debian / Debian-Security (the fast lane, `11` C3).
|
||
func IsFast(origins []string) bool {
|
||
if len(origins) == 0 {
|
||
return false
|
||
}
|
||
for _, o := range origins {
|
||
if o != "Debian" && o != "Debian-Security" {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// HealthVerdict is THE guest health rule (`11` §8.1; pinned by TestHealthVerdict*): docker answers, the guest's
|
||
// network resolves, the controller's own health check is `healthy`, and every container that was running at the
|
||
// start of the pass runs again — and healthy again if it was. "starting" is not yet healthy.
|
||
func HealthVerdict(before, after *Health) (bool, string) {
|
||
if after == nil {
|
||
return false, "no health reading"
|
||
}
|
||
if !after.DockerOK {
|
||
return false, "docker does not answer"
|
||
}
|
||
if !after.NetworkOK {
|
||
return false, "the guest cannot resolve deb.debian.org"
|
||
}
|
||
if after.Controller != "healthy" {
|
||
return false, "the controller is " + after.Controller
|
||
}
|
||
if before == nil {
|
||
return true, ""
|
||
}
|
||
names := make([]string, 0, len(before.Containers))
|
||
for n := range before.Containers {
|
||
names = append(names, n)
|
||
}
|
||
sort.Strings(names)
|
||
for _, n := range names {
|
||
b := before.Containers[n]
|
||
if b.State != "running" {
|
||
continue
|
||
}
|
||
a, ok := after.Containers[n]
|
||
if !ok || a.State != "running" {
|
||
return false, n + " was running and is not"
|
||
}
|
||
if b.Health == "healthy" && a.Health != "healthy" {
|
||
return false, n + " was healthy and is " + a.Health
|
||
}
|
||
}
|
||
return true, ""
|
||
}
|
||
|
||
// HostHealthVerdict is THE host health rule (`11` §8.2; pinned by TestHostHealthVerdict): the Proxmox daemons and the
|
||
// agent are active, the customer guest still runs, the guest's own rule still passes against the start of the pass,
|
||
// and the tunnel is `running` (R-841).
|
||
func HostHealthVerdict(before, after *Health, tunnel string) (bool, string) {
|
||
if after == nil {
|
||
return false, "no health reading"
|
||
}
|
||
svcs := make([]string, 0, len(after.HostServices))
|
||
for s := range after.HostServices {
|
||
svcs = append(svcs, s)
|
||
}
|
||
sort.Strings(svcs)
|
||
if len(svcs) == 0 {
|
||
return false, "no host service reading"
|
||
}
|
||
for _, s := range svcs {
|
||
if after.HostServices[s] != "active" {
|
||
return false, s + " is " + after.HostServices[s]
|
||
}
|
||
}
|
||
if after.GuestRunning == nil || !*after.GuestRunning {
|
||
return false, "the customer guest is not running"
|
||
}
|
||
var gb *Health
|
||
if before != nil {
|
||
gb = before.Guest
|
||
}
|
||
if ok, why := HealthVerdict(gb, after.Guest); !ok {
|
||
return false, "guest: " + why
|
||
}
|
||
if tunnel != hub.TunnelRunning {
|
||
return false, "the tunnel is " + tunnel
|
||
}
|
||
return true, ""
|
||
}
|
||
|
||
// call writes the plan and runs the wrapper once.
|
||
func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (WrapperReport, error) {
|
||
dir := l.PlanDir
|
||
if dir == "" {
|
||
dir = DefaultPlanDir
|
||
}
|
||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||
return WrapperReport{}, fmt.Errorf("osupdate: plan dir: %w", err)
|
||
}
|
||
b, _ := json.Marshal(plan)
|
||
path := filepath.Join(dir, fmt.Sprintf("plan-%s-%s-%s.json", runID, plan["layer"], plan["mode"]))
|
||
if err := os.WriteFile(path, b, 0o600); err != nil {
|
||
return WrapperReport{}, fmt.Errorf("osupdate: write plan: %w", err)
|
||
}
|
||
defer os.Remove(path)
|
||
stdout, stderr, err := l.Runner.Run(ctx, WrapperPath, "--plan", path)
|
||
for _, line := range strings.Split(strings.TrimSpace(string(stderr)), "\n") {
|
||
if strings.HasPrefix(line, "os-apply: ") {
|
||
l.log().Info("osupdate: wrapper", "line", line)
|
||
}
|
||
}
|
||
var rep WrapperReport
|
||
found := false
|
||
for _, line := range strings.Split(string(stdout), "\n") {
|
||
if strings.HasPrefix(line, "OSAPPLY-REPORT ") {
|
||
if jerr := json.Unmarshal([]byte(strings.TrimPrefix(line, "OSAPPLY-REPORT ")), &rep); jerr == nil {
|
||
found = true
|
||
}
|
||
}
|
||
}
|
||
if !found {
|
||
return rep, fmt.Errorf("osupdate: wrapper gave no report (err %v): %s", err, strings.TrimSpace(string(stderr)))
|
||
}
|
||
return rep, nil // a refusal / failure is IN the report (exit 2 / 3), not an error here
|
||
}
|
||
|
||
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer. Returns both
|
||
// reports (host empty when skipped). trigger is "night" or "debug".
|
||
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report, host Report) {
|
||
runID := l.now().UTC().Format("20060102T150405Z")
|
||
lg := l.log().With("run", runID, "vmid", vmid, "trigger", trigger)
|
||
if trigger == "night" && l.StatePath != "" {
|
||
gap := l.MinGap
|
||
if gap == 0 {
|
||
gap = 20 * time.Hour
|
||
}
|
||
if b, err := os.ReadFile(l.StatePath); err == nil {
|
||
if last, perr := time.Parse(time.RFC3339, strings.TrimSpace(string(b))); perr == nil && l.now().Sub(last) < gap {
|
||
lg.Info("osupdate: skipped — already ran tonight", "last", last.UTC().Format(time.RFC3339))
|
||
return Report{RunID: runID, Layer: LayerGuest, Outcome: "skipped"}, Report{}
|
||
}
|
||
}
|
||
}
|
||
blk := l.Block()
|
||
guest = l.runLayer(ctx, runID, LayerGuest, vmid, trigger, blk)
|
||
if trigger == "night" && l.StatePath != "" {
|
||
_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
|
||
}
|
||
switch {
|
||
case !l.Appliance:
|
||
lg.Info("osupdate: host step skipped — not an appliance install (a BYO host belongs to its owner, `11` §1)")
|
||
case !(guest.Outcome == "applied" || guest.Outcome == "nothing" || guest.Outcome == "inventory") || !guest.Healthy:
|
||
lg.Warn("osupdate: host step skipped — the guest step did not end healthy", "guest_outcome", guest.Outcome, "reason", guest.HealthReason)
|
||
default:
|
||
host = l.runLayer(ctx, runID, LayerHost, vmid, trigger, blk)
|
||
}
|
||
return guest, host
|
||
}
|
||
|
||
func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
|
||
rel := hub.WireOSRelease{ID: "ring0-" + runID}
|
||
var wire *hub.WireOSRelease
|
||
if layer == LayerGuest {
|
||
wire = blk.Release
|
||
} else {
|
||
wire = blk.HostRelease
|
||
}
|
||
if blk.Ring == 1 {
|
||
rel = hub.WireOSRelease{}
|
||
if wire != nil {
|
||
rel = *wire
|
||
}
|
||
}
|
||
rep := Report{RunID: runID, Layer: layer, Trigger: trigger, Ring: blk.Ring, VMID: vmid, ReleaseID: rel.ID}
|
||
lg := l.log().With("run", runID, "layer", layer, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
|
||
lg.Info("osupdate: START", "enabled", blk.Enabled, "release", rel.ID)
|
||
|
||
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": "fast", "vmid": vmid, "snapshot": rel.Snapshot,
|
||
"packages": []Package{}, "mode": "apply", "select": "listed"}
|
||
if rel.ID == "" {
|
||
plan["release_id"] = "none"
|
||
}
|
||
planned := map[string]bool{}
|
||
switch {
|
||
case !blk.Enabled:
|
||
plan["mode"] = "inventory"
|
||
lg.Info("osupdate: switched OFF for this box — reporting only")
|
||
case blk.Ring == 0:
|
||
plan["select"] = "pending-fast" // the wrapper picks every pending Debian / Debian-Security upgrade
|
||
case len(rel.Packages) == 0:
|
||
plan["mode"] = "inventory" // ring 1 with no approved release for this layer: nothing to install
|
||
default:
|
||
var pk []Package
|
||
for _, p := range rel.Packages {
|
||
pk = append(pk, Package{Name: p.Name, Version: p.Version, Origin: p.Origin})
|
||
planned[p.Name] = true
|
||
}
|
||
plan["packages"] = pk
|
||
}
|
||
rep.Mode = plan["mode"].(string)
|
||
wr, err := l.call(ctx, runID, plan)
|
||
switch {
|
||
case err != nil:
|
||
rep.Outcome, rep.HealthReason = "failed", err.Error()
|
||
return l.finish(ctx, lg, rep)
|
||
case wr.refused():
|
||
rep.Outcome, rep.Refused = "refused", wr.Refused
|
||
return l.finish(ctx, lg, rep)
|
||
case wr.failed():
|
||
rep.Outcome, rep.Refused = "failed", wr.Failed
|
||
}
|
||
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
|
||
if rep.Outcome == "" {
|
||
switch {
|
||
case rep.Mode == "inventory" && !blk.Enabled:
|
||
rep.Outcome = "inventory"
|
||
case len(wr.Upgraded) == 0:
|
||
rep.Outcome = "nothing"
|
||
default:
|
||
rep.Outcome = "applied"
|
||
}
|
||
}
|
||
if blk.Ring == 0 {
|
||
for _, u := range wr.Upgraded {
|
||
planned[u.Name] = true
|
||
}
|
||
}
|
||
// Health: compare with the start of the pass; give restarted services time (only after an install).
|
||
cur := wr.HealthAfter
|
||
verdict := func(h *Health) (bool, string) {
|
||
if layer == LayerHost {
|
||
t := hub.TunnelUnknown
|
||
if l.Tunnel != nil {
|
||
t, _ = l.Tunnel.Status(ctx)
|
||
}
|
||
return HostHealthVerdict(wr.HealthBefore, h, t)
|
||
}
|
||
return HealthVerdict(wr.HealthBefore, h)
|
||
}
|
||
if len(wr.Upgraded) > 0 {
|
||
wait, poll := l.HealthWait, l.HealthPoll
|
||
if wait == 0 {
|
||
wait = 5 * time.Minute
|
||
}
|
||
if poll == 0 {
|
||
poll = 15 * time.Second
|
||
}
|
||
deadline := l.now().Add(wait)
|
||
for {
|
||
ok, why := verdict(cur)
|
||
rep.Healthy, rep.HealthReason = ok, why
|
||
if ok || !l.now().Before(deadline) || ctx.Err() != nil {
|
||
break
|
||
}
|
||
l.sleep(ctx, poll)
|
||
hp := map[string]any{"release_id": plan["release_id"], "layer": layer, "lane": "fast", "vmid": vmid, "mode": "health", "packages": []Package{}}
|
||
hr, herr := l.call(ctx, runID, hp)
|
||
if herr == nil && hr.Health != nil {
|
||
cur = hr.Health
|
||
}
|
||
}
|
||
if !rep.Healthy && rep.Outcome == "applied" {
|
||
rep.Outcome = "health_failed"
|
||
}
|
||
} else {
|
||
rep.Healthy, rep.HealthReason = verdict(cur)
|
||
}
|
||
rep.Installed, rep.Pending = wr.Installed, wr.Pending
|
||
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = wr.RestartNeeded, wr.DockerRestartNeeded, wr.RebootNeeded
|
||
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
|
||
return l.finish(ctx, lg, rep)
|
||
}
|
||
|
||
// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in ring 1
|
||
// also every fast-lane update the release did not name.
|
||
func notCovered(pending []Pending, ring int, planned map[string]bool) []string {
|
||
var out []string
|
||
for _, p := range pending {
|
||
if !IsFast(p.Origin) || (ring == 1 && !planned[p.Name]) {
|
||
out = append(out, p.Name)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (l *Leg) finish(ctx context.Context, lg *slog.Logger, rep Report) Report {
|
||
lg.Info("osupdate: DONE", "outcome", rep.Outcome, "healthy", rep.Healthy, "reason", rep.HealthReason,
|
||
"upgraded", len(rep.Upgraded), "pending", len(rep.Pending), "not_covered", len(rep.NotCovered),
|
||
"restart_needed", len(rep.RestartNeeded), "reboot_needed", rep.RebootNeeded, "wrapper_seconds", rep.PassSeconds)
|
||
if l.Hub != nil {
|
||
body, _ := json.Marshal(rep)
|
||
rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
|
||
defer cancel()
|
||
if err := l.Hub.PostOSReport(rctx, body); err != nil {
|
||
lg.Warn("osupdate: reporting to the hub failed (the run itself is done)", "err", err)
|
||
}
|
||
}
|
||
return rep
|
||
}
|