felhom.eu/hub/internal/osupdates/service.go

// Package osupdates is the hub half of the OS-update lanes (`11-os-updates.md` §5.3, §8 steps 2–4).
//
// Two LAYERS, each with its own approved set (hub v0.131.0): the customer guest's Debian packages and the Proxmox
// host's Debian packages. A version approved for one layer is NOT approved for the other by that fact alone.
//
// Ring 0 (the demo boxes) installs every pending Debian / Debian-Security fix each night and reports the FULL installed
// set per layer (C9). Per layer, the CANDIDATE is every Debian-origin package=version that all ring-0 boxes having it
// agree on (the host set leaves out kernel / boot / firmware names — the host's slow lane, `11` C3). A candidate is
// APPROVED when, since it was first seen:
//   - every ring-0 box runs it (its newest report of that layer matches the candidate for every package it has),
//   - ApproveAfter (default 24 h) has passed with every ring-0 report of that layer healthy, and
//   - every ring-0 box has completed NightsRequired (default 1) post-backup night runs of that layer.
//
// The operator can approve at once ("approve now"). Ring 1 gets each layer's newest release in its desired state.
// The fleet view and the four alarms (`11` §8 step 4) live here too. Rules pinned by service_test.go and
// alarms_test.go.
package osupdates

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
	"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
)

// Layers.
const (
	LayerGuest  = "guest"
	LayerHost   = "host"
	LayerDocker = "docker" // the guest's Docker engine set — slow lane, OPERATOR-approved only (`11` §5.8, hub v0.132.0)
)

// Layers lists the layers the hub approves AUTOMATICALLY (the fast lane).
var Layers = []string{LayerGuest, LayerHost}

// AllLayers adds the Docker engine set (approved only by the operator's button, ApproveDocker).
var AllLayers = []string{LayerGuest, LayerHost, LayerDocker}

// dockerNames are the six packages of the Docker engine set (the agent wrapper's DOCKER_NAMES).
var dockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true,
	"docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true}

// hostSlowRE mirrors the wrapper's HOST_SLOW_RE (felhom-agent configs/felhom-os-apply): kernel, boot and firmware
// packages are the host's slow lane and never enter a host release (the wrapper would refuse the whole plan, R14).
// Pinned by TestCandidate_HostLeavesOutKernelBootFirmware.
var hostSlowRE = regexp.MustCompile(`^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|pve-firmware|firmware-|grub|shim|systemd-boot|intel-microcode|amd64-microcode|efibootmgr)`)

// LegMinAgent is the first agent that runs the OS leg; the stale-leg alarm only watches boxes at or above it.
const LegMinAgent = "0.140.0"

// Event types — operator-only except EventApplied, the household's line (`11` §5.7).
const (
	EventApplied        = "os_update_applied"          // info, CUSTOMER: "system security fixes installed"
	EventFailed         = "os_update_failed"           // error, operator: the run failed or was refused
	EventHealthFailed   = "os_update_health_failed"    // error, operator: not healthy after the run
	EventReleaseApprove = "os_release_approved"        // info, operator
	EventApprovedNow    = "os_release_approved_now"    // warning, operator: an operator approved at once
	EventSettings       = "os_update_settings_changed" // info, operator: ring or switch changed
	EventStale          = "os_update_stale"            // warning, operator: no successful OS leg for StaleAfter
	EventRebootNeeded   = "os_reboot_needed"           // warning, operator: reboot needed for RebootAfter
	EventRing0Stalled   = "os_ring0_stalled"           // error, operator: ring 0 approved nothing for Ring0StallAfter
	EventNotCovered     = "os_not_covered"             // warning, operator: fast-lane packages no release covers
	// EventCancelled: a TEST approval was cancelled because the hub started without the TEST override (`11` §5.3.1).
	EventCancelled = "os_release_cancelled" // warning, operator
	// EventBundleBehind: a box's root-owned config bundle has differed from the vouched one for BundleBehindAfter (R-840).
	EventBundleBehind = "os_config_bundle_behind" // warning, operator
	// EventAgentBehind: a box has run an agent older than the vouched one for AgentBehindAfter (R-530, hub v0.135.0).
	// Agents update only by a per-box signed job (R-530's ruling), so a box nobody signed for silently stays behind.
	EventAgentBehind = "agent_behind" // warning, operator
)

// Package is one name=version with its origin ("Debian" | "Debian-Security").
type Package struct {
	Name    string `json:"name"`
	Version string `json:"version"`
	Origin  string `json:"origin"`
}

// Report is what the agent POSTs after every run of a layer.
type Report struct {
	RunID               string          `json:"run_id"`
	Layer               string          `json:"layer"` // guest | host ("" from agent 0.140.0 = guest)
	Trigger             string          `json:"trigger"`
	Mode                string          `json:"mode"`
	Ring                int             `json:"ring"`
	ReleaseID           string          `json:"release_id"`
	Outcome             string          `json:"outcome"` // applied | nothing | inventory | refused | failed | health_failed
	Healthy             bool            `json:"healthy"`
	HealthReason        string          `json:"health_reason,omitempty"`
	VMID                int             `json:"vmid"`
	Upgraded            []Package       `json:"upgraded,omitempty"`
	Installed           []Package       `json:"installed,omitempty"`
	Pending             []PendingPkg    `json:"pending,omitempty"`
	NotCovered          []string        `json:"not_covered,omitempty"`
	RestartNeeded       []string        `json:"restart_needed,omitempty"`
	DockerRestartNeeded bool            `json:"docker_restart_needed,omitempty"`
	RebootNeeded        bool            `json:"reboot_needed,omitempty"`
	RebootScanned       bool            `json:"reboot_scanned,omitempty"` // agent ≥ 0.141.1: the pass looked (host: every pass)
	Refused             json.RawMessage `json:"refused,omitempty"`
	PassSeconds         float64         `json:"pass_seconds,omitempty"`
	DockerEngine        string          `json:"docker_engine,omitempty"` // docker layer (agent v0.142.0)
	Authority           string          `json:"authority,omitempty"`     // docker layer: ring0 | signed
	Undo                bool            `json:"undo,omitempty"`          // docker layer: a signed undo
}

// PendingPkg is one update the sources offer.
type PendingPkg struct {
	Name   string   `json:"name"`
	From   string   `json:"from"`
	To     string   `json:"to"`
	Origin []string `json:"origin"`
}

func isFast(origins []string) bool {
	if len(origins) == 0 {
		return false
	}
	for _, o := range origins {
		if o != "Debian" && o != "Debian-Security" {
			return false
		}
	}
	return true
}

// fastPending is the pending updates a fast-lane release COULD carry for that layer.
func fastPending(layer string, p []PendingPkg) []PendingPkg {
	var out []PendingPkg
	for _, x := range p {
		if isFast(x.Origin) && !(layer == LayerHost && hostSlowRE.MatchString(x.Name)) {
			out = append(out, x)
		}
	}
	return out
}

// Block is what a box receives in its desired state (`os_update`).
type Block struct {
	Ring        int           `json:"ring"`
	Enabled     bool          `json:"enabled"`
	Release     *ReleaseBlock `json:"release,omitempty"`
	HostRelease *ReleaseBlock `json:"host_release,omitempty"`
}

// ReleaseBlock is a layer's newest approved release, for ring 1.
type ReleaseBlock struct {
	ID       string    `json:"id"`
	Snapshot string    `json:"snapshot"` // approval time as YYYYMMDDTHHMMSSZ (decision 79: snapshot.debian.org)
	Packages []Package `json:"packages"`
}

// Service ties the store, the events and the clock together.
type Service struct {
	Store          *store.Store
	Emit           func(customerID, eventType, severity, message, details, source string)
	ApproveAfter   time.Duration
	NightsRequired int
	// DockerNights is how many healthy ring-0 night runs of the Docker step an engine set needs before the operator's
	// button may approve it (`11` §5.8: 2; 0 means 2; negative means none). OS_DOCKER_APPROVE_NIGHTS overrides it for
	// a TEST only, logged.
	DockerNights int
	// Alarm thresholds (`11` §8.3; decided by CC unattended — operator may reverse). Zero = the default.
	StaleAfter      time.Duration // 7 d
	RebootAfter     time.Duration // 14 d
	Ring0StallAfter time.Duration // 7 d
	NotCoveredAfter time.Duration // 14 d
	// BundleBehindAfter: a box's config bundle differs from the vouched one this long → an operator alarm (R-840;
	// decided by CC unattended — operator may reverse). Zero = 7 d.
	BundleBehindAfter time.Duration
	// AgentBehindAfter: a box runs an agent older than the vouched one this long → an operator alarm (R-530; decided
	// by CC — operator may reverse, env OS_ALARM_AGENT_BEHIND_AFTER). Zero = 7 d.
	AgentBehindAfter time.Duration
	Logger           *log.Logger
	Now              func() time.Time
	Bump             func(hostID string)
	// TestOverride names the TEST overrides active at start ("" = none, the ruled waits). Every approval made while it
	// is set carries the `test` mark; CancelTestReleases cancels them at a start without it (`11` §5.3.1).
	TestOverride string
}

func (s *Service) now() time.Time {
	if s.Now != nil {
		return s.Now()
	}
	return time.Now()
}

func (s *Service) logf(f string, a ...any) {
	if s.Logger != nil {
		s.Logger.Printf(f, a...)
	}
}

func dflt(d, def time.Duration) time.Duration {
	if d <= 0 {
		return def
	}
	return d
}

func (s *Service) event(customerID, typ, sev, msg string, details any) {
	dj := ""
	if details != nil {
		if b, err := json.Marshal(details); err == nil {
			dj = string(b)
		}
	}
	if _, err := s.Store.SaveEvent(customerID, typ, sev, msg, dj, "hub"); err != nil {
		s.logf("[WARN] osupdates: save event %s: %v", typ, err)
	}
	if s.Emit != nil {
		s.Emit(customerID, typ, sev, msg, dj, "hub")
	}
}

func layerOf(r Report) string {
	switch r.Layer {
	case LayerHost, LayerDocker:
		return r.Layer
	}
	return LayerGuest
}

// Ingest stores a run and raises its events. The household gets one line per run that installed something.
func (s *Service) Ingest(hostID string, r Report) error {
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return fmt.Errorf("osupdates: unknown host %q", hostID)
	}
	layer := layerOf(r)
	r.Layer = layer
	raw, _ := json.Marshal(r)
	if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, Layer: layer, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode,
		Outcome: r.Outcome, Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil {
		return err
	}
	s.logf("[INFO] osupdates: %s reported %s run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d reboot-needed=%v wrapper=%.1fs",
		hostID, layer, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.RebootNeeded, r.PassSeconds)
	details := map[string]any{"host_id": hostID, "layer": layer, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome,
		"upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason}
	where := "the box"
	switch layer {
	case LayerHost:
		where = "the box's base system"
	case LayerDocker:
		where = "the box's app engine (Docker " + r.DockerEngine + ")"
	}
	switch r.Outcome {
	case "applied", "health_failed":
		s.event(h.CustomerID, EventApplied, "info",
			fmt.Sprintf("System security fixes installed on %s (%d package(s)).", where, len(r.Upgraded)), details)
		if !r.Healthy || r.Outcome == "health_failed" {
			undo := "last night's whole-guest backup is the undo (restore by hand, decision 81)"
			switch layer {
			case LayerHost:
				undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)"
			case LayerDocker:
				undo = "sign an os_docker_step with undo for the previous engine set (runbook `os-updates-docker-undo.md`)"
			}
			s.event(h.CustomerID, EventHealthFailed, "error",
				fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.",
					layer, hostID, len(r.Upgraded), r.HealthReason, undo), details)
		}
	case "refused", "failed":
		s.event(h.CustomerID, EventFailed, "error",
			fmt.Sprintf("OS update (%s) on %s %s: %s", layer, hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details)
	}
	return nil
}

// candidate derives, for one layer, the version set every ring-0 box agrees on, from each box's newest report.
func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, error) {
	byPkg := map[string]map[string]Package{} // name -> host -> pkg
	for _, h := range ring0 {
		rep, err := s.Store.LatestOSReport(h, layer)
		if err != nil {
			return nil, err
		}
		if rep == nil {
			return nil, nil // a ring-0 box that never reported this layer: no candidate
		}
		var r Report
		if err := json.Unmarshal([]byte(rep.ReportJSON), &r); err != nil {
			return nil, err
		}
		for _, p := range r.Installed {
			if layer == LayerDocker {
				if !dockerNames[p.Name] {
					continue
				}
			} else if p.Origin != "Debian" && p.Origin != "Debian-Security" {
				continue
			}
			if layer == LayerHost && hostSlowRE.MatchString(p.Name) {
				continue
			}
			if byPkg[p.Name] == nil {
				byPkg[p.Name] = map[string]Package{}
			}
			byPkg[p.Name][h] = p
		}
	}
	cand := map[string]Package{}
	for name, hosts := range byPkg {
		var v string
		agree := true
		var pick Package
		for _, p := range hosts {
			if v == "" {
				v, pick = p.Version, p
			} else if p.Version != v {
				agree = false
			}
		}
		if agree {
			cand[name] = pick
		}
	}
	return cand, nil
}

func fingerprint(layer string, c map[string]Package) (string, []Package) {
	var list []Package
	for _, p := range c {
		list = append(list, p)
	}
	sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name })
	h := sha256.New()
	fmt.Fprintf(h, "layer=%s\n", layer)
	for _, p := range list {
		fmt.Fprintf(h, "%s=%s\n", p.Name, p.Version)
	}
	return hex.EncodeToString(h.Sum(nil))[:16], list
}

// ring0Hosts lists the ring-0 boxes that have the switch ON.
func (s *Service) ring0Hosts() ([]string, error) {
	hosts, err := s.Store.ListHosts()
	if err != nil {
		return nil, err
	}
	var out []string
	for _, h := range hosts {
		if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 0 && st.Enabled {
			out = append(out, h.HostID)
		}
	}
	sort.Strings(out)
	return out, nil
}

// Status explains the approval state of one layer (for the log and the fleet page).
type Status struct {
	Layer       string
	Fingerprint string
	FirstSeen   time.Time
	Packages    int
	Waiting     string // why not approved yet ("" = approved or nothing to do)
	Approved    string // the release id when this set is already approved
}

// Evaluate checks the approval rule of every layer and approves when it holds. Called every minute.
func (s *Service) Evaluate() ([]Status, error) {
	var out []Status
	for _, layer := range Layers {
		st, err := s.evaluateLayer(layer)
		if err != nil {
			return out, err
		}
		out = append(out, st)
	}
	return out, nil
}

func (s *Service) evaluateLayer(layer string) (Status, error) { return s.layerStatus(layer, true) }

// LayerStatus is the read-only view of a fast-lane layer's candidate (the System page): it never approves.
func (s *Service) LayerStatus(layer string) (Status, error) { return s.layerStatus(layer, false) }

func (s *Service) layerStatus(layer string, approve bool) (Status, error) {
	ring0, err := s.ring0Hosts()
	if err != nil || len(ring0) == 0 {
		return Status{Layer: layer, Waiting: "no ring-0 box"}, err
	}
	cand, err := s.candidate(layer, ring0)
	if err != nil || cand == nil {
		return Status{Layer: layer, Waiting: "a ring-0 box has not reported this layer"}, err
	}
	fp, list := fingerprint(layer, cand)
	pj, _ := json.Marshal(list)
	first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
	if err != nil {
		return Status{}, err
	}
	st := Status{Layer: layer, Fingerprint: fp, FirstSeen: first, Packages: len(list)}
	if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp {
		st.Approved = rel.ID
		return st, nil // already approved
	}
	if age := s.now().Sub(first); age < s.ApproveAfter {
		st.Waiting = fmt.Sprintf("healthy for %s of %s", age.Round(time.Minute), s.ApproveAfter)
		return st, nil
	}
	for _, h := range ring0 {
		reps, err := s.Store.OSReportsSince(h, layer, first)
		if err != nil {
			return st, err
		}
		nights := 0
		for _, r := range reps {
			if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" {
				st.Waiting = fmt.Sprintf("%s reported %s (healthy=%v) at %s since the set was first seen", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339))
				return st, nil
			}
			if r.Trigger == "night" {
				nights++
			}
		}
		if nights < s.NightsRequired {
			st.Waiting = fmt.Sprintf("%s has %d of %d night run(s) since the set was first seen", h, nights, s.NightsRequired)
			return st, nil
		}
	}
	if !approve {
		st.Waiting = "ready — approves at the next minute's check"
		return st, nil
	}
	return st, s.approve(layer, fp, list, "auto")
}

// ReleaseInfo is one layer's newest approved release, for the System page.
type ReleaseInfo struct {
	Layer      string
	ID         string
	ApprovedAt time.Time
	ApprovedBy string
	Packages   int
	Test       bool   // a TEST approval still in force: amber on the System page
	Cancelled  string // set on a cancelled one (the page lists the last 7 days')
}

// Releases lists the newest release of every layer (guest, host, Docker); a layer with none is absent.
func (s *Service) Releases() []ReleaseInfo {
	var out []ReleaseInfo
	for _, layer := range AllLayers {
		rel, _ := s.Store.LatestOSRelease(layer)
		if rel == nil {
			continue
		}
		var list []Package
		_ = json.Unmarshal([]byte(rel.PackagesJSON), &list)
		out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list), Test: rel.Test})
	}
	return out
}

// CancelledReleases lists the approvals cancelled in the last 7 days (the System page says what stopped being served).
func (s *Service) CancelledReleases() []ReleaseInfo {
	rels, _ := s.Store.CancelledOSReleasesSince(s.now().Add(-7 * 24 * time.Hour))
	var out []ReleaseInfo
	for _, r := range rels {
		var list []Package
		_ = json.Unmarshal([]byte(r.PackagesJSON), &list)
		out = append(out, ReleaseInfo{Layer: r.Layer, ID: r.ID, ApprovedAt: r.ApprovedAt, ApprovedBy: r.ApprovedBy, Packages: len(list),
			Test: r.Test, Cancelled: r.CancelledAt})
	}
	return out
}

// Candidates is the read-only candidate status of every layer (the Docker one by its own rule).
func (s *Service) Candidates() []Status {
	var out []Status
	for _, layer := range Layers {
		st, _ := s.LayerStatus(layer)
		out = append(out, st)
	}
	d, _ := s.DockerStatus()
	return append(out, d)
}

// BundleThreshold is the config-bundle alarm's wait (the System page turns the cell red at it).
func (s *Service) BundleThreshold() time.Duration { return dflt(s.BundleBehindAfter, 7*24*time.Hour) }

// AgentThreshold is the agent-behind alarm's wait (R-530; the System page turns the cell red at it).
func (s *Service) AgentThreshold() time.Duration { return dflt(s.AgentBehindAfter, 7*24*time.Hour) }

// Thresholds are the alarm numbers the System page colours by (the same values the alarms use).
func (s *Service) Thresholds() (stale, reboot, notCovered time.Duration) {
	return dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour)
}

// ApproveNow approves every layer's current candidate at once (operator, urgent fix). Returns the release ids.
func (s *Service) ApproveNow() (string, error) {
	ring0, err := s.ring0Hosts()
	if err != nil || len(ring0) == 0 {
		return "", fmt.Errorf("osupdates: no ring-0 box")
	}
	var ids []string
	for _, layer := range Layers {
		cand, err := s.candidate(layer, ring0)
		if err != nil || cand == nil {
			continue
		}
		fp, list := fingerprint(layer, cand)
		pj, _ := json.Marshal(list)
		if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil {
			return "", err
		}
		if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp {
			ids = append(ids, rel.ID)
			continue
		}
		if err := s.approve(layer, fp, list, "operator"); err != nil {
			return "", err
		}
		rel, _ := s.Store.LatestOSRelease(layer)
		s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s (%s) at once (%d packages), without the wait.", rel.ID, layer, len(list)),
			map[string]any{"release_id": rel.ID, "layer": layer, "packages": len(list)})
		ids = append(ids, rel.ID)
	}
	if len(ids) == 0 {
		return "", fmt.Errorf("osupdates: a ring-0 box has not reported yet")
	}
	return strings.Join(ids, ","), nil
}

// DockerStatus is the Docker engine set ring 0 runs now and whether the operator's button may approve it: every ring-0
// box has run it in DockerNights healthy night Docker steps since it was first seen, and none unhealthy.
func (s *Service) DockerStatus() (Status, error) {
	st := Status{Layer: LayerDocker}
	ring0, err := s.ring0Hosts()
	if err != nil || len(ring0) == 0 {
		st.Waiting = "no ring-0 box"
		return st, err
	}
	cand, err := s.candidate(LayerDocker, ring0)
	if err != nil || len(cand) == 0 {
		st.Waiting = "a ring-0 box has not reported a Docker step"
		return st, err
	}
	fp, list := fingerprint(LayerDocker, cand)
	pj, _ := json.Marshal(list)
	first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
	if err != nil {
		return st, err
	}
	st.Fingerprint, st.FirstSeen, st.Packages = fp, first, len(list)
	if rel, _ := s.Store.LatestOSRelease(LayerDocker); rel != nil && rel.Fingerprint == fp {
		st.Approved, st.Waiting = rel.ID, "already approved"
		return st, nil
	}
	need := s.DockerNights // 0 = the default 2; negative = none (a TEST override only, logged at start)
	switch {
	case need == 0:
		need = 2
	case need < 0:
		need = 0
	}
	for _, h := range ring0 {
		reps, err := s.Store.OSReportsSince(h, LayerDocker, first)
		if err != nil {
			return st, err
		}
		nights := 0
		for _, r := range reps {
			if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" {
				st.Waiting = fmt.Sprintf("%s reported a Docker step %s (healthy=%v) at %s", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339))
				return st, nil
			}
			if r.Trigger == "night" {
				nights++
			}
		}
		if nights < need {
			st.Waiting = fmt.Sprintf("%s has %d of %d healthy night Docker step(s) with this set", h, nights, need)
			return st, nil
		}
	}
	return st, nil
}

// ApproveDocker is the operator's button: it approves the Docker engine set ring 0 runs, only when DockerStatus allows.
// A ring-1 box then takes it only through a signed operator job (`11` §5.8) — approval alone installs nothing.
func (s *Service) ApproveDocker() (string, error) {
	st, err := s.DockerStatus()
	if err != nil {
		return "", err
	}
	if st.Waiting != "" {
		return "", fmt.Errorf("osupdates: the Docker set cannot be approved yet: %s", st.Waiting)
	}
	ring0, _ := s.ring0Hosts()
	cand, err := s.candidate(LayerDocker, ring0)
	if err != nil {
		return "", err
	}
	fp, list := fingerprint(LayerDocker, cand)
	if err := s.approve(LayerDocker, fp, list, "operator"); err != nil {
		return "", err
	}
	rel, _ := s.Store.LatestOSRelease(LayerDocker)
	return rel.ID, nil
}

func (s *Service) approve(layer, fp string, list []Package, by string) error {
	at := s.now().UTC().Truncate(time.Second)
	id := "os-" + layer + "-" + at.Format("20060102-150405")
	pj, _ := json.Marshal(list)
	test := s.TestOverride != ""
	if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Layer: layer, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by,
		PackagesJSON: string(pj), Test: test}); err != nil {
		return err
	}
	mark := ""
	if test {
		mark = " — TEST approval (" + s.TestOverride + "); cancelled when the hub starts without the override"
	}
	s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)%s", id, layer, by, len(list), fp, mark)
	s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages)%s.", id, layer, by, len(list), mark),
		map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp, "test": test})
	if s.Bump != nil && layer != LayerDocker { // a Docker set reaches ring 1 only by a signed job, not the desired state
		hosts, _ := s.Store.ListHosts()
		for _, h := range hosts {
			if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled {
				s.Bump(h.HostID)
			}
		}
	}
	return nil
}

// CancelTestReleases runs at every hub start. Without a TEST override it cancels every test-marked approval that no
// real (non-test) approval has superseded: no ring-1 box installs it from then on; what boxes already installed stays.
// Each cancellation is an operator event; ring-1 boxes are bumped so their next plan has no cancelled release. With the
// override still active it does nothing (the test is still running). Returns the cancelled ids.
func (s *Service) CancelTestReleases() ([]string, error) {
	if s.TestOverride != "" {
		s.logf("[INFO] osupdates: TEST override active (%s) — test approvals stay in force", s.TestOverride)
		return nil, nil
	}
	var ids []string
	for _, layer := range AllLayers {
		rels, err := s.Store.UnsupersededTestReleases(layer)
		if err != nil {
			return ids, err
		}
		for _, r := range rels {
			reason := "approved under a TEST override; the hub started without it"
			if err := s.Store.CancelOSRelease(r.ID, reason, s.now()); err != nil {
				return ids, err
			}
			ids = append(ids, r.ID)
			s.logf("[WARN] osupdates: TEST approval %s (%s, approved %s by %s) CANCELLED — no ring-1 box installs it from now on",
				r.ID, layer, r.ApprovedAt.UTC().Format(time.RFC3339), r.ApprovedBy)
			s.event("", EventCancelled, "warning", fmt.Sprintf("OS release %s (%s) was a TEST approval and is cancelled: "+
				"no further box installs it. Boxes that already installed it keep it.", r.ID, layer),
				map[string]any{"release_id": r.ID, "layer": layer, "approved_at": r.ApprovedAt.UTC().Format(time.RFC3339), "approved_by": r.ApprovedBy})
		}
	}
	if len(ids) > 0 && s.Bump != nil {
		hosts, _ := s.Store.ListHosts()
		for _, h := range hosts {
			if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled {
				s.Bump(h.HostID)
			}
		}
	}
	return ids, nil
}

func (s *Service) releaseBlock(layer string) *ReleaseBlock {
	rel, err := s.Store.LatestOSRelease(layer)
	if err != nil || rel == nil {
		return nil
	}
	var list []Package
	if json.Unmarshal([]byte(rel.PackagesJSON), &list) != nil {
		return nil
	}
	return &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list}
}

// DesiredBlock is the `os_update` block a box receives.
func (s *Service) DesiredBlock(hostID string) Block {
	st := s.Store.GetOSHostSettings(hostID)
	b := Block{Ring: st.Ring, Enabled: st.Enabled}
	if st.Ring == 1 {
		b.Release, b.HostRelease = s.releaseBlock(LayerGuest), s.releaseBlock(LayerHost)
	}
	return b
}

// SetRing / SetEnabled are operator acts; each bumps the box's desired generation and is an operator event.
func (s *Service) SetRing(hostID string, ring int) error {
	if ring != 0 && ring != 1 {
		return fmt.Errorf("osupdates: ring must be 0 or 1")
	}
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return fmt.Errorf("osupdates: unknown host %q", hostID)
	}
	if err := s.Store.SetOSRing(hostID, ring); err != nil {
		return err
	}
	s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates: %s is now ring %d.", hostID, ring), map[string]any{"host_id": hostID, "ring": ring})
	if s.Bump != nil {
		s.Bump(hostID)
	}
	return nil
}

func (s *Service) SetEnabled(hostID string, on bool) error {
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return fmt.Errorf("osupdates: unknown host %q", hostID)
	}
	if err := s.Store.SetOSEnabled(hostID, on); err != nil {
		return err
	}
	s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates on %s switched %s.", hostID, map[bool]string{true: "ON", false: "OFF"}[on]),
		map[string]any{"host_id": hostID, "enabled": on})
	if s.Bump != nil {
		s.Bump(hostID)
	}
	return nil
}

// LayerLine is one layer of one box on the fleet view.
type LayerLine struct {
	ReleaseID          string
	LastOutcome        string
	LastAt             time.Time
	LastSuccessfulLeg  time.Time
	Pending            int
	NotCovered         int
	NotCoveredFast     int // fast-lane packages no approved release covers (the alarm's input)
	RestartNeeded      int
	RebootNeededSince  time.Time // zero = not needed
	WrapperPassSeconds float64
}

// FleetLine is one box on the fleet view.
type FleetLine struct {
	HostID  string
	Ring    int
	Enabled bool
	Tunnel  string
	Guest   LayerLine
	Host    LayerLine
	Docker  LayerLine
}

func (s *Service) layerLine(hostID, layer string, ring int) LayerLine {
	var l LayerLine
	rep, _ := s.Store.LatestOSReport(hostID, layer)
	if rep == nil {
		return l
	}
	var r Report
	_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
	l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt
	l.Pending, l.NotCovered, l.RestartNeeded, l.WrapperPassSeconds = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.PassSeconds
	if ok, _ := s.Store.LatestHealthyOSReport(hostID, layer, "applied", "nothing"); ok != nil {
		l.LastSuccessfulLeg = ok.ReceivedAt
	}
	l.NotCoveredFast = len(s.notCoveredFast(layer, ring, r))
	l.RebootNeededSince = s.sinceContinuously(hostID, layer, func(x Report) (bool, bool) {
		if !x.RebootScanned && len(x.Upgraded) == 0 && x.Outcome != "applied" {
			return false, false // a pass that did not look says nothing about reboot-needed: skip it
		}
		return x.RebootNeeded, true
	})
	return l
}

// notCoveredFast: pending fast-lane updates of that layer that the box's approved release does not name. Ring 0
// installs every fast-lane update itself, so it never has any.
func (s *Service) notCoveredFast(layer string, ring int, r Report) []string {
	if ring == 0 {
		return nil
	}
	named := map[string]bool{}
	if rb := s.releaseBlock(layer); rb != nil {
		for _, p := range rb.Packages {
			named[p.Name] = true
		}
	}
	var out []string
	for _, p := range fastPending(layer, r.Pending) {
		if !named[p.Name] {
			out = append(out, p.Name)
		}
	}
	return out
}

// sinceContinuously walks the box's reports of a layer newest-first and returns the time of the OLDEST report in the
// current unbroken run where pred holds (zero when the newest relevant report does not hold). pred's second result
// false = the report does not count either way (skipped).
func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (bool, bool)) time.Time {
	reps, err := s.Store.OSReportsDesc(hostID, layer, 400)
	if err != nil {
		return time.Time{}
	}
	var since time.Time
	for _, rep := range reps {
		var r Report
		if json.Unmarshal([]byte(rep.ReportJSON), &r) != nil {
			continue
		}
		hold, counts := pred(r)
		if !counts {
			continue
		}
		if !hold {
			break
		}
		since = rep.ReceivedAt
	}
	return since
}

// Fleet lists every box.
func (s *Service) Fleet() ([]FleetLine, error) {
	hosts, err := s.Store.ListHosts()
	if err != nil {
		return nil, err
	}
	var out []FleetLine
	for _, h := range hosts {
		st := s.Store.GetOSHostSettings(h.HostID)
		l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled, Tunnel: "unknown"}
		if t, _ := s.Store.RecentCloudflaredStatuses(h.HostID, 1); len(t) == 1 && t[0] != "" {
			l.Tunnel = t[0]
		}
		l.Guest = s.layerLine(h.HostID, LayerGuest, st.Ring)
		l.Host = s.layerLine(h.HostID, LayerHost, st.Ring)
		l.Docker = s.layerLine(h.HostID, LayerDocker, st.Ring)
		out = append(out, l)
	}
	return out, nil
}

// FleetJSON is Fleet plus the approval status, for the operator's fleet route.
func (s *Service) FleetJSON() (any, error) {
	lines, err := s.Fleet()
	if err != nil {
		return nil, err
	}
	out := map[string]any{"boxes": lines}
	for _, layer := range AllLayers {
		if rel, _ := s.Store.LatestOSRelease(layer); rel != nil {
			out["latest_"+layer+"_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy}
		}
	}
	return out, nil
}

// ---------------- the alarms (`11` §8.3) ----------------

// raise sends an alarm when its condition holds and it was not raised in the last week; clears it when it does not
// hold, so the next occurrence is announced again. Returns true when an event was sent.
func (s *Service) raise(key string, holds bool, customerID, typ, sev, msg string, details any) bool {
	last := s.Store.OSAlarmRaised(key)
	if !holds {
		if !last.IsZero() {
			_ = s.Store.SetOSAlarmRaised(key, time.Time{})
		}
		return false
	}
	if !last.IsZero() && s.now().Sub(last) < 7*24*time.Hour {
		return false
	}
	s.event(customerID, typ, sev, msg, details)
	_ = s.Store.SetOSAlarmRaised(key, s.now())
	return true
}

// Alarms evaluates the four OS-update alarms. Called hourly. Returns the event types it sent (for the log/tests).
func (s *Service) Alarms() ([]string, error) {
	now := s.now()
	stale, reboot, stall, nc := dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour),
		dflt(s.Ring0StallAfter, 7*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour)
	hosts, err := s.Store.ListHosts()
	if err != nil {
		return nil, err
	}
	var sent []string
	for _, h := range hosts {
		st := s.Store.GetOSHostSettings(h.HostID)
		// 1. No successful OS leg for `stale` while the switch is ON — the leg runs only after a good night backup,
		//    so a box with a broken backup silently stops updating. Only boxes whose agent CAN run the leg.
		canRun := h.AgentVersion != "" && semver.Valid(h.AgentVersion) && semver.Compare(h.AgentVersion, LegMinAgent) >= 0
		var ref time.Time
		if ok, _ := s.Store.LatestHealthyOSReport(h.HostID, LayerGuest, "applied", "nothing"); ok != nil {
			ref = ok.ReceivedAt
		} else if seen, _ := s.Store.HostAgentVersionsSeen(h.HostID); len(seen) > 0 {
			for v, at := range seen {
				if semver.Valid(v) && semver.Compare(v, LegMinAgent) >= 0 && (ref.IsZero() || at.Before(ref)) {
					ref = at
				}
			}
		}
		holds := st.Enabled && canRun && !ref.IsZero() && now.Sub(ref) >= stale
		if s.raise("stale:"+h.HostID, holds, h.CustomerID, EventStale, "warning",
			fmt.Sprintf("OS updates: %s has not completed an OS update leg for %s (since %s). Likely reason: %s.",
				h.HostID, now.Sub(ref).Round(time.Hour), ref.UTC().Format("2006-01-02 15:04"), s.staleReason(h)),
			map[string]any{"host_id": h.HostID, "since": ref}) {
			sent = append(sent, EventStale)
		}
		// 2. Reboot needed for `reboot` (host layer): a fix is installed but not fully in force.
		since := s.layerLine(h.HostID, LayerHost, st.Ring).RebootNeededSince
		if s.raise("reboot:"+h.HostID, !since.IsZero() && now.Sub(since) >= reboot, h.CustomerID, EventRebootNeeded, "warning",
			fmt.Sprintf("OS updates (for information): %s has needed a reboot since %s — host fixes are installed but not fully in force (PID 1 or lxc-start still run the old libraries). Nothing reboots automatically until the kernel lane exists.",
				h.HostID, since.UTC().Format("2006-01-02")), map[string]any{"host_id": h.HostID, "since": since}) {
			sent = append(sent, EventRebootNeeded)
		}
		// 4. Not-covered fast-lane packages for `nc` (ring 1: hardware ring 0 does not have, `11` edge case 9).
		for _, layer := range Layers {
			since := s.sinceContinuously(h.HostID, layer, func(r Report) (bool, bool) {
				return len(s.notCoveredFast(layer, st.Ring, r)) > 0, true
			})
			var names []string
			if rep, _ := s.Store.LatestOSReport(h.HostID, layer); rep != nil {
				var r Report
				_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
				names = s.notCoveredFast(layer, st.Ring, r)
			}
			if s.raise("notcovered:"+layer+":"+h.HostID, !since.IsZero() && now.Sub(since) >= nc, h.CustomerID, EventNotCovered, "warning",
				fmt.Sprintf("OS updates (%s): %s has had %d fast-lane package(s) no approved release covers since %s: %s. Ring 0 does not run them (other hardware?) — add matching hardware to ring 0, or approve by hand.",
					layer, h.HostID, len(names), since.UTC().Format("2006-01-02"), strings.Join(names, ", ")),
				map[string]any{"host_id": h.HostID, "layer": layer, "packages": names}) {
				sent = append(sent, EventNotCovered)
			}
		}
	}
	// 5. R-840: the box's ROOT-OWNED config bundle (sudoers, wrappers, units) differs from the vouched agent's for
	//    BundleBehindAfter. "none" (no bundle ever reached the box) counts as behind; "unknown" (the box could not say)
	//    counts as nothing — never a guess. Only when the vouched agent carries a bundle at all.
	man := s.Store.GetArtifactManifest()
	for _, h := range hosts {
		if man.BundleSHA256 == "" {
			break
		}
		rj, _ := s.Store.GetLatestHostReportJSON(h.CustomerID)
		sys := sysfacts.Parse(rj)
		if !sys.Present || sys.Bundle.Version == sysfacts.Unknown {
			continue
		}
		behind := sys.Bundle.BundleSHA256 != man.BundleSHA256
		since := s.Store.BundleBehindSince(h.HostID)
		switch {
		case !behind && !since.IsZero():
			_ = s.Store.SetBundleBehindSince(h.HostID, time.Time{})
			since = time.Time{}
		case behind && since.IsZero():
			since = now
			_ = s.Store.SetBundleBehindSince(h.HostID, since)
		}
		after := dflt(s.BundleBehindAfter, 7*24*time.Hour)
		if s.raise("bundle:"+h.HostID, behind && now.Sub(since) >= after, h.CustomerID, EventBundleBehind, "warning",
			fmt.Sprintf("Root files: %s still runs config bundle %s; the vouched agent %s carries a newer one (since %s). "+
				"Send it with a signed agent_config_update (`11` §5.4.2).", h.HostID, sys.Bundle.Version, man.AgentVersion,
				since.UTC().Format("2006-01-02")),
			map[string]any{"host_id": h.HostID, "box_bundle": sys.Bundle.Version, "vouched_agent": man.AgentVersion, "since": since}) {
			sent = append(sent, EventBundleBehind)
		}
	}
	// 6. R-530 (v0.135.0): a box runs an agent OLDER than the vouched one for AgentBehindAfter. Agents update only by a
	//    per-box signed job, so a box nobody signed for stays behind silently. An unreadable version (empty, not
	//    semver) is never a fact; nothing vouched → nothing is behind.
	for _, h := range hosts {
		if !semver.Valid(man.AgentVersion) {
			break
		}
		if !semver.Valid(h.AgentVersion) {
			continue
		}
		behind := semver.Compare(h.AgentVersion, man.AgentVersion) < 0
		since := s.Store.AgentBehindSince(h.HostID)
		switch {
		case !behind && !since.IsZero():
			_ = s.Store.SetAgentBehindSince(h.HostID, time.Time{})
			since = time.Time{}
		case behind && since.IsZero():
			since = now
			_ = s.Store.SetAgentBehindSince(h.HostID, since)
		}
		if s.raise("agent:"+h.HostID, behind && now.Sub(since) >= s.AgentThreshold(), h.CustomerID, EventAgentBehind, "warning",
			fmt.Sprintf("Agent: %s still runs agent %s; the vouched agent is %s (%s, behind since %s; last report %s). "+
				"Sign an agent_update for this box (felhom-opsign, `04` §3.1).", h.HostID, h.AgentVersion, man.AgentVersion,
				ReleasesBehind(h.AgentVersion, man.AgentVersion), since.UTC().Format("2006-01-02"), fmtTime(h.LastReportAt)),
			map[string]any{"host_id": h.HostID, "box_agent": h.AgentVersion, "vouched_agent": man.AgentVersion, "since": since}) {
			sent = append(sent, EventAgentBehind)
		}
	}
	// 3. Ring 0 approved nothing for `stall` while ring 0 has pending fast-lane updates: the whole fleet stopped
	//    getting fixes.
	ring0, _ := s.ring0Hosts()
	for _, layer := range Layers {
		var last time.Time
		if rel, _ := s.Store.LatestOSRelease(layer); rel != nil {
			last = rel.ApprovedAt
		} else if first, _ := s.Store.FirstOSReport(layer, ring0); first != nil {
			last = first.ReceivedAt
		}
		pendingAny := 0
		for _, hID := range ring0 {
			if rep, _ := s.Store.LatestOSReport(hID, layer); rep != nil {
				var r Report
				_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
				pendingAny += len(fastPending(layer, r.Pending))
			}
		}
		holds := len(ring0) > 0 && !last.IsZero() && now.Sub(last) >= stall && pendingAny > 0
		if s.raise("ring0stall:"+layer, holds, "", EventRing0Stalled, "error",
			fmt.Sprintf("OS updates (%s): ring 0 has approved nothing since %s while it has %d pending fast-lane update(s). The demo boxes are broken or unhealthy, and the whole fleet has stopped getting fixes.",
				layer, last.UTC().Format("2006-01-02 15:04"), pendingAny), map[string]any{"layer": layer, "pending": pendingAny}) {
			sent = append(sent, EventRing0Stalled)
		}
	}
	return sent, nil
}

// staleReason names the likely reason a box stopped completing the OS leg (for the operator's mail).
func (s *Service) staleReason(h store.Host) string {
	if h.LastReportAt == nil || s.now().Sub(*h.LastReportAt) > 24*time.Hour {
		return "the box is off or not reporting (last host report " + fmtTime(h.LastReportAt) + ")"
	}
	if rep, _ := s.Store.LatestOSReport(h.HostID, LayerGuest); rep != nil && (rep.Outcome == "failed" || rep.Outcome == "refused" || rep.Outcome == "health_failed" || !rep.Healthy) {
		var r Report
		_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
		return fmt.Sprintf("the last OS leg ended %s (%s) on %s", rep.Outcome, strings.TrimSpace(r.HealthReason+" "+string(r.Refused)), rep.ReceivedAt.UTC().Format("2006-01-02"))
	}
	return "no successful whole-guest backup reached the OS leg (it runs only after one) — check the box's night backups"
}

func fmtTime(t *time.Time) string {
	if t == nil {
		return "never"
	}
	return t.UTC().Format("2006-01-02 15:04")
}

// ReleasesBehind says how far version a is behind b, for the operator: "3 minor releases behind",
// "2 patch releases behind", "a major release behind". "" when a is not behind b or either is not semver.
func ReleasesBehind(a, b string) string {
	if !semver.Valid(a) || !semver.Valid(b) || semver.Compare(a, b) >= 0 {
		return ""
	}
	pa, pb := semverParts(a), semverParts(b)
	switch {
	case pa[0] != pb[0]:
		return "a major release behind"
	case pa[1] != pb[1]:
		n := pb[1] - pa[1]
		if n == 1 {
			return "1 minor release behind"
		}
		return fmt.Sprintf("%d minor releases behind", n)
	}
	n := pb[2] - pa[2]
	if n == 1 {
		return "1 patch release behind"
	}
	return fmt.Sprintf("%d patch releases behind", n)
}

func semverParts(v string) [3]int {
	var p [3]int
	v = strings.TrimPrefix(v, "v")
	if i := strings.IndexAny(v, "-+"); i >= 0 {
		v = v[:i]
	}
	for i, part := range strings.SplitN(v, ".", 3) {
		if i < 3 {
			p[i], _ = strconv.Atoi(part)
		}
	}
	return p
}