felhom.eu/hub/internal/osupdates/service.go

// Package osupdates is the hub half of the OS-update lanes (`11-os-updates.md` §5.3, §8 steps 2–4).
//
// Two LAYERS, each with its own approved set (hub v0.131.0): the customer guest's Debian packages and the Proxmox
// host's Debian packages. A version approved for one layer is NOT approved for the other by that fact alone.
//
// Ring 0 (the demo boxes) installs every pending Debian / Debian-Security fix each night and reports the FULL installed
// set per layer (C9). Per layer, the CANDIDATE is every Debian-origin package=version that all ring-0 boxes having it
// agree on (the host set leaves out kernel / boot / firmware names — the host's slow lane, `11` C3). A candidate is
// APPROVED when, since it was first seen:
//   - every ring-0 box runs it (its newest report of that layer matches the candidate for every package it has),
//   - ApproveAfter (default 24 h) has passed with every ring-0 report of that layer healthy, and
//   - every ring-0 box has completed NightsRequired (default 1) post-backup night runs of that layer.
//
// The operator can approve at once ("approve now"). Ring 1 gets each layer's newest release in its desired state.
// The fleet view and the four alarms (`11` §8 step 4) live here too. Rules pinned by service_test.go and
// alarms_test.go.
package osupdates

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"sort"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// Layers.
const (
	LayerGuest = "guest"
	LayerHost  = "host"
)

// Layers lists every layer the hub approves.
var Layers = []string{LayerGuest, LayerHost}

// hostSlowRE mirrors the wrapper's HOST_SLOW_RE (felhom-agent configs/felhom-os-apply): kernel, boot and firmware
// packages are the host's slow lane and never enter a host release (the wrapper would refuse the whole plan, R14).
// Pinned by TestCandidate_HostLeavesOutKernelBootFirmware.
var hostSlowRE = regexp.MustCompile(`^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|pve-firmware|firmware-|grub|shim|systemd-boot|intel-microcode|amd64-microcode|efibootmgr)`)

// LegMinAgent is the first agent that runs the OS leg; the stale-leg alarm only watches boxes at or above it.
const LegMinAgent = "0.140.0"

// Event types — operator-only except EventApplied, the household's line (`11` §5.7).
const (
	EventApplied        = "os_update_applied"          // info, CUSTOMER: "system security fixes installed"
	EventFailed         = "os_update_failed"           // error, operator: the run failed or was refused
	EventHealthFailed   = "os_update_health_failed"    // error, operator: not healthy after the run
	EventReleaseApprove = "os_release_approved"        // info, operator
	EventApprovedNow    = "os_release_approved_now"    // warning, operator: an operator approved at once
	EventSettings       = "os_update_settings_changed" // info, operator: ring or switch changed
	EventStale          = "os_update_stale"            // warning, operator: no successful OS leg for StaleAfter
	EventRebootNeeded   = "os_reboot_needed"           // warning, operator: reboot needed for RebootAfter
	EventRing0Stalled   = "os_ring0_stalled"           // error, operator: ring 0 approved nothing for Ring0StallAfter
	EventNotCovered     = "os_not_covered"             // warning, operator: fast-lane packages no release covers
)

// Package is one name=version with its origin ("Debian" | "Debian-Security").
type Package struct {
	Name    string `json:"name"`
	Version string `json:"version"`
	Origin  string `json:"origin"`
}

// Report is what the agent POSTs after every run of a layer.
type Report struct {
	RunID               string          `json:"run_id"`
	Layer               string          `json:"layer"` // guest | host ("" from agent 0.140.0 = guest)
	Trigger             string          `json:"trigger"`
	Mode                string          `json:"mode"`
	Ring                int             `json:"ring"`
	ReleaseID           string          `json:"release_id"`
	Outcome             string          `json:"outcome"` // applied | nothing | inventory | refused | failed | health_failed
	Healthy             bool            `json:"healthy"`
	HealthReason        string          `json:"health_reason,omitempty"`
	VMID                int             `json:"vmid"`
	Upgraded            []Package       `json:"upgraded,omitempty"`
	Installed           []Package       `json:"installed,omitempty"`
	Pending             []PendingPkg    `json:"pending,omitempty"`
	NotCovered          []string        `json:"not_covered,omitempty"`
	RestartNeeded       []string        `json:"restart_needed,omitempty"`
	DockerRestartNeeded bool            `json:"docker_restart_needed,omitempty"`
	RebootNeeded        bool            `json:"reboot_needed,omitempty"`
	RebootScanned       bool            `json:"reboot_scanned,omitempty"` // agent ≥ 0.141.1: the pass looked (host: every pass)
	Refused             json.RawMessage `json:"refused,omitempty"`
	PassSeconds         float64         `json:"pass_seconds,omitempty"`
}

// PendingPkg is one update the sources offer.
type PendingPkg struct {
	Name   string   `json:"name"`
	From   string   `json:"from"`
	To     string   `json:"to"`
	Origin []string `json:"origin"`
}

func isFast(origins []string) bool {
	if len(origins) == 0 {
		return false
	}
	for _, o := range origins {
		if o != "Debian" && o != "Debian-Security" {
			return false
		}
	}
	return true
}

// fastPending is the pending updates a fast-lane release COULD carry for that layer.
func fastPending(layer string, p []PendingPkg) []PendingPkg {
	var out []PendingPkg
	for _, x := range p {
		if isFast(x.Origin) && !(layer == LayerHost && hostSlowRE.MatchString(x.Name)) {
			out = append(out, x)
		}
	}
	return out
}

// Block is what a box receives in its desired state (`os_update`).
type Block struct {
	Ring        int           `json:"ring"`
	Enabled     bool          `json:"enabled"`
	Release     *ReleaseBlock `json:"release,omitempty"`
	HostRelease *ReleaseBlock `json:"host_release,omitempty"`
}

// ReleaseBlock is a layer's newest approved release, for ring 1.
type ReleaseBlock struct {
	ID       string    `json:"id"`
	Snapshot string    `json:"snapshot"` // approval time as YYYYMMDDTHHMMSSZ (decision 79: snapshot.debian.org)
	Packages []Package `json:"packages"`
}

// Service ties the store, the events and the clock together.
type Service struct {
	Store          *store.Store
	Emit           func(customerID, eventType, severity, message, details, source string)
	ApproveAfter   time.Duration
	NightsRequired int
	// Alarm thresholds (`11` §8.3; decided by CC unattended — operator may reverse). Zero = the default.
	StaleAfter      time.Duration // 7 d
	RebootAfter     time.Duration // 14 d
	Ring0StallAfter time.Duration // 7 d
	NotCoveredAfter time.Duration // 14 d
	Logger          *log.Logger
	Now             func() time.Time
	Bump            func(hostID string)
}

func (s *Service) now() time.Time {
	if s.Now != nil {
		return s.Now()
	}
	return time.Now()
}

func (s *Service) logf(f string, a ...any) {
	if s.Logger != nil {
		s.Logger.Printf(f, a...)
	}
}

func dflt(d, def time.Duration) time.Duration {
	if d <= 0 {
		return def
	}
	return d
}

func (s *Service) event(customerID, typ, sev, msg string, details any) {
	dj := ""
	if details != nil {
		if b, err := json.Marshal(details); err == nil {
			dj = string(b)
		}
	}
	if _, err := s.Store.SaveEvent(customerID, typ, sev, msg, dj, "hub"); err != nil {
		s.logf("[WARN] osupdates: save event %s: %v", typ, err)
	}
	if s.Emit != nil {
		s.Emit(customerID, typ, sev, msg, dj, "hub")
	}
}

func layerOf(r Report) string {
	if r.Layer == LayerHost {
		return LayerHost
	}
	return LayerGuest
}

// Ingest stores a run and raises its events. The household gets one line per run that installed something.
func (s *Service) Ingest(hostID string, r Report) error {
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return fmt.Errorf("osupdates: unknown host %q", hostID)
	}
	layer := layerOf(r)
	r.Layer = layer
	raw, _ := json.Marshal(r)
	if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, Layer: layer, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode,
		Outcome: r.Outcome, Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil {
		return err
	}
	s.logf("[INFO] osupdates: %s reported %s run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d reboot-needed=%v wrapper=%.1fs",
		hostID, layer, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.RebootNeeded, r.PassSeconds)
	details := map[string]any{"host_id": hostID, "layer": layer, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome,
		"upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason}
	where := "the box"
	if layer == LayerHost {
		where = "the box's base system"
	}
	switch r.Outcome {
	case "applied", "health_failed":
		s.event(h.CustomerID, EventApplied, "info",
			fmt.Sprintf("System security fixes installed on %s (%d package(s)).", where, len(r.Upgraded)), details)
		if !r.Healthy || r.Outcome == "health_failed" {
			undo := "last night's whole-guest backup is the undo (restore by hand, decision 81)"
			if layer == LayerHost {
				undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)"
			}
			s.event(h.CustomerID, EventHealthFailed, "error",
				fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.",
					layer, hostID, len(r.Upgraded), r.HealthReason, undo), details)
		}
	case "refused", "failed":
		s.event(h.CustomerID, EventFailed, "error",
			fmt.Sprintf("OS update (%s) on %s %s: %s", layer, hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details)
	}
	return nil
}

// candidate derives, for one layer, the version set every ring-0 box agrees on, from each box's newest report.
func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, error) {
	byPkg := map[string]map[string]Package{} // name -> host -> pkg
	for _, h := range ring0 {
		rep, err := s.Store.LatestOSReport(h, layer)
		if err != nil {
			return nil, err
		}
		if rep == nil {
			return nil, nil // a ring-0 box that never reported this layer: no candidate
		}
		var r Report
		if err := json.Unmarshal([]byte(rep.ReportJSON), &r); err != nil {
			return nil, err
		}
		for _, p := range r.Installed {
			if p.Origin != "Debian" && p.Origin != "Debian-Security" {
				continue
			}
			if layer == LayerHost && hostSlowRE.MatchString(p.Name) {
				continue
			}
			if byPkg[p.Name] == nil {
				byPkg[p.Name] = map[string]Package{}
			}
			byPkg[p.Name][h] = p
		}
	}
	cand := map[string]Package{}
	for name, hosts := range byPkg {
		var v string
		agree := true
		var pick Package
		for _, p := range hosts {
			if v == "" {
				v, pick = p.Version, p
			} else if p.Version != v {
				agree = false
			}
		}
		if agree {
			cand[name] = pick
		}
	}
	return cand, nil
}

func fingerprint(layer string, c map[string]Package) (string, []Package) {
	var list []Package
	for _, p := range c {
		list = append(list, p)
	}
	sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name })
	h := sha256.New()
	fmt.Fprintf(h, "layer=%s\n", layer)
	for _, p := range list {
		fmt.Fprintf(h, "%s=%s\n", p.Name, p.Version)
	}
	return hex.EncodeToString(h.Sum(nil))[:16], list
}

// ring0Hosts lists the ring-0 boxes that have the switch ON.
func (s *Service) ring0Hosts() ([]string, error) {
	hosts, err := s.Store.ListHosts()
	if err != nil {
		return nil, err
	}
	var out []string
	for _, h := range hosts {
		if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 0 && st.Enabled {
			out = append(out, h.HostID)
		}
	}
	sort.Strings(out)
	return out, nil
}

// Status explains the approval state of one layer (for the log and the fleet page).
type Status struct {
	Layer       string
	Fingerprint string
	FirstSeen   time.Time
	Packages    int
	Waiting     string // why not approved yet ("" = approved or nothing to do)
}

// Evaluate checks the approval rule of every layer and approves when it holds. Called every minute.
func (s *Service) Evaluate() ([]Status, error) {
	var out []Status
	for _, layer := range Layers {
		st, err := s.evaluateLayer(layer)
		if err != nil {
			return out, err
		}
		out = append(out, st)
	}
	return out, nil
}

func (s *Service) evaluateLayer(layer string) (Status, error) {
	ring0, err := s.ring0Hosts()
	if err != nil || len(ring0) == 0 {
		return Status{Layer: layer, Waiting: "no ring-0 box"}, err
	}
	cand, err := s.candidate(layer, ring0)
	if err != nil || cand == nil {
		return Status{Layer: layer, Waiting: "a ring-0 box has not reported this layer"}, err
	}
	fp, list := fingerprint(layer, cand)
	pj, _ := json.Marshal(list)
	first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
	if err != nil {
		return Status{}, err
	}
	st := Status{Layer: layer, Fingerprint: fp, FirstSeen: first, Packages: len(list)}
	if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp {
		return st, nil // already approved
	}
	if age := s.now().Sub(first); age < s.ApproveAfter {
		st.Waiting = fmt.Sprintf("healthy for %s of %s", age.Round(time.Minute), s.ApproveAfter)
		return st, nil
	}
	for _, h := range ring0 {
		reps, err := s.Store.OSReportsSince(h, layer, first)
		if err != nil {
			return st, err
		}
		nights := 0
		for _, r := range reps {
			if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" {
				st.Waiting = fmt.Sprintf("%s reported %s (healthy=%v) at %s since the set was first seen", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339))
				return st, nil
			}
			if r.Trigger == "night" {
				nights++
			}
		}
		if nights < s.NightsRequired {
			st.Waiting = fmt.Sprintf("%s has %d of %d night run(s) since the set was first seen", h, nights, s.NightsRequired)
			return st, nil
		}
	}
	return st, s.approve(layer, fp, list, "auto")
}

// ApproveNow approves every layer's current candidate at once (operator, urgent fix). Returns the release ids.
func (s *Service) ApproveNow() (string, error) {
	ring0, err := s.ring0Hosts()
	if err != nil || len(ring0) == 0 {
		return "", fmt.Errorf("osupdates: no ring-0 box")
	}
	var ids []string
	for _, layer := range Layers {
		cand, err := s.candidate(layer, ring0)
		if err != nil || cand == nil {
			continue
		}
		fp, list := fingerprint(layer, cand)
		pj, _ := json.Marshal(list)
		if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil {
			return "", err
		}
		if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp {
			ids = append(ids, rel.ID)
			continue
		}
		if err := s.approve(layer, fp, list, "operator"); err != nil {
			return "", err
		}
		rel, _ := s.Store.LatestOSRelease(layer)
		s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s (%s) at once (%d packages), without the wait.", rel.ID, layer, len(list)),
			map[string]any{"release_id": rel.ID, "layer": layer, "packages": len(list)})
		ids = append(ids, rel.ID)
	}
	if len(ids) == 0 {
		return "", fmt.Errorf("osupdates: a ring-0 box has not reported yet")
	}
	return strings.Join(ids, ","), nil
}

func (s *Service) approve(layer, fp string, list []Package, by string) error {
	at := s.now().UTC().Truncate(time.Second)
	id := "os-" + layer + "-" + at.Format("20060102-150405")
	pj, _ := json.Marshal(list)
	if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Layer: layer, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil {
		return err
	}
	s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)", id, layer, by, len(list), fp)
	s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages).", id, layer, by, len(list)),
		map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp})
	if s.Bump != nil {
		hosts, _ := s.Store.ListHosts()
		for _, h := range hosts {
			if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled {
				s.Bump(h.HostID)
			}
		}
	}
	return nil
}

func (s *Service) releaseBlock(layer string) *ReleaseBlock {
	rel, err := s.Store.LatestOSRelease(layer)
	if err != nil || rel == nil {
		return nil
	}
	var list []Package
	if json.Unmarshal([]byte(rel.PackagesJSON), &list) != nil {
		return nil
	}
	return &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list}
}

// DesiredBlock is the `os_update` block a box receives.
func (s *Service) DesiredBlock(hostID string) Block {
	st := s.Store.GetOSHostSettings(hostID)
	b := Block{Ring: st.Ring, Enabled: st.Enabled}
	if st.Ring == 1 {
		b.Release, b.HostRelease = s.releaseBlock(LayerGuest), s.releaseBlock(LayerHost)
	}
	return b
}

// SetRing / SetEnabled are operator acts; each bumps the box's desired generation and is an operator event.
func (s *Service) SetRing(hostID string, ring int) error {
	if ring != 0 && ring != 1 {
		return fmt.Errorf("osupdates: ring must be 0 or 1")
	}
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return fmt.Errorf("osupdates: unknown host %q", hostID)
	}
	if err := s.Store.SetOSRing(hostID, ring); err != nil {
		return err
	}
	s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates: %s is now ring %d.", hostID, ring), map[string]any{"host_id": hostID, "ring": ring})
	if s.Bump != nil {
		s.Bump(hostID)
	}
	return nil
}

func (s *Service) SetEnabled(hostID string, on bool) error {
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return fmt.Errorf("osupdates: unknown host %q", hostID)
	}
	if err := s.Store.SetOSEnabled(hostID, on); err != nil {
		return err
	}
	s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates on %s switched %s.", hostID, map[bool]string{true: "ON", false: "OFF"}[on]),
		map[string]any{"host_id": hostID, "enabled": on})
	if s.Bump != nil {
		s.Bump(hostID)
	}
	return nil
}

// LayerLine is one layer of one box on the fleet view.
type LayerLine struct {
	ReleaseID          string
	LastOutcome        string
	LastAt             time.Time
	LastSuccessfulLeg  time.Time
	Pending            int
	NotCovered         int
	NotCoveredFast     int // fast-lane packages no approved release covers (the alarm's input)
	RestartNeeded      int
	RebootNeededSince  time.Time // zero = not needed
	WrapperPassSeconds float64
}

// FleetLine is one box on the fleet view.
type FleetLine struct {
	HostID  string
	Ring    int
	Enabled bool
	Tunnel  string
	Guest   LayerLine
	Host    LayerLine
}

func (s *Service) layerLine(hostID, layer string, ring int) LayerLine {
	var l LayerLine
	rep, _ := s.Store.LatestOSReport(hostID, layer)
	if rep == nil {
		return l
	}
	var r Report
	_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
	l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt
	l.Pending, l.NotCovered, l.RestartNeeded, l.WrapperPassSeconds = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.PassSeconds
	if ok, _ := s.Store.LatestHealthyOSReport(hostID, layer, "applied", "nothing"); ok != nil {
		l.LastSuccessfulLeg = ok.ReceivedAt
	}
	l.NotCoveredFast = len(s.notCoveredFast(layer, ring, r))
	l.RebootNeededSince = s.sinceContinuously(hostID, layer, func(x Report) (bool, bool) {
		if !x.RebootScanned && len(x.Upgraded) == 0 && x.Outcome != "applied" {
			return false, false // a pass that did not look says nothing about reboot-needed: skip it
		}
		return x.RebootNeeded, true
	})
	return l
}

// notCoveredFast: pending fast-lane updates of that layer that the box's approved release does not name. Ring 0
// installs every fast-lane update itself, so it never has any.
func (s *Service) notCoveredFast(layer string, ring int, r Report) []string {
	if ring == 0 {
		return nil
	}
	named := map[string]bool{}
	if rb := s.releaseBlock(layer); rb != nil {
		for _, p := range rb.Packages {
			named[p.Name] = true
		}
	}
	var out []string
	for _, p := range fastPending(layer, r.Pending) {
		if !named[p.Name] {
			out = append(out, p.Name)
		}
	}
	return out
}

// sinceContinuously walks the box's reports of a layer newest-first and returns the time of the OLDEST report in the
// current unbroken run where pred holds (zero when the newest relevant report does not hold). pred's second result
// false = the report does not count either way (skipped).
func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (bool, bool)) time.Time {
	reps, err := s.Store.OSReportsDesc(hostID, layer, 400)
	if err != nil {
		return time.Time{}
	}
	var since time.Time
	for _, rep := range reps {
		var r Report
		if json.Unmarshal([]byte(rep.ReportJSON), &r) != nil {
			continue
		}
		hold, counts := pred(r)
		if !counts {
			continue
		}
		if !hold {
			break
		}
		since = rep.ReceivedAt
	}
	return since
}

// Fleet lists every box.
func (s *Service) Fleet() ([]FleetLine, error) {
	hosts, err := s.Store.ListHosts()
	if err != nil {
		return nil, err
	}
	var out []FleetLine
	for _, h := range hosts {
		st := s.Store.GetOSHostSettings(h.HostID)
		l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled, Tunnel: "unknown"}
		if t, _ := s.Store.RecentCloudflaredStatuses(h.HostID, 1); len(t) == 1 && t[0] != "" {
			l.Tunnel = t[0]
		}
		l.Guest = s.layerLine(h.HostID, LayerGuest, st.Ring)
		l.Host = s.layerLine(h.HostID, LayerHost, st.Ring)
		out = append(out, l)
	}
	return out, nil
}

// FleetJSON is Fleet plus the approval status, for the operator's fleet route.
func (s *Service) FleetJSON() (any, error) {
	lines, err := s.Fleet()
	if err != nil {
		return nil, err
	}
	out := map[string]any{"boxes": lines}
	for _, layer := range Layers {
		if rel, _ := s.Store.LatestOSRelease(layer); rel != nil {
			out["latest_"+layer+"_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy}
		}
	}
	return out, nil
}

// ---------------- the alarms (`11` §8.3) ----------------

// raise sends an alarm when its condition holds and it was not raised in the last week; clears it when it does not
// hold, so the next occurrence is announced again. Returns true when an event was sent.
func (s *Service) raise(key string, holds bool, customerID, typ, sev, msg string, details any) bool {
	last := s.Store.OSAlarmRaised(key)
	if !holds {
		if !last.IsZero() {
			_ = s.Store.SetOSAlarmRaised(key, time.Time{})
		}
		return false
	}
	if !last.IsZero() && s.now().Sub(last) < 7*24*time.Hour {
		return false
	}
	s.event(customerID, typ, sev, msg, details)
	_ = s.Store.SetOSAlarmRaised(key, s.now())
	return true
}

// Alarms evaluates the four OS-update alarms. Called hourly. Returns the event types it sent (for the log/tests).
func (s *Service) Alarms() ([]string, error) {
	now := s.now()
	stale, reboot, stall, nc := dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour),
		dflt(s.Ring0StallAfter, 7*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour)
	hosts, err := s.Store.ListHosts()
	if err != nil {
		return nil, err
	}
	var sent []string
	for _, h := range hosts {
		st := s.Store.GetOSHostSettings(h.HostID)
		// 1. No successful OS leg for `stale` while the switch is ON — the leg runs only after a good night backup,
		//    so a box with a broken backup silently stops updating. Only boxes whose agent CAN run the leg.
		canRun := h.AgentVersion != "" && semver.Valid(h.AgentVersion) && semver.Compare(h.AgentVersion, LegMinAgent) >= 0
		var ref time.Time
		if ok, _ := s.Store.LatestHealthyOSReport(h.HostID, LayerGuest, "applied", "nothing"); ok != nil {
			ref = ok.ReceivedAt
		} else if seen, _ := s.Store.HostAgentVersionsSeen(h.HostID); len(seen) > 0 {
			for v, at := range seen {
				if semver.Valid(v) && semver.Compare(v, LegMinAgent) >= 0 && (ref.IsZero() || at.Before(ref)) {
					ref = at
				}
			}
		}
		holds := st.Enabled && canRun && !ref.IsZero() && now.Sub(ref) >= stale
		if s.raise("stale:"+h.HostID, holds, h.CustomerID, EventStale, "warning",
			fmt.Sprintf("OS updates: %s has not completed an OS update leg for %s (since %s). Likely reason: %s.",
				h.HostID, now.Sub(ref).Round(time.Hour), ref.UTC().Format("2006-01-02 15:04"), s.staleReason(h)),
			map[string]any{"host_id": h.HostID, "since": ref}) {
			sent = append(sent, EventStale)
		}
		// 2. Reboot needed for `reboot` (host layer): a fix is installed but not fully in force.
		since := s.layerLine(h.HostID, LayerHost, st.Ring).RebootNeededSince
		if s.raise("reboot:"+h.HostID, !since.IsZero() && now.Sub(since) >= reboot, h.CustomerID, EventRebootNeeded, "warning",
			fmt.Sprintf("OS updates (for information): %s has needed a reboot since %s — host fixes are installed but not fully in force (PID 1 or lxc-start still run the old libraries). Nothing reboots automatically until the kernel lane exists.",
				h.HostID, since.UTC().Format("2006-01-02")), map[string]any{"host_id": h.HostID, "since": since}) {
			sent = append(sent, EventRebootNeeded)
		}
		// 4. Not-covered fast-lane packages for `nc` (ring 1: hardware ring 0 does not have, `11` edge case 9).
		for _, layer := range Layers {
			since := s.sinceContinuously(h.HostID, layer, func(r Report) (bool, bool) {
				return len(s.notCoveredFast(layer, st.Ring, r)) > 0, true
			})
			var names []string
			if rep, _ := s.Store.LatestOSReport(h.HostID, layer); rep != nil {
				var r Report
				_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
				names = s.notCoveredFast(layer, st.Ring, r)
			}
			if s.raise("notcovered:"+layer+":"+h.HostID, !since.IsZero() && now.Sub(since) >= nc, h.CustomerID, EventNotCovered, "warning",
				fmt.Sprintf("OS updates (%s): %s has had %d fast-lane package(s) no approved release covers since %s: %s. Ring 0 does not run them (other hardware?) — add matching hardware to ring 0, or approve by hand.",
					layer, h.HostID, len(names), since.UTC().Format("2006-01-02"), strings.Join(names, ", ")),
				map[string]any{"host_id": h.HostID, "layer": layer, "packages": names}) {
				sent = append(sent, EventNotCovered)
			}
		}
	}
	// 3. Ring 0 approved nothing for `stall` while ring 0 has pending fast-lane updates: the whole fleet stopped
	//    getting fixes.
	ring0, _ := s.ring0Hosts()
	for _, layer := range Layers {
		var last time.Time
		if rel, _ := s.Store.LatestOSRelease(layer); rel != nil {
			last = rel.ApprovedAt
		} else if first, _ := s.Store.FirstOSReport(layer, ring0); first != nil {
			last = first.ReceivedAt
		}
		pendingAny := 0
		for _, hID := range ring0 {
			if rep, _ := s.Store.LatestOSReport(hID, layer); rep != nil {
				var r Report
				_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
				pendingAny += len(fastPending(layer, r.Pending))
			}
		}
		holds := len(ring0) > 0 && !last.IsZero() && now.Sub(last) >= stall && pendingAny > 0
		if s.raise("ring0stall:"+layer, holds, "", EventRing0Stalled, "error",
			fmt.Sprintf("OS updates (%s): ring 0 has approved nothing since %s while it has %d pending fast-lane update(s). The demo boxes are broken or unhealthy, and the whole fleet has stopped getting fixes.",
				layer, last.UTC().Format("2006-01-02 15:04"), pendingAny), map[string]any{"layer": layer, "pending": pendingAny}) {
			sent = append(sent, EventRing0Stalled)
		}
	}
	return sent, nil
}

// staleReason names the likely reason a box stopped completing the OS leg (for the operator's mail).
func (s *Service) staleReason(h store.Host) string {
	if h.LastReportAt == nil || s.now().Sub(*h.LastReportAt) > 24*time.Hour {
		return "the box is off or not reporting (last host report " + fmtTime(h.LastReportAt) + ")"
	}
	if rep, _ := s.Store.LatestOSReport(h.HostID, LayerGuest); rep != nil && (rep.Outcome == "failed" || rep.Outcome == "refused" || rep.Outcome == "health_failed" || !rep.Healthy) {
		var r Report
		_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
		return fmt.Sprintf("the last OS leg ended %s (%s) on %s", rep.Outcome, strings.TrimSpace(r.HealthReason+" "+string(r.Refused)), rep.ReceivedAt.UTC().Format("2006-01-02"))
	}
	return "no successful whole-guest backup reached the OS leg (it runs only after one) — check the box's night backups"
}

func fmtTime(t *time.Time) string {
	if t == nil {
		return "never"
	}
	return t.UTC().Format("2006-01-02 15:04")
}