// Package osupdates is the hub half of the OS-update lanes (`11-os-updates.md` §5.3, §8 steps 2–4). // // Two LAYERS, each with its own approved set (hub v0.131.0): the customer guest's Debian packages and the Proxmox // host's Debian packages. A version approved for one layer is NOT approved for the other by that fact alone. // // Ring 0 (the demo boxes) installs every pending Debian / Debian-Security fix each night and reports the FULL installed // set per layer (C9). Per layer, the CANDIDATE is every Debian-origin package=version that all ring-0 boxes having it // agree on (the host set leaves out kernel / boot / firmware names — the host's slow lane, `11` C3). A candidate is // APPROVED when, since it was first seen: // - every ring-0 box runs it (its newest report of that layer matches the candidate for every package it has), // - ApproveAfter (default 24 h) has passed with every ring-0 report of that layer healthy, and // - every ring-0 box has completed NightsRequired (default 1) post-backup night runs of that layer. // // The operator can approve at once ("approve now"). Ring 1 gets each layer's newest release in its desired state. // The fleet view and the four alarms (`11` §8 step 4) live here too. Rules pinned by service_test.go and // alarms_test.go. package osupdates import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "log" "regexp" "sort" "strconv" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/semver" "gitea.dooplex.hu/admin/felhom-hub/internal/store" "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" ) // Layers. const ( LayerGuest = "guest" LayerHost = "host" LayerDocker = "docker" // the guest's Docker engine set — slow lane, OPERATOR-approved only (`11` §5.8, hub v0.132.0) ) // Layers lists the layers the hub approves AUTOMATICALLY (the fast lane). var Layers = []string{LayerGuest, LayerHost} // AllLayers adds the Docker engine set (approved only by the operator's button, ApproveDocker). var AllLayers = []string{LayerGuest, LayerHost, LayerDocker} // dockerNames are the six packages of the Docker engine set (the agent wrapper's DOCKER_NAMES). var dockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true, "docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true} // hostSlowRE mirrors the wrapper's HOST_SLOW_RE (felhom-agent configs/felhom-os-apply): kernel, boot and firmware // packages are the host's slow lane and never enter a host release (the wrapper would refuse the whole plan, R14). // Pinned by TestCandidate_HostLeavesOutKernelBootFirmware. var hostSlowRE = regexp.MustCompile(`^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|pve-firmware|firmware-|grub|shim|systemd-boot|intel-microcode|amd64-microcode|efibootmgr)`) // LegMinAgent is the first agent that runs the OS leg; the stale-leg alarm only watches boxes at or above it. const LegMinAgent = "0.140.0" // Event types — operator-only except EventApplied, the household's line (`11` §5.7). const ( EventApplied = "os_update_applied" // info, CUSTOMER: "system security fixes installed" EventFailed = "os_update_failed" // error, operator: the run failed or was refused EventHealthFailed = "os_update_health_failed" // error, operator: not healthy after the run EventReleaseApprove = "os_release_approved" // info, operator EventApprovedNow = "os_release_approved_now" // warning, operator: an operator approved at once EventSettings = "os_update_settings_changed" // info, operator: ring or switch changed EventStale = "os_update_stale" // warning, operator: no successful OS leg for StaleAfter EventRebootNeeded = "os_reboot_needed" // warning, operator: reboot needed for RebootAfter EventRing0Stalled = "os_ring0_stalled" // error, operator: ring 0 approved nothing for Ring0StallAfter EventNotCovered = "os_not_covered" // warning, operator: fast-lane packages no release covers // EventCancelled: a TEST approval was cancelled because the hub started without the TEST override (`11` §5.3.1). EventCancelled = "os_release_cancelled" // warning, operator // EventBundleBehind: a box's root-owned config bundle has differed from the vouched one for BundleBehindAfter (R-840). EventBundleBehind = "os_config_bundle_behind" // warning, operator // EventAgentBehind: a box has run an agent older than the vouched one for AgentBehindAfter (R-530, hub v0.135.0). // Agents update only by a per-box signed job (R-530's ruling), so a box nobody signed for silently stays behind. EventAgentBehind = "agent_behind" // warning, operator ) // Package is one name=version with its origin ("Debian" | "Debian-Security"). type Package struct { Name string `json:"name"` Version string `json:"version"` Origin string `json:"origin"` } // Report is what the agent POSTs after every run of a layer. type Report struct { RunID string `json:"run_id"` Layer string `json:"layer"` // guest | host ("" from agent 0.140.0 = guest) Trigger string `json:"trigger"` Mode string `json:"mode"` Ring int `json:"ring"` ReleaseID string `json:"release_id"` Outcome string `json:"outcome"` // applied | nothing | inventory | refused | failed | health_failed Healthy bool `json:"healthy"` HealthReason string `json:"health_reason,omitempty"` VMID int `json:"vmid"` Upgraded []Package `json:"upgraded,omitempty"` Installed []Package `json:"installed,omitempty"` Pending []PendingPkg `json:"pending,omitempty"` NotCovered []string `json:"not_covered,omitempty"` RestartNeeded []string `json:"restart_needed,omitempty"` DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"` RebootNeeded bool `json:"reboot_needed,omitempty"` RebootScanned bool `json:"reboot_scanned,omitempty"` // agent ≥ 0.141.1: the pass looked (host: every pass) Refused json.RawMessage `json:"refused,omitempty"` PassSeconds float64 `json:"pass_seconds,omitempty"` DockerEngine string `json:"docker_engine,omitempty"` // docker layer (agent v0.142.0) Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed Undo bool `json:"undo,omitempty"` // docker layer: a signed undo // OOMCheck (agent v0.150.0, decision 157): the Docker step's memory-kill check, kept verbatim — oomCheckWaiting // reads it from the stored report. OOMCheck json.RawMessage `json:"oom_check,omitempty"` } // PendingPkg is one update the sources offer. type PendingPkg struct { Name string `json:"name"` From string `json:"from"` To string `json:"to"` Origin []string `json:"origin"` } func isFast(origins []string) bool { if len(origins) == 0 { return false } for _, o := range origins { if o != "Debian" && o != "Debian-Security" { return false } } return true } // fastPending is the pending updates a fast-lane release COULD carry for that layer. func fastPending(layer string, p []PendingPkg) []PendingPkg { var out []PendingPkg for _, x := range p { if isFast(x.Origin) && !(layer == LayerHost && hostSlowRE.MatchString(x.Name)) { out = append(out, x) } } return out } // Block is what a box receives in its desired state (`os_update`). type Block struct { Ring int `json:"ring"` Enabled bool `json:"enabled"` Release *ReleaseBlock `json:"release,omitempty"` HostRelease *ReleaseBlock `json:"host_release,omitempty"` } // ReleaseBlock is a layer's newest approved release, for ring 1. type ReleaseBlock struct { ID string `json:"id"` Snapshot string `json:"snapshot"` // approval time as YYYYMMDDTHHMMSSZ (decision 79: snapshot.debian.org) Packages []Package `json:"packages"` } // DockerNightsEffective is the number of healthy ring-0 nights the Docker engine set actually needs: 0 means // the default 2, negative means none (a TEST override only). The start log prints THIS, not the raw field — // the raw "none" value read "after -1 healthy ring-0 night(s)" (R-855). Pinned by TestDockerNightsEffective. func (s *Service) DockerNightsEffective() int { switch { case s.DockerNights == 0: return 2 case s.DockerNights < 0: return 0 } return s.DockerNights } // Service ties the store, the events and the clock together. type Service struct { Store *store.Store Emit func(customerID, eventType, severity, message, details, source string) ApproveAfter time.Duration NightsRequired int // DockerNights is how many healthy ring-0 night runs of the Docker step an engine set needs before the operator's // button may approve it (`11` §5.8: 2; 0 means 2; negative means none). OS_DOCKER_APPROVE_NIGHTS overrides it for // a TEST only, logged. DockerNights int // Alarm thresholds (`11` §8.3; decided by CC unattended — operator may reverse). Zero = the default. StaleAfter time.Duration // 7 d RebootAfter time.Duration // 14 d Ring0StallAfter time.Duration // 7 d NotCoveredAfter time.Duration // 14 d // BundleBehindAfter: a box's config bundle differs from the vouched one this long → an operator alarm (R-840; // decided by CC unattended — operator may reverse). Zero = 7 d. BundleBehindAfter time.Duration // AgentBehindAfter: a box runs an agent older than the vouched one this long → an operator alarm (R-530; decided // by CC — operator may reverse, env OS_ALARM_AGENT_BEHIND_AFTER). Zero = 7 d. AgentBehindAfter time.Duration Logger *log.Logger Now func() time.Time Bump func(hostID string) // TestOverride names the TEST overrides active at start ("" = none, the ruled waits). Every approval made while it // is set carries the `test` mark; CancelTestReleases cancels them at a start without it (`11` §5.3.1). TestOverride string } func (s *Service) now() time.Time { if s.Now != nil { return s.Now() } return time.Now() } func (s *Service) logf(f string, a ...any) { if s.Logger != nil { s.Logger.Printf(f, a...) } } func dflt(d, def time.Duration) time.Duration { if d <= 0 { return def } return d } func (s *Service) event(customerID, typ, sev, msg string, details any) { dj := "" if details != nil { if b, err := json.Marshal(details); err == nil { dj = string(b) } } if _, err := s.Store.SaveEvent(customerID, typ, sev, msg, dj, "hub"); err != nil { s.logf("[WARN] osupdates: save event %s: %v", typ, err) } if s.Emit != nil { s.Emit(customerID, typ, sev, msg, dj, "hub") } } func layerOf(r Report) string { switch r.Layer { case LayerHost, LayerDocker: return r.Layer } return LayerGuest } // Ingest stores a run and raises its events. The household gets one line per run that installed something. func (s *Service) Ingest(hostID string, r Report) error { h, err := s.Store.GetHost(hostID) if err != nil || h == nil { return fmt.Errorf("osupdates: unknown host %q", hostID) } layer := layerOf(r) r.Layer = layer raw, _ := json.Marshal(r) if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, Layer: layer, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode, Outcome: r.Outcome, Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil { return err } s.logf("[INFO] osupdates: %s reported %s run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d reboot-needed=%v wrapper=%.1fs", hostID, layer, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.RebootNeeded, r.PassSeconds) details := map[string]any{"host_id": hostID, "layer": layer, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome, "upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason} where := "the box" switch layer { case LayerHost: where = "the box's base system" case LayerDocker: where = "the box's app engine (Docker " + r.DockerEngine + ")" } switch r.Outcome { case "applied", "health_failed": s.event(h.CustomerID, EventApplied, "info", fmt.Sprintf("System security fixes installed on %s (%d package(s)).", where, len(r.Upgraded)), details) if !r.Healthy || r.Outcome == "health_failed" { undo := "last night's whole-guest backup is the undo (restore by hand, decision 81)" switch layer { case LayerHost: undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)" case LayerDocker: undo = "sign an os_docker_step with undo for the previous engine set (runbook `os-updates-docker-undo.md`)" } s.event(h.CustomerID, EventHealthFailed, "error", fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.", layer, hostID, len(r.Upgraded), r.HealthReason, undo), details) } case "refused", "failed": s.event(h.CustomerID, EventFailed, "error", fmt.Sprintf("OS update (%s) on %s %s: %s", layer, hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details) } return nil } // candidate derives, for one layer, the version set every ring-0 box agrees on, from each box's newest report. func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, error) { byPkg := map[string]map[string]Package{} // name -> host -> pkg for _, h := range ring0 { rep, err := s.Store.LatestOSReport(h, layer) if err != nil { return nil, err } if rep == nil { return nil, nil // a ring-0 box that never reported this layer: no candidate } var r Report if err := json.Unmarshal([]byte(rep.ReportJSON), &r); err != nil { return nil, err } for _, p := range r.Installed { if layer == LayerDocker { if !dockerNames[p.Name] { continue } } else if p.Origin != "Debian" && p.Origin != "Debian-Security" { continue } if layer == LayerHost && hostSlowRE.MatchString(p.Name) { continue } if byPkg[p.Name] == nil { byPkg[p.Name] = map[string]Package{} } byPkg[p.Name][h] = p } } cand := map[string]Package{} for name, hosts := range byPkg { var v string agree := true var pick Package for _, p := range hosts { if v == "" { v, pick = p.Version, p } else if p.Version != v { agree = false } } if agree { cand[name] = pick } } return cand, nil } func fingerprint(layer string, c map[string]Package) (string, []Package) { var list []Package for _, p := range c { list = append(list, p) } sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name }) h := sha256.New() fmt.Fprintf(h, "layer=%s\n", layer) for _, p := range list { fmt.Fprintf(h, "%s=%s\n", p.Name, p.Version) } return hex.EncodeToString(h.Sum(nil))[:16], list } // ring0Hosts lists the ring-0 boxes that have the switch ON. func (s *Service) ring0Hosts() ([]string, error) { hosts, err := s.Store.ListHosts() if err != nil { return nil, err } var out []string for _, h := range hosts { if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 0 && st.Enabled { out = append(out, h.HostID) } } sort.Strings(out) return out, nil } // Status explains the approval state of one layer (for the log and the fleet page). type Status struct { Layer string Fingerprint string FirstSeen time.Time Packages int Waiting string // why not approved yet ("" = approved or nothing to do) Approved string // the release id when this set is already approved } // Evaluate checks the approval rule of every layer and approves when it holds. Called every minute. func (s *Service) Evaluate() ([]Status, error) { var out []Status for _, layer := range Layers { st, err := s.evaluateLayer(layer) if err != nil { return out, err } out = append(out, st) } // The Docker set approves only by the operator's button, but its "first seen" must still be stamped on the tick // like the other layers: otherwise it was stamped only when someone opened the System page, and the healthy // nights before that view did not count (found 2026-10-05, a flaky TestSystemPage_DockerButtonOnlyWhenReady). // Pinned by TestEvaluate_StampsDockerFirstSeenWithoutAPageView. if _, err := s.DockerStatus(); err != nil { return out, err } return out, nil } func (s *Service) evaluateLayer(layer string) (Status, error) { return s.layerStatus(layer, true) } // LayerStatus is the read-only view of a fast-lane layer's candidate (the System page): it never approves. func (s *Service) LayerStatus(layer string) (Status, error) { return s.layerStatus(layer, false) } func (s *Service) layerStatus(layer string, approve bool) (Status, error) { ring0, err := s.ring0Hosts() if err != nil || len(ring0) == 0 { return Status{Layer: layer, Waiting: "no ring-0 box"}, err } cand, err := s.candidate(layer, ring0) if err != nil || cand == nil { return Status{Layer: layer, Waiting: "a ring-0 box has not reported this layer"}, err } fp, list := fingerprint(layer, cand) pj, _ := json.Marshal(list) first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()) if err != nil { return Status{}, err } st := Status{Layer: layer, Fingerprint: fp, FirstSeen: first, Packages: len(list)} if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp { st.Approved = rel.ID return st, nil // already approved } if age := s.now().Sub(first); age < s.ApproveAfter { st.Waiting = fmt.Sprintf("healthy for %s of %s", age.Round(time.Minute), s.ApproveAfter) return st, nil } for _, h := range ring0 { reps, err := s.Store.OSReportsSince(h, layer, first) if err != nil { return st, err } nights := 0 for _, r := range reps { if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" { st.Waiting = fmt.Sprintf("%s reported %s (healthy=%v) at %s since the set was first seen", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339)) return st, nil } if r.Trigger == "night" { nights++ } } if nights < s.NightsRequired { st.Waiting = fmt.Sprintf("%s has %d of %d night run(s) since the set was first seen", h, nights, s.NightsRequired) return st, nil } } if !approve { st.Waiting = "ready — approves at the next minute's check" return st, nil } return st, s.approve(layer, fp, list, "auto") } // ReleaseInfo is one layer's newest approved release, for the System page. type ReleaseInfo struct { Layer string ID string ApprovedAt time.Time ApprovedBy string Packages int Test bool // a TEST approval still in force: amber on the System page Cancelled string // set on a cancelled one (the page lists the last 7 days') } // Releases lists the newest release of every layer (guest, host, Docker); a layer with none is absent. func (s *Service) Releases() []ReleaseInfo { var out []ReleaseInfo for _, layer := range AllLayers { rel, _ := s.Store.LatestOSRelease(layer) if rel == nil { continue } var list []Package _ = json.Unmarshal([]byte(rel.PackagesJSON), &list) out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list), Test: rel.Test}) } return out } // CancelledReleases lists the approvals cancelled in the last 7 days (the System page says what stopped being served). func (s *Service) CancelledReleases() []ReleaseInfo { rels, _ := s.Store.CancelledOSReleasesSince(s.now().Add(-7 * 24 * time.Hour)) var out []ReleaseInfo for _, r := range rels { var list []Package _ = json.Unmarshal([]byte(r.PackagesJSON), &list) out = append(out, ReleaseInfo{Layer: r.Layer, ID: r.ID, ApprovedAt: r.ApprovedAt, ApprovedBy: r.ApprovedBy, Packages: len(list), Test: r.Test, Cancelled: r.CancelledAt}) } return out } // Candidates is the read-only candidate status of every layer (the Docker one by its own rule). func (s *Service) Candidates() []Status { var out []Status for _, layer := range Layers { st, _ := s.LayerStatus(layer) out = append(out, st) } d, _ := s.DockerStatus() return append(out, d) } // BundleThreshold is the config-bundle alarm's wait (the System page turns the cell red at it). func (s *Service) BundleThreshold() time.Duration { return dflt(s.BundleBehindAfter, 7*24*time.Hour) } // AgentThreshold is the agent-behind alarm's wait (R-530; the System page turns the cell red at it). func (s *Service) AgentThreshold() time.Duration { return dflt(s.AgentBehindAfter, 7*24*time.Hour) } // Thresholds are the alarm numbers the System page colours by (the same values the alarms use). func (s *Service) Thresholds() (stale, reboot, notCovered time.Duration) { return dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour) } // ApproveNow approves every layer's current candidate at once (operator, urgent fix). Returns the release ids. func (s *Service) ApproveNow() (string, error) { ring0, err := s.ring0Hosts() if err != nil || len(ring0) == 0 { return "", fmt.Errorf("osupdates: no ring-0 box") } var ids []string for _, layer := range Layers { cand, err := s.candidate(layer, ring0) if err != nil || cand == nil { continue } fp, list := fingerprint(layer, cand) pj, _ := json.Marshal(list) if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil { return "", err } if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp { ids = append(ids, rel.ID) continue } if err := s.approve(layer, fp, list, "operator"); err != nil { return "", err } rel, _ := s.Store.LatestOSRelease(layer) s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s (%s) at once (%d packages), without the wait.", rel.ID, layer, len(list)), map[string]any{"release_id": rel.ID, "layer": layer, "packages": len(list)}) ids = append(ids, rel.ID) } if len(ids) == 0 { return "", fmt.Errorf("osupdates: a ring-0 box has not reported yet") } return strings.Join(ids, ","), nil } // DockerStatus is the Docker engine set ring 0 runs now and whether the operator's button may approve it: every ring-0 // box has run it in DockerNights healthy night Docker steps since it was first seen, and none unhealthy. func (s *Service) DockerStatus() (Status, error) { st := Status{Layer: LayerDocker} ring0, err := s.ring0Hosts() if err != nil || len(ring0) == 0 { st.Waiting = "no ring-0 box" return st, err } cand, err := s.candidate(LayerDocker, ring0) if err != nil || len(cand) == 0 { st.Waiting = "a ring-0 box has not reported a Docker step" return st, err } fp, list := fingerprint(LayerDocker, cand) pj, _ := json.Marshal(list) first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()) if err != nil { return st, err } st.Fingerprint, st.FirstSeen, st.Packages = fp, first, len(list) if rel, _ := s.Store.LatestOSRelease(LayerDocker); rel != nil && rel.Fingerprint == fp { st.Approved, st.Waiting = rel.ID, "already approved" return st, nil } need := s.DockerNightsEffective() for _, h := range ring0 { reps, err := s.Store.OSReportsSince(h, LayerDocker, first) if err != nil { return st, err } nights := 0 for _, r := range reps { if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" { st.Waiting = fmt.Sprintf("%s reported a Docker step %s (healthy=%v) at %s", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339)) return st, nil } if r.Trigger == "night" { nights++ } } if nights < need { st.Waiting = fmt.Sprintf("%s has %d of %d healthy night Docker step(s) with this set", h, nights, need) return st, nil } if why := oomCheckWaiting(h, reps); why != "" { st.Waiting = why return st, nil } } return st, nil } // oomCheckWaiting is `09` §3 decision 157 (R-528): a Docker engine set is approved only when every ring-0 box showed, // after its Docker step with this set, that the engine reports a memory kill correctly — the wrapper's `oom_check` // (agent v0.150.0: a throwaway container under a 64 MB cap killed for memory; pass = `OOMKilled=true` AND the `oom` // event). A failed or errored check on ANY report of the set blocks; no passing check at all blocks („missing"). // Returns "" when the box is clear. Pinned by TestDockerApproval_*. func oomCheckWaiting(host string, reps []store.OSReport) string { passed := false for _, r := range reps { var body struct { OOMCheck *struct { Result string `json:"result"` Detail string `json:"detail"` } `json:"oom_check"` } if json.Unmarshal([]byte(r.ReportJSON), &body) != nil || body.OOMCheck == nil { continue } switch body.OOMCheck.Result { case "pass": passed = true default: return fmt.Sprintf("%s: the Docker step's memory-kill check did not pass (%s: %s) at %s — a set whose engine "+ "may miss a memory kill is not approved (decision 157)", host, body.OOMCheck.Result, body.OOMCheck.Detail, r.ReceivedAt.UTC().Format(time.RFC3339)) } } if !passed { return fmt.Sprintf("%s has not reported the Docker step's memory-kill check with this set (agent v0.150.0 or newer runs it; decision 157)", host) } return "" } // ApproveDocker is the operator's button: it approves the Docker engine set ring 0 runs, only when DockerStatus allows. // A ring-1 box then takes it only through a signed operator job (`11` §5.8) — approval alone installs nothing. func (s *Service) ApproveDocker() (string, error) { st, err := s.DockerStatus() if err != nil { return "", err } if st.Waiting != "" { return "", fmt.Errorf("osupdates: the Docker set cannot be approved yet: %s", st.Waiting) } ring0, _ := s.ring0Hosts() cand, err := s.candidate(LayerDocker, ring0) if err != nil { return "", err } fp, list := fingerprint(LayerDocker, cand) if err := s.approve(LayerDocker, fp, list, "operator"); err != nil { return "", err } rel, _ := s.Store.LatestOSRelease(LayerDocker) return rel.ID, nil } func (s *Service) approve(layer, fp string, list []Package, by string) error { at := s.now().UTC().Truncate(time.Second) id := "os-" + layer + "-" + at.Format("20060102-150405") pj, _ := json.Marshal(list) test := s.TestOverride != "" if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Layer: layer, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj), Test: test}); err != nil { return err } mark := "" if test { mark = " — TEST approval (" + s.TestOverride + "); cancelled when the hub starts without the override" } s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)%s", id, layer, by, len(list), fp, mark) s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages)%s.", id, layer, by, len(list), mark), map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp, "test": test}) if s.Bump != nil && layer != LayerDocker { // a Docker set reaches ring 1 only by a signed job, not the desired state hosts, _ := s.Store.ListHosts() for _, h := range hosts { if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled { s.Bump(h.HostID) } } } return nil } // CancelTestReleases runs at every hub start. Without a TEST override it cancels every test-marked approval that no // real (non-test) approval has superseded: no ring-1 box installs it from then on; what boxes already installed stays. // Each cancellation is an operator event; ring-1 boxes are bumped so their next plan has no cancelled release. With the // override still active it does nothing (the test is still running). Returns the cancelled ids. func (s *Service) CancelTestReleases() ([]string, error) { if s.TestOverride != "" { s.logf("[INFO] osupdates: TEST override active (%s) — test approvals stay in force", s.TestOverride) return nil, nil } var ids []string for _, layer := range AllLayers { rels, err := s.Store.UnsupersededTestReleases(layer) if err != nil { return ids, err } for _, r := range rels { reason := "approved under a TEST override; the hub started without it" if err := s.Store.CancelOSRelease(r.ID, reason, s.now()); err != nil { return ids, err } ids = append(ids, r.ID) s.logf("[WARN] osupdates: TEST approval %s (%s, approved %s by %s) CANCELLED — no ring-1 box installs it from now on", r.ID, layer, r.ApprovedAt.UTC().Format(time.RFC3339), r.ApprovedBy) s.event("", EventCancelled, "warning", fmt.Sprintf("OS release %s (%s) was a TEST approval and is cancelled: "+ "no further box installs it. Boxes that already installed it keep it.", r.ID, layer), map[string]any{"release_id": r.ID, "layer": layer, "approved_at": r.ApprovedAt.UTC().Format(time.RFC3339), "approved_by": r.ApprovedBy}) } } if len(ids) > 0 && s.Bump != nil { hosts, _ := s.Store.ListHosts() for _, h := range hosts { if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled { s.Bump(h.HostID) } } } return ids, nil } func (s *Service) releaseBlock(layer string) *ReleaseBlock { rel, err := s.Store.LatestOSRelease(layer) if err != nil || rel == nil { return nil } var list []Package if json.Unmarshal([]byte(rel.PackagesJSON), &list) != nil { return nil } return &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list} } // DesiredBlock is the `os_update` block a box receives. func (s *Service) DesiredBlock(hostID string) Block { st := s.Store.GetOSHostSettings(hostID) b := Block{Ring: st.Ring, Enabled: st.Enabled} if st.Ring == 1 { b.Release, b.HostRelease = s.releaseBlock(LayerGuest), s.releaseBlock(LayerHost) } return b } // SetRing / SetEnabled are operator acts; each bumps the box's desired generation and is an operator event. func (s *Service) SetRing(hostID string, ring int) error { if ring != 0 && ring != 1 { return fmt.Errorf("osupdates: ring must be 0 or 1") } h, err := s.Store.GetHost(hostID) if err != nil || h == nil { return fmt.Errorf("osupdates: unknown host %q", hostID) } if err := s.Store.SetOSRing(hostID, ring); err != nil { return err } s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates: %s is now ring %d.", hostID, ring), map[string]any{"host_id": hostID, "ring": ring}) if s.Bump != nil { s.Bump(hostID) } return nil } func (s *Service) SetEnabled(hostID string, on bool) error { h, err := s.Store.GetHost(hostID) if err != nil || h == nil { return fmt.Errorf("osupdates: unknown host %q", hostID) } if err := s.Store.SetOSEnabled(hostID, on); err != nil { return err } s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates on %s switched %s.", hostID, map[bool]string{true: "ON", false: "OFF"}[on]), map[string]any{"host_id": hostID, "enabled": on}) if s.Bump != nil { s.Bump(hostID) } return nil } // LayerLine is one layer of one box on the fleet view. type LayerLine struct { ReleaseID string LastOutcome string LastAt time.Time LastSuccessfulLeg time.Time Pending int NotCovered int NotCoveredFast int // fast-lane packages no approved release covers (the alarm's input) RestartNeeded int RebootNeededSince time.Time // zero = not needed WrapperPassSeconds float64 } // FleetLine is one box on the fleet view. type FleetLine struct { HostID string Ring int Enabled bool Tunnel string Guest LayerLine Host LayerLine Docker LayerLine } func (s *Service) layerLine(hostID, layer string, ring int) LayerLine { var l LayerLine rep, _ := s.Store.LatestOSReport(hostID, layer) if rep == nil { return l } var r Report _ = json.Unmarshal([]byte(rep.ReportJSON), &r) l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt l.Pending, l.NotCovered, l.RestartNeeded, l.WrapperPassSeconds = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded), r.PassSeconds if ok, _ := s.Store.LatestHealthyOSReport(hostID, layer, "applied", "nothing"); ok != nil { l.LastSuccessfulLeg = ok.ReceivedAt } l.NotCoveredFast = len(s.notCoveredFast(layer, ring, r)) l.RebootNeededSince = s.sinceContinuously(hostID, layer, func(x Report) (bool, bool) { if !x.RebootScanned && len(x.Upgraded) == 0 && x.Outcome != "applied" { return false, false // a pass that did not look says nothing about reboot-needed: skip it } return x.RebootNeeded, true }) return l } // notCoveredFast: pending fast-lane updates of that layer that the box's approved release does not name. Ring 0 // installs every fast-lane update itself, so it never has any. func (s *Service) notCoveredFast(layer string, ring int, r Report) []string { if ring == 0 { return nil } named := map[string]bool{} if rb := s.releaseBlock(layer); rb != nil { for _, p := range rb.Packages { named[p.Name] = true } } var out []string for _, p := range fastPending(layer, r.Pending) { if !named[p.Name] { out = append(out, p.Name) } } return out } // sinceContinuously walks the box's reports of a layer newest-first and returns the time of the OLDEST report in the // current unbroken run where pred holds (zero when the newest relevant report does not hold). pred's second result // false = the report does not count either way (skipped). func (s *Service) sinceContinuously(hostID, layer string, pred func(Report) (bool, bool)) time.Time { reps, err := s.Store.OSReportsDesc(hostID, layer, 400) if err != nil { return time.Time{} } var since time.Time for _, rep := range reps { var r Report if json.Unmarshal([]byte(rep.ReportJSON), &r) != nil { continue } hold, counts := pred(r) if !counts { continue } if !hold { break } since = rep.ReceivedAt } return since } // Fleet lists every box. func (s *Service) Fleet() ([]FleetLine, error) { hosts, err := s.Store.ListHosts() if err != nil { return nil, err } var out []FleetLine for _, h := range hosts { st := s.Store.GetOSHostSettings(h.HostID) l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled, Tunnel: "unknown"} if t, _ := s.Store.RecentCloudflaredStatuses(h.HostID, 1); len(t) == 1 && t[0] != "" { l.Tunnel = t[0] } l.Guest = s.layerLine(h.HostID, LayerGuest, st.Ring) l.Host = s.layerLine(h.HostID, LayerHost, st.Ring) l.Docker = s.layerLine(h.HostID, LayerDocker, st.Ring) out = append(out, l) } return out, nil } // FleetJSON is Fleet plus the approval status, for the operator's fleet route. func (s *Service) FleetJSON() (any, error) { lines, err := s.Fleet() if err != nil { return nil, err } out := map[string]any{"boxes": lines} for _, layer := range AllLayers { if rel, _ := s.Store.LatestOSRelease(layer); rel != nil { out["latest_"+layer+"_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy} } } return out, nil } // ---------------- the alarms (`11` §8.3) ---------------- // raise sends an alarm when its condition holds and it was not raised in the last week; clears it when it does not // hold, so the next occurrence is announced again. Returns true when an event was sent. func (s *Service) raise(key string, holds bool, customerID, typ, sev, msg string, details any) bool { last := s.Store.OSAlarmRaised(key) if !holds { if !last.IsZero() { _ = s.Store.SetOSAlarmRaised(key, time.Time{}) } return false } if !last.IsZero() && s.now().Sub(last) < 7*24*time.Hour { return false } s.event(customerID, typ, sev, msg, details) _ = s.Store.SetOSAlarmRaised(key, s.now()) return true } // Alarms evaluates the four OS-update alarms. Called hourly. Returns the event types it sent (for the log/tests). func (s *Service) Alarms() ([]string, error) { now := s.now() stale, reboot, stall, nc := dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), dflt(s.Ring0StallAfter, 7*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour) hosts, err := s.Store.ListHosts() if err != nil { return nil, err } var sent []string for _, h := range hosts { st := s.Store.GetOSHostSettings(h.HostID) // 1. No successful OS leg for `stale` while the switch is ON — the leg runs only after a good night backup, // so a box with a broken backup silently stops updating. Only boxes whose agent CAN run the leg. canRun := h.AgentVersion != "" && semver.Valid(h.AgentVersion) && semver.Compare(h.AgentVersion, LegMinAgent) >= 0 var ref time.Time if ok, _ := s.Store.LatestHealthyOSReport(h.HostID, LayerGuest, "applied", "nothing"); ok != nil { ref = ok.ReceivedAt } else if seen, _ := s.Store.HostAgentVersionsSeen(h.HostID); len(seen) > 0 { for v, at := range seen { if semver.Valid(v) && semver.Compare(v, LegMinAgent) >= 0 && (ref.IsZero() || at.Before(ref)) { ref = at } } } holds := st.Enabled && canRun && !ref.IsZero() && now.Sub(ref) >= stale if s.raise("stale:"+h.HostID, holds, h.CustomerID, EventStale, "warning", fmt.Sprintf("OS updates: %s has not completed an OS update leg for %s (since %s). Likely reason: %s.", h.HostID, now.Sub(ref).Round(time.Hour), ref.UTC().Format("2006-01-02 15:04"), s.staleReason(h)), map[string]any{"host_id": h.HostID, "since": ref}) { sent = append(sent, EventStale) } // 2. Reboot needed for `reboot` (host layer): a fix is installed but not fully in force. since := s.layerLine(h.HostID, LayerHost, st.Ring).RebootNeededSince if s.raise("reboot:"+h.HostID, !since.IsZero() && now.Sub(since) >= reboot, h.CustomerID, EventRebootNeeded, "warning", fmt.Sprintf("OS updates (for information): %s has needed a reboot since %s — host fixes are installed but not fully in force (PID 1 or lxc-start still run the old libraries). Nothing reboots automatically until the kernel lane exists.", h.HostID, since.UTC().Format("2006-01-02")), map[string]any{"host_id": h.HostID, "since": since}) { sent = append(sent, EventRebootNeeded) } // 4. Not-covered fast-lane packages for `nc` (ring 1: hardware ring 0 does not have, `11` edge case 9). for _, layer := range Layers { since := s.sinceContinuously(h.HostID, layer, func(r Report) (bool, bool) { return len(s.notCoveredFast(layer, st.Ring, r)) > 0, true }) var names []string if rep, _ := s.Store.LatestOSReport(h.HostID, layer); rep != nil { var r Report _ = json.Unmarshal([]byte(rep.ReportJSON), &r) names = s.notCoveredFast(layer, st.Ring, r) } if s.raise("notcovered:"+layer+":"+h.HostID, !since.IsZero() && now.Sub(since) >= nc, h.CustomerID, EventNotCovered, "warning", fmt.Sprintf("OS updates (%s): %s has had %d fast-lane package(s) no approved release covers since %s: %s. Ring 0 does not run them (other hardware?) — add matching hardware to ring 0, or approve by hand.", layer, h.HostID, len(names), since.UTC().Format("2006-01-02"), strings.Join(names, ", ")), map[string]any{"host_id": h.HostID, "layer": layer, "packages": names}) { sent = append(sent, EventNotCovered) } } } // 5. R-840: the box's ROOT-OWNED config bundle (sudoers, wrappers, units) differs from the vouched agent's for // BundleBehindAfter. "none" (no bundle ever reached the box) counts as behind; "unknown" (the box could not say) // counts as nothing — never a guess. Only when the vouched agent carries a bundle at all. man := s.Store.GetArtifactManifest() for _, h := range hosts { if man.BundleSHA256 == "" { break } rj, _ := s.Store.GetLatestHostReportJSON(h.CustomerID) sys := sysfacts.Parse(rj) if !sys.Present || sys.Bundle.Version == sysfacts.Unknown { continue } behind := sys.Bundle.BundleSHA256 != man.BundleSHA256 since := s.Store.BundleBehindSince(h.HostID) switch { case !behind && !since.IsZero(): _ = s.Store.SetBundleBehindSince(h.HostID, time.Time{}) since = time.Time{} case behind && since.IsZero(): since = now _ = s.Store.SetBundleBehindSince(h.HostID, since) } after := dflt(s.BundleBehindAfter, 7*24*time.Hour) if s.raise("bundle:"+h.HostID, behind && now.Sub(since) >= after, h.CustomerID, EventBundleBehind, "warning", fmt.Sprintf("Root files: %s still runs config bundle %s; the vouched agent %s carries a newer one (since %s). "+ "Send it with a signed agent_config_update (`11` §5.4.2).", h.HostID, sys.Bundle.Version, man.AgentVersion, since.UTC().Format("2006-01-02")), map[string]any{"host_id": h.HostID, "box_bundle": sys.Bundle.Version, "vouched_agent": man.AgentVersion, "since": since}) { sent = append(sent, EventBundleBehind) } } // 6. R-530 (v0.135.0): a box runs an agent OLDER than the vouched one for AgentBehindAfter. Agents update only by a // per-box signed job, so a box nobody signed for stays behind silently. An unreadable version (empty, not // semver) is never a fact; nothing vouched → nothing is behind. for _, h := range hosts { if !semver.Valid(man.AgentVersion) { break } if !semver.Valid(h.AgentVersion) { continue } behind := semver.Compare(h.AgentVersion, man.AgentVersion) < 0 since := s.Store.AgentBehindSince(h.HostID) switch { case !behind && !since.IsZero(): _ = s.Store.SetAgentBehindSince(h.HostID, time.Time{}) since = time.Time{} case behind && since.IsZero(): since = now _ = s.Store.SetAgentBehindSince(h.HostID, since) } if s.raise("agent:"+h.HostID, behind && now.Sub(since) >= s.AgentThreshold(), h.CustomerID, EventAgentBehind, "warning", fmt.Sprintf("Agent: %s still runs agent %s; the vouched agent is %s (%s, behind since %s; last report %s). "+ "Sign an agent_update for this box (felhom-opsign, `04` §3.1).", h.HostID, h.AgentVersion, man.AgentVersion, ReleasesBehind(h.AgentVersion, man.AgentVersion), since.UTC().Format("2006-01-02"), fmtTime(h.LastReportAt)), map[string]any{"host_id": h.HostID, "box_agent": h.AgentVersion, "vouched_agent": man.AgentVersion, "since": since}) { sent = append(sent, EventAgentBehind) } } // 3. Ring 0 approved nothing for `stall` while ring 0 has pending fast-lane updates: the whole fleet stopped // getting fixes. ring0, _ := s.ring0Hosts() for _, layer := range Layers { var last time.Time if rel, _ := s.Store.LatestOSRelease(layer); rel != nil { last = rel.ApprovedAt } else if first, _ := s.Store.FirstOSReport(layer, ring0); first != nil { last = first.ReceivedAt } pendingAny := 0 for _, hID := range ring0 { if rep, _ := s.Store.LatestOSReport(hID, layer); rep != nil { var r Report _ = json.Unmarshal([]byte(rep.ReportJSON), &r) pendingAny += len(fastPending(layer, r.Pending)) } } holds := len(ring0) > 0 && !last.IsZero() && now.Sub(last) >= stall && pendingAny > 0 if s.raise("ring0stall:"+layer, holds, "", EventRing0Stalled, "error", fmt.Sprintf("OS updates (%s): ring 0 has approved nothing since %s while it has %d pending fast-lane update(s). The demo boxes are broken or unhealthy, and the whole fleet has stopped getting fixes.", layer, last.UTC().Format("2006-01-02 15:04"), pendingAny), map[string]any{"layer": layer, "pending": pendingAny}) { sent = append(sent, EventRing0Stalled) } } return sent, nil } // staleReason names the likely reason a box stopped completing the OS leg (for the operator's mail). func (s *Service) staleReason(h store.Host) string { if h.LastReportAt == nil || s.now().Sub(*h.LastReportAt) > 24*time.Hour { return "the box is off or not reporting (last host report " + fmtTime(h.LastReportAt) + ")" } if rep, _ := s.Store.LatestOSReport(h.HostID, LayerGuest); rep != nil && (rep.Outcome == "failed" || rep.Outcome == "refused" || rep.Outcome == "health_failed" || !rep.Healthy) { var r Report _ = json.Unmarshal([]byte(rep.ReportJSON), &r) return fmt.Sprintf("the last OS leg ended %s (%s) on %s", rep.Outcome, strings.TrimSpace(r.HealthReason+" "+string(r.Refused)), rep.ReceivedAt.UTC().Format("2006-01-02")) } return "no successful whole-guest backup reached the OS leg (it runs only after one) — check the box's night backups" } func fmtTime(t *time.Time) string { if t == nil { return "never" } return t.UTC().Format("2006-01-02 15:04") } // ReleasesBehind says how far version a is behind b, for the operator: "3 minor releases behind", // "2 patch releases behind", "a major release behind". "" when a is not behind b or either is not semver. func ReleasesBehind(a, b string) string { if !semver.Valid(a) || !semver.Valid(b) || semver.Compare(a, b) >= 0 { return "" } pa, pb := semverParts(a), semverParts(b) switch { case pa[0] != pb[0]: return "a major release behind" case pa[1] != pb[1]: n := pb[1] - pa[1] if n == 1 { return "1 minor release behind" } return fmt.Sprintf("%d minor releases behind", n) } n := pb[2] - pa[2] if n == 1 { return "1 patch release behind" } return fmt.Sprintf("%d patch releases behind", n) } func semverParts(v string) [3]int { var p [3]int v = strings.TrimPrefix(v, "v") if i := strings.IndexAny(v, "-+"); i >= 0 { v = v[:i] } for i, part := range strings.SplitN(v, ".", 3) { if i < 3 { p[i], _ = strconv.Atoi(part) } } return p }