// Package osupdates is the hub half of the guest fast lane (`11-os-updates.md` §5.3, §8 step 2; hub v0.130.0). // // Ring 0 (the demo boxes) installs every pending Debian / Debian-Security fix each night and reports the FULL installed // set (C9). The hub derives the CANDIDATE: every Debian-origin package=version that all ring-0 boxes having that // package agree on. A candidate is APPROVED when, since it was first seen: // - every ring-0 box runs it (its newest report matches the candidate for every package it has), // - ApproveAfter (default 24 h) has passed with every ring-0 report healthy, and // - every ring-0 box has completed NightsRequired (default 1) post-backup night runs. // // The operator can approve at once ("approve now", an urgent fix). Ring 1 then gets the release in its desired state // and installs exactly those versions. Rules pinned by service_test.go. package osupdates import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "log" "sort" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // Event types — operator-only except EventApplied, the household's line (`11` §5.7). const ( EventApplied = "os_update_applied" // info, CUSTOMER: "system security fixes installed" EventFailed = "os_update_failed" // error, operator: the run failed or was refused EventHealthFailed = "os_update_health_failed" // error, operator: the guest was not healthy after the run EventReleaseApprove = "os_release_approved" // info, operator EventApprovedNow = "os_release_approved_now" // warning, operator: an operator approved at once EventSettings = "os_update_settings_changed" // info, operator: ring or switch changed ) // Package is one name=version with its origin ("Debian" | "Debian-Security"). type Package struct { Name string `json:"name"` Version string `json:"version"` Origin string `json:"origin"` } // Report is what the agent POSTs after every run (the wrapper's report plus the leg's verdict). type Report struct { RunID string `json:"run_id"` Trigger string `json:"trigger"` // night | debug Mode string `json:"mode"` // apply | inventory Ring int `json:"ring"` ReleaseID string `json:"release_id"` Outcome string `json:"outcome"` // applied | nothing | inventory | refused | failed | health_failed Healthy bool `json:"healthy"` HealthReason string `json:"health_reason,omitempty"` VMID int `json:"vmid"` Upgraded []Package `json:"upgraded,omitempty"` Installed []Package `json:"installed,omitempty"` Pending []PendingPkg `json:"pending,omitempty"` NotCovered []string `json:"not_covered,omitempty"` RestartNeeded []string `json:"restart_needed,omitempty"` DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"` RebootNeeded bool `json:"reboot_needed,omitempty"` Refused json.RawMessage `json:"refused,omitempty"` Log []string `json:"log,omitempty"` } // PendingPkg is one update the guest's sources offer. type PendingPkg struct { Name string `json:"name"` From string `json:"from"` To string `json:"to"` Origin []string `json:"origin"` } // Block is what a box receives in its desired state (`os_update`). type Block struct { Ring int `json:"ring"` Enabled bool `json:"enabled"` Release *ReleaseBlock `json:"release,omitempty"` } // ReleaseBlock is the newest approved release, for ring 1. type ReleaseBlock struct { ID string `json:"id"` Snapshot string `json:"snapshot"` // approval time as YYYYMMDDTHHMMSSZ (decision 79: snapshot.debian.org) Packages []Package `json:"packages"` } // Service ties the store, the events and the clock together. type Service struct { Store *store.Store Emit func(customerID, eventType, severity, message, details, source string) // dispatcher; nil in tests ApproveAfter time.Duration NightsRequired int Logger *log.Logger Now func() time.Time Bump func(hostID string) // bump a host's desired generation (store.BumpHostDesired); nil in tests } func (s *Service) now() time.Time { if s.Now != nil { return s.Now() } return time.Now() } func (s *Service) logf(f string, a ...any) { if s.Logger != nil { s.Logger.Printf(f, a...) } } func (s *Service) event(customerID, typ, sev, msg string, details any) { dj := "" if details != nil { if b, err := json.Marshal(details); err == nil { dj = string(b) } } if _, err := s.Store.SaveEvent(customerID, typ, sev, msg, dj, "hub"); err != nil { s.logf("[WARN] osupdates: save event %s: %v", typ, err) } if s.Emit != nil { s.Emit(customerID, typ, sev, msg, dj, "hub") } } // Ingest stores a run and raises its events. The household gets one line per run that installed something. func (s *Service) Ingest(hostID string, r Report) error { h, err := s.Store.GetHost(hostID) if err != nil || h == nil { return fmt.Errorf("osupdates: unknown host %q", hostID) } raw, _ := json.Marshal(r) if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode, Outcome: r.Outcome, Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil { return err } s.logf("[INFO] osupdates: %s reported run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d", hostID, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded)) details := map[string]any{"host_id": hostID, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome, "upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason} switch r.Outcome { case "applied", "health_failed": s.event(h.CustomerID, EventApplied, "info", fmt.Sprintf("System security fixes installed (%d package(s)).", len(r.Upgraded)), details) if !r.Healthy || r.Outcome == "health_failed" { s.event(h.CustomerID, EventHealthFailed, "error", fmt.Sprintf("OS update on %s: the guest was NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically (no guest snapshot is possible, R-837); last night's whole-guest backup is the undo.", hostID, len(r.Upgraded), r.HealthReason), details) } case "refused", "failed": s.event(h.CustomerID, EventFailed, "error", fmt.Sprintf("OS update on %s %s: %s", hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details) } return nil } // candidate derives the version set every ring-0 box agrees on, from each box's newest report. func (s *Service) candidate(ring0 []string) (map[string]Package, map[string]*store.OSReport, error) { latest := map[string]*store.OSReport{} byPkg := map[string]map[string]Package{} // name -> host -> pkg for _, h := range ring0 { rep, err := s.Store.LatestOSReport(h) if err != nil { return nil, nil, err } if rep == nil { return nil, nil, nil // a ring-0 box that never reported: no candidate } latest[h] = rep var r Report if err := json.Unmarshal([]byte(rep.ReportJSON), &r); err != nil { return nil, nil, err } for _, p := range r.Installed { if p.Origin != "Debian" && p.Origin != "Debian-Security" { continue } if byPkg[p.Name] == nil { byPkg[p.Name] = map[string]Package{} } byPkg[p.Name][h] = p } } cand := map[string]Package{} for name, hosts := range byPkg { var v string agree := true var pick Package for _, p := range hosts { if v == "" { v, pick = p.Version, p } else if p.Version != v { agree = false } } if agree { cand[name] = pick } } return cand, latest, nil } func fingerprint(c map[string]Package) (string, []Package) { var list []Package for _, p := range c { list = append(list, p) } sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name }) h := sha256.New() for _, p := range list { fmt.Fprintf(h, "%s=%s\n", p.Name, p.Version) } return hex.EncodeToString(h.Sum(nil))[:16], list } // ring0Hosts lists the ring-0 boxes that have the switch ON. func (s *Service) ring0Hosts() ([]string, error) { hosts, err := s.Store.ListHosts() if err != nil { return nil, err } var out []string for _, h := range hosts { if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 0 && st.Enabled { out = append(out, h.HostID) } } sort.Strings(out) return out, nil } // Status explains the approval state (for the log and the fleet page). type Status struct { Fingerprint string FirstSeen time.Time Packages int Waiting string // why not approved yet ("" = approved or nothing to do) } // Evaluate checks the approval rule and approves when it holds. Called every minute. func (s *Service) Evaluate() (Status, error) { ring0, err := s.ring0Hosts() if err != nil || len(ring0) == 0 { return Status{Waiting: "no ring-0 box"}, err } cand, _, err := s.candidate(ring0) if err != nil || cand == nil { return Status{Waiting: "a ring-0 box has not reported"}, err } fp, list := fingerprint(cand) pj, _ := json.Marshal(list) first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()) if err != nil { return Status{}, err } st := Status{Fingerprint: fp, FirstSeen: first, Packages: len(list)} if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp { st.Waiting = "" return st, nil // already approved } if age := s.now().Sub(first); age < s.ApproveAfter { st.Waiting = fmt.Sprintf("healthy for %s of %s", age.Round(time.Minute), s.ApproveAfter) return st, nil } for _, h := range ring0 { reps, err := s.Store.OSReportsSince(h, first) if err != nil { return st, err } nights := 0 for _, r := range reps { if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" { st.Waiting = fmt.Sprintf("%s reported %s (healthy=%v) at %s since the set was first seen", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339)) return st, nil } if r.Trigger == "night" { nights++ } } if nights < s.NightsRequired { st.Waiting = fmt.Sprintf("%s has %d of %d night run(s) since the set was first seen", h, nights, s.NightsRequired) return st, nil } } if err := s.approve(fp, list, "auto"); err != nil { return st, err } return st, nil } // ApproveNow approves the current candidate at once (operator, urgent fix). func (s *Service) ApproveNow() (string, error) { ring0, err := s.ring0Hosts() if err != nil || len(ring0) == 0 { return "", fmt.Errorf("osupdates: no ring-0 box") } cand, _, err := s.candidate(ring0) if err != nil || cand == nil { return "", fmt.Errorf("osupdates: a ring-0 box has not reported yet") } fp, list := fingerprint(cand) pj, _ := json.Marshal(list) if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil { return "", err } if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp { return rel.ID, nil } if err := s.approve(fp, list, "operator"); err != nil { return "", err } rel, _ := s.Store.LatestOSRelease() s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s at once (%d packages), without the wait.", rel.ID, len(list)), map[string]any{"release_id": rel.ID, "packages": len(list)}) return rel.ID, nil } func (s *Service) approve(fp string, list []Package, by string) error { at := s.now().UTC().Truncate(time.Second) id := "os-" + at.Format("20060102-150405") pj, _ := json.Marshal(list) if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil { return err } s.logf("[WARN] osupdates: OS release %s APPROVED by %s (%d packages, fingerprint %s)", id, by, len(list), fp) s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s approved (%s, %d packages).", id, by, len(list)), map[string]any{"release_id": id, "approved_by": by, "packages": len(list), "fingerprint": fp}) if s.Bump != nil { hosts, _ := s.Store.ListHosts() for _, h := range hosts { if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled { s.Bump(h.HostID) } } } return nil } // DesiredBlock is the `os_update` block a box receives. func (s *Service) DesiredBlock(hostID string) Block { st := s.Store.GetOSHostSettings(hostID) b := Block{Ring: st.Ring, Enabled: st.Enabled} if st.Ring == 1 { if rel, err := s.Store.LatestOSRelease(); err == nil && rel != nil { var list []Package if json.Unmarshal([]byte(rel.PackagesJSON), &list) == nil { b.Release = &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list} } } } return b } // SetRing / SetEnabled are operator acts; each bumps the box's desired generation and is an operator event. func (s *Service) SetRing(hostID string, ring int) error { if ring != 0 && ring != 1 { return fmt.Errorf("osupdates: ring must be 0 or 1") } h, err := s.Store.GetHost(hostID) if err != nil || h == nil { return fmt.Errorf("osupdates: unknown host %q", hostID) } if err := s.Store.SetOSRing(hostID, ring); err != nil { return err } s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates: %s is now ring %d.", hostID, ring), map[string]any{"host_id": hostID, "ring": ring}) if s.Bump != nil { s.Bump(hostID) } return nil } func (s *Service) SetEnabled(hostID string, on bool) error { h, err := s.Store.GetHost(hostID) if err != nil || h == nil { return fmt.Errorf("osupdates: unknown host %q", hostID) } if err := s.Store.SetOSEnabled(hostID, on); err != nil { return err } s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates on %s switched %s.", hostID, map[bool]string{true: "ON", false: "OFF"}[on]), map[string]any{"host_id": hostID, "enabled": on}) if s.Bump != nil { s.Bump(hostID) } return nil } // FleetLine is one box on the fleet page. type FleetLine struct { HostID string Ring int Enabled bool ReleaseID string LastOutcome string LastAt time.Time Pending int NotCovered int RestartNeeded int } // Fleet lists every box. func (s *Service) Fleet() ([]FleetLine, error) { hosts, err := s.Store.ListHosts() if err != nil { return nil, err } var out []FleetLine for _, h := range hosts { st := s.Store.GetOSHostSettings(h.HostID) l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled} if rep, _ := s.Store.LatestOSReport(h.HostID); rep != nil { var r Report _ = json.Unmarshal([]byte(rep.ReportJSON), &r) l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt l.Pending, l.NotCovered, l.RestartNeeded = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded) } out = append(out, l) } return out, nil } // FleetJSON is Fleet plus the approval status, for the operator's fleet route. func (s *Service) FleetJSON() (any, error) { lines, err := s.Fleet() if err != nil { return nil, err } rel, _ := s.Store.LatestOSRelease() out := map[string]any{"boxes": lines} if rel != nil { out["latest_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy} } return out, nil }