felhom.eu/hub/internal/osupdates/service.go

// Package osupdates is the hub half of the guest fast lane (`11-os-updates.md` §5.3, §8 step 2; hub v0.130.0).
//
// Ring 0 (the demo boxes) installs every pending Debian / Debian-Security fix each night and reports the FULL installed
// set (C9). The hub derives the CANDIDATE: every Debian-origin package=version that all ring-0 boxes having that
// package agree on. A candidate is APPROVED when, since it was first seen:
//   - every ring-0 box runs it (its newest report matches the candidate for every package it has),
//   - ApproveAfter (default 24 h) has passed with every ring-0 report healthy, and
//   - every ring-0 box has completed NightsRequired (default 1) post-backup night runs.
//
// The operator can approve at once ("approve now", an urgent fix). Ring 1 then gets the release in its desired state
// and installs exactly those versions. Rules pinned by service_test.go.
package osupdates

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// Event types — operator-only except EventApplied, the household's line (`11` §5.7).
const (
	EventApplied        = "os_update_applied"          // info, CUSTOMER: "system security fixes installed"
	EventFailed         = "os_update_failed"           // error, operator: the run failed or was refused
	EventHealthFailed   = "os_update_health_failed"    // error, operator: the guest was not healthy after the run
	EventReleaseApprove = "os_release_approved"        // info, operator
	EventApprovedNow    = "os_release_approved_now"    // warning, operator: an operator approved at once
	EventSettings       = "os_update_settings_changed" // info, operator: ring or switch changed
)

// Package is one name=version with its origin ("Debian" | "Debian-Security").
type Package struct {
	Name    string `json:"name"`
	Version string `json:"version"`
	Origin  string `json:"origin"`
}

// Report is what the agent POSTs after every run (the wrapper's report plus the leg's verdict).
type Report struct {
	RunID               string          `json:"run_id"`
	Trigger             string          `json:"trigger"` // night | debug
	Mode                string          `json:"mode"`    // apply | inventory
	Ring                int             `json:"ring"`
	ReleaseID           string          `json:"release_id"`
	Outcome             string          `json:"outcome"` // applied | nothing | inventory | refused | failed | health_failed
	Healthy             bool            `json:"healthy"`
	HealthReason        string          `json:"health_reason,omitempty"`
	VMID                int             `json:"vmid"`
	Upgraded            []Package       `json:"upgraded,omitempty"`
	Installed           []Package       `json:"installed,omitempty"`
	Pending             []PendingPkg    `json:"pending,omitempty"`
	NotCovered          []string        `json:"not_covered,omitempty"`
	RestartNeeded       []string        `json:"restart_needed,omitempty"`
	DockerRestartNeeded bool            `json:"docker_restart_needed,omitempty"`
	RebootNeeded        bool            `json:"reboot_needed,omitempty"`
	Refused             json.RawMessage `json:"refused,omitempty"`
	Log                 []string        `json:"log,omitempty"`
}

// PendingPkg is one update the guest's sources offer.
type PendingPkg struct {
	Name   string   `json:"name"`
	From   string   `json:"from"`
	To     string   `json:"to"`
	Origin []string `json:"origin"`
}

// Block is what a box receives in its desired state (`os_update`).
type Block struct {
	Ring    int           `json:"ring"`
	Enabled bool          `json:"enabled"`
	Release *ReleaseBlock `json:"release,omitempty"`
}

// ReleaseBlock is the newest approved release, for ring 1.
type ReleaseBlock struct {
	ID       string    `json:"id"`
	Snapshot string    `json:"snapshot"` // approval time as YYYYMMDDTHHMMSSZ (decision 79: snapshot.debian.org)
	Packages []Package `json:"packages"`
}

// Service ties the store, the events and the clock together.
type Service struct {
	Store          *store.Store
	Emit           func(customerID, eventType, severity, message, details, source string) // dispatcher; nil in tests
	ApproveAfter   time.Duration
	NightsRequired int
	Logger         *log.Logger
	Now            func() time.Time
	Bump           func(hostID string) // bump a host's desired generation (store.BumpHostDesired); nil in tests
}

func (s *Service) now() time.Time {
	if s.Now != nil {
		return s.Now()
	}
	return time.Now()
}

func (s *Service) logf(f string, a ...any) {
	if s.Logger != nil {
		s.Logger.Printf(f, a...)
	}
}

func (s *Service) event(customerID, typ, sev, msg string, details any) {
	dj := ""
	if details != nil {
		if b, err := json.Marshal(details); err == nil {
			dj = string(b)
		}
	}
	if _, err := s.Store.SaveEvent(customerID, typ, sev, msg, dj, "hub"); err != nil {
		s.logf("[WARN] osupdates: save event %s: %v", typ, err)
	}
	if s.Emit != nil {
		s.Emit(customerID, typ, sev, msg, dj, "hub")
	}
}

// Ingest stores a run and raises its events. The household gets one line per run that installed something.
func (s *Service) Ingest(hostID string, r Report) error {
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return fmt.Errorf("osupdates: unknown host %q", hostID)
	}
	raw, _ := json.Marshal(r)
	if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode, Outcome: r.Outcome,
		Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil {
		return err
	}
	s.logf("[INFO] osupdates: %s reported run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d",
		hostID, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded))
	details := map[string]any{"host_id": hostID, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome,
		"upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason}
	switch r.Outcome {
	case "applied", "health_failed":
		s.event(h.CustomerID, EventApplied, "info",
			fmt.Sprintf("System security fixes installed (%d package(s)).", len(r.Upgraded)), details)
		if !r.Healthy || r.Outcome == "health_failed" {
			s.event(h.CustomerID, EventHealthFailed, "error",
				fmt.Sprintf("OS update on %s: the guest was NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically (no guest snapshot is possible, R-837); last night's whole-guest backup is the undo.",
					hostID, len(r.Upgraded), r.HealthReason), details)
		}
	case "refused", "failed":
		s.event(h.CustomerID, EventFailed, "error",
			fmt.Sprintf("OS update on %s %s: %s", hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details)
	}
	return nil
}

// candidate derives the version set every ring-0 box agrees on, from each box's newest report.
func (s *Service) candidate(ring0 []string) (map[string]Package, map[string]*store.OSReport, error) {
	latest := map[string]*store.OSReport{}
	byPkg := map[string]map[string]Package{} // name -> host -> pkg
	for _, h := range ring0 {
		rep, err := s.Store.LatestOSReport(h)
		if err != nil {
			return nil, nil, err
		}
		if rep == nil {
			return nil, nil, nil // a ring-0 box that never reported: no candidate
		}
		latest[h] = rep
		var r Report
		if err := json.Unmarshal([]byte(rep.ReportJSON), &r); err != nil {
			return nil, nil, err
		}
		for _, p := range r.Installed {
			if p.Origin != "Debian" && p.Origin != "Debian-Security" {
				continue
			}
			if byPkg[p.Name] == nil {
				byPkg[p.Name] = map[string]Package{}
			}
			byPkg[p.Name][h] = p
		}
	}
	cand := map[string]Package{}
	for name, hosts := range byPkg {
		var v string
		agree := true
		var pick Package
		for _, p := range hosts {
			if v == "" {
				v, pick = p.Version, p
			} else if p.Version != v {
				agree = false
			}
		}
		if agree {
			cand[name] = pick
		}
	}
	return cand, latest, nil
}

func fingerprint(c map[string]Package) (string, []Package) {
	var list []Package
	for _, p := range c {
		list = append(list, p)
	}
	sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name })
	h := sha256.New()
	for _, p := range list {
		fmt.Fprintf(h, "%s=%s\n", p.Name, p.Version)
	}
	return hex.EncodeToString(h.Sum(nil))[:16], list
}

// ring0Hosts lists the ring-0 boxes that have the switch ON.
func (s *Service) ring0Hosts() ([]string, error) {
	hosts, err := s.Store.ListHosts()
	if err != nil {
		return nil, err
	}
	var out []string
	for _, h := range hosts {
		if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 0 && st.Enabled {
			out = append(out, h.HostID)
		}
	}
	sort.Strings(out)
	return out, nil
}

// Status explains the approval state (for the log and the fleet page).
type Status struct {
	Fingerprint string
	FirstSeen   time.Time
	Packages    int
	Waiting     string // why not approved yet ("" = approved or nothing to do)
}

// Evaluate checks the approval rule and approves when it holds. Called every minute.
func (s *Service) Evaluate() (Status, error) {
	ring0, err := s.ring0Hosts()
	if err != nil || len(ring0) == 0 {
		return Status{Waiting: "no ring-0 box"}, err
	}
	cand, _, err := s.candidate(ring0)
	if err != nil || cand == nil {
		return Status{Waiting: "a ring-0 box has not reported"}, err
	}
	fp, list := fingerprint(cand)
	pj, _ := json.Marshal(list)
	first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
	if err != nil {
		return Status{}, err
	}
	st := Status{Fingerprint: fp, FirstSeen: first, Packages: len(list)}
	if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp {
		st.Waiting = ""
		return st, nil // already approved
	}
	if age := s.now().Sub(first); age < s.ApproveAfter {
		st.Waiting = fmt.Sprintf("healthy for %s of %s", age.Round(time.Minute), s.ApproveAfter)
		return st, nil
	}
	for _, h := range ring0 {
		reps, err := s.Store.OSReportsSince(h, first)
		if err != nil {
			return st, err
		}
		nights := 0
		for _, r := range reps {
			if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" {
				st.Waiting = fmt.Sprintf("%s reported %s (healthy=%v) at %s since the set was first seen", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339))
				return st, nil
			}
			if r.Trigger == "night" {
				nights++
			}
		}
		if nights < s.NightsRequired {
			st.Waiting = fmt.Sprintf("%s has %d of %d night run(s) since the set was first seen", h, nights, s.NightsRequired)
			return st, nil
		}
	}
	if err := s.approve(fp, list, "auto"); err != nil {
		return st, err
	}
	return st, nil
}

// ApproveNow approves the current candidate at once (operator, urgent fix).
func (s *Service) ApproveNow() (string, error) {
	ring0, err := s.ring0Hosts()
	if err != nil || len(ring0) == 0 {
		return "", fmt.Errorf("osupdates: no ring-0 box")
	}
	cand, _, err := s.candidate(ring0)
	if err != nil || cand == nil {
		return "", fmt.Errorf("osupdates: a ring-0 box has not reported yet")
	}
	fp, list := fingerprint(cand)
	pj, _ := json.Marshal(list)
	if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil {
		return "", err
	}
	if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp {
		return rel.ID, nil
	}
	if err := s.approve(fp, list, "operator"); err != nil {
		return "", err
	}
	rel, _ := s.Store.LatestOSRelease()
	s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s at once (%d packages), without the wait.", rel.ID, len(list)),
		map[string]any{"release_id": rel.ID, "packages": len(list)})
	return rel.ID, nil
}

func (s *Service) approve(fp string, list []Package, by string) error {
	at := s.now().UTC().Truncate(time.Second)
	id := "os-" + at.Format("20060102-150405")
	pj, _ := json.Marshal(list)
	if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil {
		return err
	}
	s.logf("[WARN] osupdates: OS release %s APPROVED by %s (%d packages, fingerprint %s)", id, by, len(list), fp)
	s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s approved (%s, %d packages).", id, by, len(list)),
		map[string]any{"release_id": id, "approved_by": by, "packages": len(list), "fingerprint": fp})
	if s.Bump != nil {
		hosts, _ := s.Store.ListHosts()
		for _, h := range hosts {
			if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled {
				s.Bump(h.HostID)
			}
		}
	}
	return nil
}

// DesiredBlock is the `os_update` block a box receives.
func (s *Service) DesiredBlock(hostID string) Block {
	st := s.Store.GetOSHostSettings(hostID)
	b := Block{Ring: st.Ring, Enabled: st.Enabled}
	if st.Ring == 1 {
		if rel, err := s.Store.LatestOSRelease(); err == nil && rel != nil {
			var list []Package
			if json.Unmarshal([]byte(rel.PackagesJSON), &list) == nil {
				b.Release = &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list}
			}
		}
	}
	return b
}

// SetRing / SetEnabled are operator acts; each bumps the box's desired generation and is an operator event.
func (s *Service) SetRing(hostID string, ring int) error {
	if ring != 0 && ring != 1 {
		return fmt.Errorf("osupdates: ring must be 0 or 1")
	}
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return fmt.Errorf("osupdates: unknown host %q", hostID)
	}
	if err := s.Store.SetOSRing(hostID, ring); err != nil {
		return err
	}
	s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates: %s is now ring %d.", hostID, ring), map[string]any{"host_id": hostID, "ring": ring})
	if s.Bump != nil {
		s.Bump(hostID)
	}
	return nil
}

func (s *Service) SetEnabled(hostID string, on bool) error {
	h, err := s.Store.GetHost(hostID)
	if err != nil || h == nil {
		return fmt.Errorf("osupdates: unknown host %q", hostID)
	}
	if err := s.Store.SetOSEnabled(hostID, on); err != nil {
		return err
	}
	s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates on %s switched %s.", hostID, map[bool]string{true: "ON", false: "OFF"}[on]),
		map[string]any{"host_id": hostID, "enabled": on})
	if s.Bump != nil {
		s.Bump(hostID)
	}
	return nil
}

// FleetLine is one box on the fleet page.
type FleetLine struct {
	HostID        string
	Ring          int
	Enabled       bool
	ReleaseID     string
	LastOutcome   string
	LastAt        time.Time
	Pending       int
	NotCovered    int
	RestartNeeded int
}

// Fleet lists every box.
func (s *Service) Fleet() ([]FleetLine, error) {
	hosts, err := s.Store.ListHosts()
	if err != nil {
		return nil, err
	}
	var out []FleetLine
	for _, h := range hosts {
		st := s.Store.GetOSHostSettings(h.HostID)
		l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled}
		if rep, _ := s.Store.LatestOSReport(h.HostID); rep != nil {
			var r Report
			_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
			l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt
			l.Pending, l.NotCovered, l.RestartNeeded = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded)
		}
		out = append(out, l)
	}
	return out, nil
}

// FleetJSON is Fleet plus the approval status, for the operator's fleet route.
func (s *Service) FleetJSON() (any, error) {
	lines, err := s.Fleet()
	if err != nil {
		return nil, err
	}
	rel, _ := s.Store.LatestOSRelease()
	out := map[string]any{"boxes": lines}
	if rel != nil {
		out["latest_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy}
	}
	return out, nil
}