felhom.eu/hub/internal/web/system.go

package web

import (
	"fmt"
	"net/http"
	"sort"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/osupdates"
	"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
	"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
)

// The System page (hub v0.132.0; R-852, `09` decision 89, `11` §5.7): every box's real versions and its OS-update
// state, with the operator's buttons (ring, switch, approve now, approve the Docker set). Read from the hub's own
// records only: the newest host report's `system` stanza (agent ≥ v0.142.0) and the OS fleet lines.

// cell is one value with its colour: "" plain, "warn" amber, "bad" red (red = an alarm would fire).
type cell struct {
	Text  string
	Class string
	Title string
}

type systemRow struct {
	HostID, CustomerName string
	Ring                 int
	Enabled              bool
	Tunnel               cell
	HasFacts             bool
	FactsNote            string
	// host
	PVE, KernelRunning, KernelNextBoot, HostDebian cell
	HostRelease, HostPending, HostNotCovered       cell
	Held, RebootSince, KernelPanic, Oops           cell
	CrashRestarts24h, Guard                        cell
	Bundle                                         cell // R-840: the root-owned config bundle
	Agent                                          cell // R-530: the box's agent against the vouched one
	// guest
	GuestDebian, GuestRelease, GuestPending, GuestRestart cell
	// docker
	Engine, Containerd, LiveRestore, DockerRelease cell
	// last leg
	LastLeg cell
}

// OSSystemView is what the System page needs beyond OSUpdateAdmin (implemented by *osupdates.Service).
type OSSystemView interface {
	Fleet() ([]osupdates.FleetLine, error)
	Releases() []osupdates.ReleaseInfo
	CancelledReleases() []osupdates.ReleaseInfo
	Candidates() []osupdates.Status
	Thresholds() (stale, reboot, notCovered time.Duration)
	BundleThreshold() time.Duration
	AgentThreshold() time.Duration
	ApproveDocker() (string, error)
}

// agentCell is the "Agent" cell (R-530, hub v0.135.0): the box's agent against the vouched one, how far behind and
// since when. Amber while behind; red from the alarm's wait on (the operator alarm fires then). An unreadable
// version is "unknown", never a guess; nothing vouched → the version alone.
func agentCell(boxAgent, vouched string, since time.Time, after time.Duration, now time.Time) cell {
	if !semver.Valid(boxAgent) {
		return unknownCell("")
	}
	c := cell{Text: boxAgent}
	if !semver.Valid(vouched) {
		c.Title = "no vouched agent to compare with"
		return c
	}
	if semver.Compare(boxAgent, vouched) >= 0 {
		c.Title = "current (vouched " + vouched + ")"
		return c
	}
	c.Class = "warn"
	c.Text = boxAgent + " → " + vouched
	c.Title = osupdates.ReleasesBehind(boxAgent, vouched) + " — sign an agent_update for this box"
	if !since.IsZero() {
		c.Text += " (since " + since.UTC().Format("2006-01-02") + ")"
		if now.Sub(since) >= after {
			c.Class = "bad"
		}
	}
	return c
}

// floorRow is one line of the System page's "Version floors" table (R-604).
type floorRow struct {
	CustomerID, CustomerName, Version string
	Age                               cell
	HeldBack                          bool // the override is BELOW the global floor: the global does not move this box
}

func buildFloorRows(ovs []store.CustomerFloorOverride, global string, now time.Time) []floorRow {
	var out []floorRow
	for _, o := range ovs {
		r := floorRow{CustomerID: o.CustomerID, CustomerName: o.CustomerName, Version: o.Version}
		if o.SetAt.IsZero() {
			r.Age = cell{Text: "unknown", Class: "warn", Title: "set before hub v0.135.0 — the hub did not record when"}
		} else {
			r.Age = plain(ago(o.SetAt, now) + " (" + o.SetAt.UTC().Format("2006-01-02") + ")")
		}
		if semver.Valid(global) && semver.Valid(o.Version) && semver.Compare(global, o.Version) > 0 {
			r.HeldBack = true
		}
		out = append(out, r)
	}
	return out
}

func plain(s string) cell { return cell{Text: s} }

// bundleCell is the "Root files" cell (R-840): the box's config bundle against the vouched agent's. Amber while behind,
// red from the alarm's wait on, amber when a file was changed by hand (drift); "unknown" is never coloured as a fact.
func bundleCell(f sysfacts.System, vouchedAgent, vouchedSHA string, since time.Time, after time.Duration, now time.Time) cell {
	b := f.Bundle
	if !f.Present || b.Version == "" || b.Version == sysfacts.Unknown {
		return unknownCell("")
	}
	c := cell{Text: b.Version}
	switch {
	case vouchedSHA == "":
		c.Title = "no vouched bundle to compare with (the vouched agent carries none)"
	case b.BundleSHA256 != vouchedSHA:
		c.Class, c.Title = "warn", "behind the vouched agent "+vouchedAgent+"'s bundle — send it with a signed agent_config_update"
		if !since.IsZero() {
			c.Title += " (behind since " + since.UTC().Format("2006-01-02 15:04") + " UTC)"
			if now.Sub(since) >= after {
				c.Class = "bad"
			}
		}
	}
	if len(b.Drift) > 0 {
		c.Text += " (changed by hand)"
		c.Class = "warn"
		c.Title = "files differ from the installed bundle: " + strings.Join(b.Drift, ", ")
	}
	return c
}

func unknownCell(s string) cell {
	if s == "" || s == sysfacts.Unknown {
		return cell{Text: "unknown", Class: "warn", Title: "the box could not read it (agent older than v0.142.0, or the guest is down)"}
	}
	return cell{Text: s}
}

func ago(t time.Time, now time.Time) string {
	if t.IsZero() {
		return "never"
	}
	d := now.Sub(t)
	switch {
	case d < time.Hour:
		return fmt.Sprintf("%d min ago", int(d.Minutes()))
	case d < 48*time.Hour:
		return fmt.Sprintf("%d h ago", int(d.Hours()))
	}
	return fmt.Sprintf("%d days ago", int(d.Hours()/24))
}

// buildSystemRows is pure (the render test feeds it directly).
func buildSystemRows(lines []osupdates.FleetLine, facts map[string]sysfacts.System, names map[string]string,
	stale, reboot, notCov time.Duration, now time.Time) []systemRow {
	var rows []systemRow
	for _, l := range lines {
		f := facts[l.HostID]
		r := systemRow{HostID: l.HostID, CustomerName: names[l.HostID], Ring: l.Ring, Enabled: l.Enabled, HasFacts: f.Present}
		switch l.Tunnel {
		case "running":
			r.Tunnel = plain("running")
		case "not_running", "inactive":
			r.Tunnel = cell{Text: l.Tunnel, Class: "bad"}
		default:
			r.Tunnel = cell{Text: l.Tunnel, Class: "warn"}
		}
		if !f.Present {
			r.FactsNote = "no versions reported (agent older than v0.142.0)"
		} else if f.FactsError != "" {
			r.FactsNote = "partial: " + f.FactsError
		}
		r.PVE = unknownCell(sysfacts.ShortPVE(f.PVEVersion))
		r.KernelRunning = unknownCell(f.Host.KernelRunning)
		r.KernelNextBoot = unknownCell(f.Host.KernelNextBoot)
		if f.NextBootDiffers() {
			r.KernelNextBoot.Class, r.KernelNextBoot.Title = "warn", "the next boot changes the kernel ("+f.Host.KernelNextBootSource+")"
		}
		r.HostDebian = unknownCell(f.Host.Debian)
		r.HostRelease = plain(orDash(l.Host.ReleaseID))
		r.HostPending = plain(fmt.Sprint(l.Host.Pending))
		r.HostNotCovered = plain(fmt.Sprint(l.Host.NotCoveredFast))
		if l.Host.NotCoveredFast > 0 {
			r.HostNotCovered.Class = "warn"
		}
		switch {
		case f.Host.Held == nil:
			r.Held = unknownCell("")
		case len(f.Host.Held) == 0:
			r.Held = plain("none")
		default:
			r.Held = cell{Text: strings.Join(f.Host.Held, ", "), Class: "warn", Title: "held by hand (an undo) — the hub cannot see it otherwise (R-848)"}
		}
		if l.Host.RebootNeededSince.IsZero() {
			r.RebootSince = plain("no")
		} else {
			r.RebootSince = cell{Text: "since " + l.Host.RebootNeededSince.UTC().Format("2006-01-02"), Class: "warn"}
			if now.Sub(l.Host.RebootNeededSince) >= reboot {
				r.RebootSince.Class = "bad"
			}
		}
		if f.Host.KernelPanic != nil {
			r.KernelPanic = plain(fmt.Sprintf("%d s", *f.Host.KernelPanic))
			if *f.Host.KernelPanic == 0 {
				r.KernelPanic = cell{Text: "0 (stays off)", Class: "warn"}
			}
		} else {
			r.KernelPanic = unknownCell("")
		}
		r.Oops = plain("no")
		if f.Host.OopsThisBoot != nil && *f.Host.OopsThisBoot {
			r.Oops = cell{Text: "yes", Class: "warn", Title: "a kernel oops this boot"}
		}
		if cg := f.Host.CrashGuard; cg != nil {
			r.CrashRestarts24h = plain(fmt.Sprint(cg.In24h))
			if cg.In24h > 0 {
				r.CrashRestarts24h.Class = "warn"
			}
			if cg.Tripped {
				r.Guard = cell{Text: "TRIPPED " + cg.TrippedAt, Class: "bad", Title: cg.TrippedReason}
			} else {
				r.Guard = plain("armed")
			}
		} else {
			r.CrashRestarts24h, r.Guard = unknownCell(""), cell{Text: "not installed", Class: "warn"}
		}
		r.GuestDebian = unknownCell(f.Guest.Debian)
		r.GuestRelease = plain(orDash(l.Guest.ReleaseID))
		r.GuestPending = plain(fmt.Sprint(l.Guest.Pending))
		r.GuestRestart = plain(fmt.Sprint(l.Guest.RestartNeeded))
		r.Engine, r.Containerd = unknownCell(f.Guest.DockerEngine), unknownCell(f.Guest.Containerd)
		r.LiveRestore = unknownCell(f.Guest.LiveRestore)
		if f.Guest.LiveRestore == "off" {
			r.LiveRestore.Class, r.LiveRestore.Title = "warn", "a Docker step is refused until it is on (decision 87)"
		}
		r.DockerRelease = plain(orDash(l.Docker.ReleaseID))
		last := l.Guest
		if l.Host.LastAt.After(last.LastAt) {
			last = l.Host
		}
		if l.Docker.LastAt.After(last.LastAt) {
			last = l.Docker
		}
		ok := l.Guest.LastSuccessfulLeg
		r.LastLeg = cell{Text: fmt.Sprintf("%s · %s · %.0f s", ago(last.LastAt, now), orDash(last.LastOutcome), last.WrapperPassSeconds),
			Title: "last successful leg: " + ago(ok, now)}
		if l.Enabled && !ok.IsZero() && now.Sub(ok) >= stale {
			r.LastLeg.Class = "bad"
		} else if last.LastOutcome == "health_failed" || last.LastOutcome == "failed" || last.LastOutcome == "refused" {
			r.LastLeg.Class = "warn"
		}
		rows = append(rows, r)
	}
	sort.Slice(rows, func(i, j int) bool { return rows[i].HostID < rows[j].HostID })
	return rows
}

func orDash(s string) string {
	if s == "" {
		return "—"
	}
	return s
}

func (s *Server) handleSystem(w http.ResponseWriter, r *http.Request) {
	view, ok := s.osUpdates.(OSSystemView)
	if s.osUpdates == nil || !ok {
		http.Error(w, "os updates not configured", http.StatusServiceUnavailable)
		return
	}
	lines, err := view.Fleet()
	if err != nil {
		s.logger.Printf("[ERROR] system page: fleet: %v", err)
		http.Error(w, "Internal error", http.StatusInternalServerError)
		return
	}
	hosts, _ := s.store.ListHosts()
	facts, names := map[string]sysfacts.System{}, map[string]string{}
	for _, h := range hosts {
		names[h.HostID] = s.customerName(h.CustomerID)
		if rj, _ := s.store.GetLatestHostReportJSON(h.CustomerID); rj != "" {
			facts[h.HostID] = sysfacts.Parse(rj)
		}
	}
	stale, reboot, notCov := view.Thresholds()
	rows := buildSystemRows(lines, facts, names, stale, reboot, notCov, time.Now())
	man := s.store.GetArtifactManifest()
	agents := map[string]string{}
	for _, h := range hosts {
		agents[h.HostID] = h.AgentVersion
	}
	for i := range rows {
		rows[i].Bundle = bundleCell(facts[rows[i].HostID], man.AgentVersion, man.BundleSHA256,
			s.store.BundleBehindSince(rows[i].HostID), view.BundleThreshold(), time.Now())
		rows[i].Agent = agentCell(agents[rows[i].HostID], man.AgentVersion,
			s.store.AgentBehindSince(rows[i].HostID), view.AgentThreshold(), time.Now())
	}
	global := s.store.GetGlobalMinControllerVersion()
	ovs, oerr := s.store.CustomerFloorOverrides()
	if oerr != nil {
		s.logger.Printf("[ERROR] system page: floor overrides: %v", oerr)
	}
	data := map[string]interface{}{
		"Rows":       rows,
		"GlobalFloor":  global,
		"VouchedAgent": man.AgentVersion,
		"Floors":       buildFloorRows(ovs, global, time.Now()),
		"Releases":   view.Releases(),
		"Cancelled":  view.CancelledReleases(),
		"Candidates": view.Candidates(),
		"Flash":      r.URL.Query().Get("flash"),
		"FlashErr":   r.URL.Query().Get("err"),
		"CSRFToken":  s.getCSRFToken(r),
	}
	if err := s.templates.ExecuteTemplate(w, "system.html", data); err != nil {
		s.logger.Printf("[ERROR] system.html template: %v", err)
	}
}