admin
ab3b7ea2f4
gates / gates (push) Successful in 2m47s
KernelDue / KernelNotify (09-20 h Budapest, one per 20 h, max 3, registered
address, only an accepted mail counts) / os_update.kernel {kver, tonight}
(no mail, no step) / layer kernel ingest + operator events / Approve kernel
set after every ring-0 box booted it healthily after a night stage / two
System page cells. 11 §5.11 written; §5.10 status corrected (proven).
Installer uninstall knows the two GRUB generators (unreleased).
Evidence: audits/kernel-lane-2026-10-07/ (red-proofs, boot timing).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
454 lines
17 KiB
Go
454 lines
17 KiB
Go
package web
|
||
|
||
import (
|
||
"fmt"
|
||
"net/http"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/osupdates"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
|
||
)
|
||
|
||
// The System page (hub v0.132.0; R-852, `09` decision 89, `11` §5.7): every box's real versions and its OS-update
|
||
// state, with the operator's buttons (ring, switch, approve now, approve the Docker set). Read from the hub's own
|
||
// records only: the newest host report's `system` stanza (agent ≥ v0.142.0) and the OS fleet lines.
|
||
|
||
// cell is one value with its colour: "" plain, "warn" amber, "bad" red (red = an alarm would fire).
|
||
type cell struct {
|
||
Text string
|
||
Class string
|
||
Title string
|
||
}
|
||
|
||
type systemRow struct {
|
||
HostID, CustomerName string
|
||
Ring int
|
||
Enabled bool
|
||
Tunnel cell
|
||
HasFacts bool
|
||
FactsNote string
|
||
// host
|
||
PVE, KernelRunning, KernelNextBoot, HostDebian cell
|
||
HostRelease, HostPending, HostNotCovered cell
|
||
Held, RebootSince, KernelPanic, Oops cell
|
||
CrashRestarts24h, Guard cell
|
||
KernelDefault, KernelStep cell // R-836: the GRUB default kernel; the kernel lane's step
|
||
Bundle cell // R-840: the root-owned config bundle
|
||
Agent cell // R-530: the box's agent against the vouched one
|
||
// guest
|
||
GuestDebian, GuestRelease, GuestPending, GuestRestart cell
|
||
Trim cell // R-444: the last `pct fstrim` of the guest
|
||
// docker
|
||
Engine, Containerd, LiveRestore, DockerRelease cell
|
||
// last leg
|
||
LastLeg cell
|
||
}
|
||
|
||
// OSSystemView is what the System page needs beyond OSUpdateAdmin (implemented by *osupdates.Service).
|
||
type OSSystemView interface {
|
||
Fleet() ([]osupdates.FleetLine, error)
|
||
Releases() []osupdates.ReleaseInfo
|
||
CancelledReleases() []osupdates.ReleaseInfo
|
||
Candidates() []osupdates.Status
|
||
Thresholds() (stale, reboot, notCovered time.Duration)
|
||
BundleThreshold() time.Duration
|
||
AgentThreshold() time.Duration
|
||
ApproveDocker() (string, error)
|
||
ApprovePVE() (string, error) // R-812 option A: the Proxmox package set
|
||
ApproveKernel() (string, error) // R-836: the kernel set
|
||
KernelLineFor(hostID string) osupdates.KernelLine // R-836: one box's kernel step and day-before mail
|
||
}
|
||
|
||
// kernelCells are the System page's two kernel-lane cells (R-836, `11` §5.11): the GRUB default (amber while a one-shot
|
||
// flag names another kernel for the next boot), and the step — the newest result, the kernel the box is due, whether
|
||
// the household was told for tonight. "unknown" when the box reports no kernel lane (an older agent).
|
||
func kernelCells(f sysfacts.System, kl osupdates.KernelLine, now time.Time) (dflt, step cell) {
|
||
lane := f.Host.KernelLane
|
||
if lane == nil {
|
||
if kl.LastOutcome == "" {
|
||
return unknownCell(""), cell{Text: "—", Title: "the box reports no kernel lane (agent older than v0.152.0)"}
|
||
}
|
||
lane = &sysfacts.KernelLane{} // a step result without facts (an older facts read) still shows
|
||
}
|
||
dflt = unknownCell(lane.Default)
|
||
if lane.Flag != nil && *lane.Flag != "" {
|
||
dflt.Text += " (once: " + *lane.Flag + ")"
|
||
dflt.Class, dflt.Title = "warn", "the next boot runs "+*lane.Flag+" ONCE; the boot after it the default again"
|
||
}
|
||
var parts []string
|
||
if kl.LastOutcome != "" {
|
||
parts = append(parts, fmt.Sprintf("%s %s %s", kl.LastKernel, kl.LastOutcome, ago(kl.LastAt, now)))
|
||
}
|
||
switch {
|
||
case kl.Due != "" && kl.Tonight:
|
||
parts = append(parts, "due "+kl.Due+" — household told "+ago(kl.NoticeAt, now)+": TONIGHT")
|
||
case kl.Due != "":
|
||
parts = append(parts, "due "+kl.Due+" — not told yet (mails 09–20 h)")
|
||
}
|
||
if lane.Phase != "" && lane.Phase != "none" {
|
||
parts = append(parts, "phase "+lane.Phase)
|
||
}
|
||
if len(parts) == 0 {
|
||
parts = append(parts, "—")
|
||
}
|
||
step = cell{Text: strings.Join(parts, " · "), Title: strings.TrimSpace(kl.Why + " " + kl.LastReason)}
|
||
switch kl.LastOutcome {
|
||
case "fell_back", "health_failed", "self_reverted", "refused", "failed":
|
||
step.Class = "warn"
|
||
case "revert_failed":
|
||
step.Class = "bad"
|
||
}
|
||
if len(lane.SetupProblems) > 0 {
|
||
step.Class = "warn"
|
||
step.Title = "cannot do a one-shot boot: " + strings.Join(lane.SetupProblems, "; ")
|
||
}
|
||
return dflt, step
|
||
}
|
||
|
||
// agentCell is the "Agent" cell (R-530, hub v0.135.0): the box's agent against the vouched one, how far behind and
|
||
// since when. Amber while behind; red from the alarm's wait on (the operator alarm fires then). An unreadable
|
||
// version is "unknown", never a guess; nothing vouched → the version alone.
|
||
func agentCell(boxAgent, vouched string, since time.Time, after time.Duration, now time.Time) cell {
|
||
if !semver.Valid(boxAgent) {
|
||
return unknownCell("")
|
||
}
|
||
c := cell{Text: boxAgent}
|
||
if !semver.Valid(vouched) {
|
||
c.Title = "no vouched agent to compare with"
|
||
return c
|
||
}
|
||
if semver.Compare(boxAgent, vouched) >= 0 {
|
||
c.Title = "current (vouched " + vouched + ")"
|
||
return c
|
||
}
|
||
c.Class = "warn"
|
||
c.Text = boxAgent + " → " + vouched
|
||
c.Title = osupdates.ReleasesBehind(boxAgent, vouched) + " — sign an agent_update for this box"
|
||
if !since.IsZero() {
|
||
c.Text += " (since " + since.UTC().Format("2006-01-02") + ")"
|
||
if now.Sub(since) >= after {
|
||
c.Class = "bad"
|
||
}
|
||
}
|
||
return c
|
||
}
|
||
|
||
// floorRow is one line of the System page's "Version floors" table (R-604).
|
||
type floorRow struct {
|
||
CustomerID, CustomerName, Version string
|
||
Age cell
|
||
HeldBack bool // the override is BELOW the global floor: the global does not move this box
|
||
}
|
||
|
||
func buildFloorRows(ovs []store.CustomerFloorOverride, global string, now time.Time) []floorRow {
|
||
var out []floorRow
|
||
for _, o := range ovs {
|
||
r := floorRow{CustomerID: o.CustomerID, CustomerName: o.CustomerName, Version: o.Version}
|
||
if o.SetAt.IsZero() {
|
||
r.Age = cell{Text: "unknown", Class: "warn", Title: "set before hub v0.135.0 — the hub did not record when"}
|
||
} else {
|
||
r.Age = plain(ago(o.SetAt, now) + " (" + o.SetAt.UTC().Format("2006-01-02") + ")")
|
||
}
|
||
if semver.Valid(global) && semver.Valid(o.Version) && semver.Compare(global, o.Version) > 0 {
|
||
r.HeldBack = true
|
||
}
|
||
out = append(out, r)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func plain(s string) cell { return cell{Text: s} }
|
||
|
||
// trimStaleAfter is when a guest's last SUCCESSFUL disk trim reads amber: the boxes trim weekly (`09` §3 decision
|
||
// 139), so two missed weeks are worth a look.
|
||
const trimStaleAfter = 14 * 24 * time.Hour
|
||
|
||
// trimCell is the "Last disk trim" cell (R-444): per guest, the last successful `pct fstrim` (from `last_ok_at`) and what
|
||
// the newest attempt freed. Amber when the newest attempt failed (its error is shown) or the last success is older than
|
||
// 14 days — judged on `last_ok_at`, never on the attempt time (an attempt is not a result). "—" when the agent sends no
|
||
// stanza (no trimmer); an unreadable time is "unknown", never a guess.
|
||
func trimCell(dt *sysfacts.DiskTrim, now time.Time) cell {
|
||
if dt == nil {
|
||
return cell{Text: "—", Title: "the agent reports no disk trim (an agent without the weekly trim)"}
|
||
}
|
||
if len(dt.Guests) == 0 {
|
||
return cell{Text: "none yet", Title: "the weekly trim runs (" + dt.Schedule + ") and has not trimmed a guest yet"}
|
||
}
|
||
out := cell{Title: "schedule: " + dt.Schedule}
|
||
var texts []string
|
||
for _, g := range dt.Guests {
|
||
t, warn, why := guestTrimText(g, now)
|
||
if len(dt.Guests) > 1 {
|
||
t = fmt.Sprintf("%d: %s", g.VMID, t)
|
||
}
|
||
texts = append(texts, t)
|
||
if warn {
|
||
out.Class = "warn"
|
||
out.Title = fmt.Sprintf("guest %d: %s — %s", g.VMID, why, out.Title)
|
||
}
|
||
}
|
||
out.Text = strings.Join(texts, " / ")
|
||
return out
|
||
}
|
||
|
||
func guestTrimText(g sysfacts.GuestTrim, now time.Time) (text string, warn bool, why string) {
|
||
var okAt time.Time
|
||
if g.LastOKAt != "" {
|
||
t, err := time.Parse(time.RFC3339, g.LastOKAt)
|
||
if err != nil {
|
||
return "unknown", true, fmt.Sprintf("unreadable last_ok_at %q", g.LastOKAt)
|
||
}
|
||
okAt = t
|
||
}
|
||
attAt, aerr := time.Parse(time.RFC3339, g.LastAttemptAt)
|
||
if aerr != nil {
|
||
return "unknown", true, fmt.Sprintf("unreadable last_attempt_at %q", g.LastAttemptAt)
|
||
}
|
||
if g.OK {
|
||
text = fmt.Sprintf("%s · %.1f GiB", ago(attAt, now), float64(g.BytesTrimmed)/(1<<30))
|
||
if !okAt.IsZero() {
|
||
text = fmt.Sprintf("%s · %.1f GiB", ago(okAt, now), float64(g.BytesTrimmed)/(1<<30))
|
||
} else {
|
||
okAt = attAt
|
||
}
|
||
} else {
|
||
last := "never ok"
|
||
if !okAt.IsZero() {
|
||
last = "last ok " + ago(okAt, now)
|
||
}
|
||
errText := g.Error
|
||
if errText == "" {
|
||
errText = "no error text"
|
||
}
|
||
text = fmt.Sprintf("FAILED %s: %s (%s)", ago(attAt, now), errText, last)
|
||
warn, why = true, "the newest trim failed"
|
||
}
|
||
if !okAt.IsZero() && now.Sub(okAt) > trimStaleAfter {
|
||
warn = true
|
||
why = strings.TrimPrefix(why+"; ", "; ") + "the last successful trim is older than 14 days (the boxes trim weekly)"
|
||
}
|
||
return text, warn, why
|
||
}
|
||
|
||
// bundleCell is the "Root files" cell (R-840): the box's config bundle against the vouched agent's. Amber while behind,
|
||
// red from the alarm's wait on, amber when a file was changed by hand (drift); "unknown" is never coloured as a fact.
|
||
func bundleCell(f sysfacts.System, vouchedAgent, vouchedSHA string, since time.Time, after time.Duration, now time.Time) cell {
|
||
b := f.Bundle
|
||
if !f.Present || b.Version == "" || b.Version == sysfacts.Unknown {
|
||
return unknownCell("")
|
||
}
|
||
c := cell{Text: b.Version}
|
||
switch {
|
||
case vouchedSHA == "":
|
||
c.Title = "no vouched bundle to compare with (the vouched agent carries none)"
|
||
case b.BundleSHA256 != vouchedSHA:
|
||
c.Class, c.Title = "warn", "behind the vouched agent "+vouchedAgent+"'s bundle — send it with a signed agent_config_update"
|
||
if !since.IsZero() {
|
||
c.Title += " (behind since " + since.UTC().Format("2006-01-02 15:04") + " UTC)"
|
||
if now.Sub(since) >= after {
|
||
c.Class = "bad"
|
||
}
|
||
}
|
||
}
|
||
if len(b.Drift) > 0 {
|
||
c.Text += " (changed by hand)"
|
||
c.Class = "warn"
|
||
c.Title = "files differ from the installed bundle: " + strings.Join(b.Drift, ", ")
|
||
}
|
||
return c
|
||
}
|
||
|
||
func unknownCell(s string) cell {
|
||
if s == "" || s == sysfacts.Unknown {
|
||
return cell{Text: "unknown", Class: "warn", Title: "the box could not read it (agent older than v0.142.0, or the guest is down)"}
|
||
}
|
||
return cell{Text: s}
|
||
}
|
||
|
||
func ago(t time.Time, now time.Time) string {
|
||
if t.IsZero() {
|
||
return "never"
|
||
}
|
||
d := now.Sub(t)
|
||
switch {
|
||
case d < time.Hour:
|
||
return fmt.Sprintf("%d min ago", int(d.Minutes()))
|
||
case d < 48*time.Hour:
|
||
return fmt.Sprintf("%d h ago", int(d.Hours()))
|
||
}
|
||
return fmt.Sprintf("%d days ago", int(d.Hours()/24))
|
||
}
|
||
|
||
// buildSystemRows is pure (the render test feeds it directly).
|
||
func buildSystemRows(lines []osupdates.FleetLine, facts map[string]sysfacts.System, names map[string]string,
|
||
stale, reboot, notCov time.Duration, now time.Time) []systemRow {
|
||
var rows []systemRow
|
||
for _, l := range lines {
|
||
f := facts[l.HostID]
|
||
r := systemRow{HostID: l.HostID, CustomerName: names[l.HostID], Ring: l.Ring, Enabled: l.Enabled, HasFacts: f.Present}
|
||
switch l.Tunnel {
|
||
case "running":
|
||
r.Tunnel = plain("running")
|
||
case "not_running", "inactive":
|
||
r.Tunnel = cell{Text: l.Tunnel, Class: "bad"}
|
||
default:
|
||
r.Tunnel = cell{Text: l.Tunnel, Class: "warn"}
|
||
}
|
||
if !f.Present {
|
||
r.FactsNote = "no versions reported (agent older than v0.142.0)"
|
||
} else if f.FactsError != "" {
|
||
r.FactsNote = "partial: " + f.FactsError
|
||
}
|
||
r.PVE = unknownCell(sysfacts.ShortPVE(f.PVEVersion))
|
||
r.KernelRunning = unknownCell(f.Host.KernelRunning)
|
||
r.KernelNextBoot = unknownCell(f.Host.KernelNextBoot)
|
||
if f.NextBootDiffers() {
|
||
r.KernelNextBoot.Class, r.KernelNextBoot.Title = "warn", "the next boot changes the kernel ("+f.Host.KernelNextBootSource+")"
|
||
}
|
||
r.HostDebian = unknownCell(f.Host.Debian)
|
||
r.HostRelease = plain(orDash(l.Host.ReleaseID))
|
||
r.HostPending = plain(fmt.Sprint(l.Host.Pending))
|
||
r.HostNotCovered = plain(fmt.Sprint(l.Host.NotCoveredFast))
|
||
if l.Host.NotCoveredFast > 0 {
|
||
r.HostNotCovered.Class = "warn"
|
||
}
|
||
switch {
|
||
case f.Host.Held == nil:
|
||
r.Held = unknownCell("")
|
||
case len(f.Host.Held) == 0:
|
||
r.Held = plain("none")
|
||
default:
|
||
r.Held = cell{Text: strings.Join(f.Host.Held, ", "), Class: "warn", Title: "held by hand (an undo) — the hub cannot see it otherwise (R-848)"}
|
||
}
|
||
if l.Host.RebootNeededSince.IsZero() {
|
||
r.RebootSince = plain("no")
|
||
} else {
|
||
r.RebootSince = cell{Text: "since " + l.Host.RebootNeededSince.UTC().Format("2006-01-02"), Class: "warn"}
|
||
if now.Sub(l.Host.RebootNeededSince) >= reboot {
|
||
r.RebootSince.Class = "bad"
|
||
}
|
||
}
|
||
if f.Host.KernelPanic != nil {
|
||
r.KernelPanic = plain(fmt.Sprintf("%d s", *f.Host.KernelPanic))
|
||
if *f.Host.KernelPanic == 0 {
|
||
r.KernelPanic = cell{Text: "0 (stays off)", Class: "warn"}
|
||
}
|
||
} else {
|
||
r.KernelPanic = unknownCell("")
|
||
}
|
||
r.Oops = plain("no")
|
||
if f.Host.OopsThisBoot != nil && *f.Host.OopsThisBoot {
|
||
r.Oops = cell{Text: "yes", Class: "warn", Title: "a kernel oops this boot"}
|
||
}
|
||
if cg := f.Host.CrashGuard; cg != nil {
|
||
r.CrashRestarts24h = plain(fmt.Sprint(cg.In24h))
|
||
if cg.In24h > 0 {
|
||
r.CrashRestarts24h.Class = "warn"
|
||
}
|
||
if cg.Tripped {
|
||
r.Guard = cell{Text: "TRIPPED " + cg.TrippedAt, Class: "bad", Title: cg.TrippedReason}
|
||
} else {
|
||
r.Guard = plain("armed")
|
||
}
|
||
} else {
|
||
r.CrashRestarts24h, r.Guard = unknownCell(""), cell{Text: "not installed", Class: "warn"}
|
||
}
|
||
r.GuestDebian = unknownCell(f.Guest.Debian)
|
||
r.GuestRelease = plain(orDash(l.Guest.ReleaseID))
|
||
r.GuestPending = plain(fmt.Sprint(l.Guest.Pending))
|
||
r.GuestRestart = plain(fmt.Sprint(l.Guest.RestartNeeded))
|
||
r.Trim = trimCell(f.DiskTrim, now)
|
||
r.Engine, r.Containerd = unknownCell(f.Guest.DockerEngine), unknownCell(f.Guest.Containerd)
|
||
r.LiveRestore = unknownCell(f.Guest.LiveRestore)
|
||
if f.Guest.LiveRestore == "off" {
|
||
r.LiveRestore.Class, r.LiveRestore.Title = "warn", "a Docker step is refused until it is on (decision 87)"
|
||
}
|
||
r.DockerRelease = plain(orDash(l.Docker.ReleaseID))
|
||
last := l.Guest
|
||
if l.Host.LastAt.After(last.LastAt) {
|
||
last = l.Host
|
||
}
|
||
if l.Docker.LastAt.After(last.LastAt) {
|
||
last = l.Docker
|
||
}
|
||
ok := l.Guest.LastSuccessfulLeg
|
||
r.LastLeg = cell{Text: fmt.Sprintf("%s · %s · %.0f s", ago(last.LastAt, now), orDash(last.LastOutcome), last.WrapperPassSeconds),
|
||
Title: "last successful leg: " + ago(ok, now)}
|
||
if l.Enabled && !ok.IsZero() && now.Sub(ok) >= stale {
|
||
r.LastLeg.Class = "bad"
|
||
} else if last.LastOutcome == "health_failed" || last.LastOutcome == "failed" || last.LastOutcome == "refused" {
|
||
r.LastLeg.Class = "warn"
|
||
}
|
||
rows = append(rows, r)
|
||
}
|
||
sort.Slice(rows, func(i, j int) bool { return rows[i].HostID < rows[j].HostID })
|
||
return rows
|
||
}
|
||
|
||
func orDash(s string) string {
|
||
if s == "" {
|
||
return "—"
|
||
}
|
||
return s
|
||
}
|
||
|
||
func (s *Server) handleSystem(w http.ResponseWriter, r *http.Request) {
|
||
view, ok := s.osUpdates.(OSSystemView)
|
||
if s.osUpdates == nil || !ok {
|
||
http.Error(w, "os updates not configured", http.StatusServiceUnavailable)
|
||
return
|
||
}
|
||
lines, err := view.Fleet()
|
||
if err != nil {
|
||
s.logger.Printf("[ERROR] system page: fleet: %v", err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
hosts, _ := s.store.ListHosts()
|
||
facts, names := map[string]sysfacts.System{}, map[string]string{}
|
||
for _, h := range hosts {
|
||
names[h.HostID] = s.customerName(h.CustomerID)
|
||
if rj, _ := s.store.GetLatestHostReportJSON(h.CustomerID); rj != "" {
|
||
facts[h.HostID] = sysfacts.Parse(rj)
|
||
}
|
||
}
|
||
stale, reboot, notCov := view.Thresholds()
|
||
rows := buildSystemRows(lines, facts, names, stale, reboot, notCov, time.Now())
|
||
man := s.store.GetArtifactManifest()
|
||
agents := map[string]string{}
|
||
for _, h := range hosts {
|
||
agents[h.HostID] = h.AgentVersion
|
||
}
|
||
for i := range rows {
|
||
rows[i].KernelDefault, rows[i].KernelStep = kernelCells(facts[rows[i].HostID], view.KernelLineFor(rows[i].HostID), time.Now())
|
||
rows[i].Bundle = bundleCell(facts[rows[i].HostID], man.AgentVersion, man.BundleSHA256,
|
||
s.store.BundleBehindSince(rows[i].HostID), view.BundleThreshold(), time.Now())
|
||
rows[i].Agent = agentCell(agents[rows[i].HostID], man.AgentVersion,
|
||
s.store.AgentBehindSince(rows[i].HostID), view.AgentThreshold(), time.Now())
|
||
}
|
||
global := s.store.GetGlobalMinControllerVersion()
|
||
ovs, oerr := s.store.CustomerFloorOverrides()
|
||
if oerr != nil {
|
||
s.logger.Printf("[ERROR] system page: floor overrides: %v", oerr)
|
||
}
|
||
data := map[string]interface{}{
|
||
"Rows": rows,
|
||
"GlobalFloor": global,
|
||
"VouchedAgent": man.AgentVersion,
|
||
"Floors": buildFloorRows(ovs, global, time.Now()),
|
||
"Releases": view.Releases(),
|
||
"Cancelled": view.CancelledReleases(),
|
||
"Candidates": view.Candidates(),
|
||
"Flash": r.URL.Query().Get("flash"),
|
||
"FlashErr": r.URL.Query().Get("err"),
|
||
"CSRFToken": s.getCSRFToken(r),
|
||
}
|
||
if err := s.templates.ExecuteTemplate(w, "system.html", data); err != nil {
|
||
s.logger.Printf("[ERROR] system.html template: %v", err)
|
||
}
|
||
}
|