package web import ( "fmt" "net/http" "sort" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/osupdates" "gitea.dooplex.hu/admin/felhom-hub/internal/semver" "gitea.dooplex.hu/admin/felhom-hub/internal/store" "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" ) // The System page (hub v0.132.0; R-852, `09` decision 89, `11` §5.7): every box's real versions and its OS-update // state, with the operator's buttons (ring, switch, approve now, approve the Docker set). Read from the hub's own // records only: the newest host report's `system` stanza (agent ≥ v0.142.0) and the OS fleet lines. // cell is one value with its colour: "" plain, "warn" amber, "bad" red (red = an alarm would fire). type cell struct { Text string Class string Title string } type systemRow struct { HostID, CustomerName string Ring int Enabled bool Tunnel cell HasFacts bool FactsNote string // host PVE, KernelRunning, KernelNextBoot, HostDebian cell HostRelease, HostPending, HostNotCovered cell Held, RebootSince, KernelPanic, Oops cell CrashRestarts24h, Guard cell KernelDefault, KernelStep cell // R-836: the GRUB default kernel; the kernel lane's step Bundle cell // R-840: the root-owned config bundle Agent cell // R-530: the box's agent against the vouched one // guest GuestDebian, GuestRelease, GuestPending, GuestRestart cell Trim cell // R-444: the last `pct fstrim` of the guest // docker Engine, Containerd, LiveRestore, DockerRelease cell // last leg LastLeg cell } // OSSystemView is what the System page needs beyond OSUpdateAdmin (implemented by *osupdates.Service). type OSSystemView interface { Fleet() ([]osupdates.FleetLine, error) Releases() []osupdates.ReleaseInfo CancelledReleases() []osupdates.ReleaseInfo Candidates() []osupdates.Status Thresholds() (stale, reboot, notCovered time.Duration) BundleThreshold() time.Duration AgentThreshold() time.Duration ApproveDocker() (string, error) ApprovePVE() (string, error) // R-812 option A: the Proxmox package set ApproveKernel() (string, error) // R-836: the kernel set KernelLineFor(hostID string) osupdates.KernelLine // R-836: one box's kernel step and day-before mail } // kernelCells are the System page's two kernel-lane cells (R-836, `11` §5.11): the GRUB default (amber while a one-shot // flag names another kernel for the next boot), and the step — the newest result, the kernel the box is due, whether // the household was told for tonight. "unknown" when the box reports no kernel lane (an older agent). func kernelCells(f sysfacts.System, kl osupdates.KernelLine, now time.Time) (dflt, step cell) { lane := f.Host.KernelLane if lane == nil { if kl.LastOutcome == "" { return unknownCell(""), cell{Text: "—", Title: "the box reports no kernel lane (agent older than v0.152.0)"} } lane = &sysfacts.KernelLane{} // a step result without facts (an older facts read) still shows } dflt = unknownCell(lane.Default) if lane.Flag != nil && *lane.Flag != "" { dflt.Text += " (once: " + *lane.Flag + ")" dflt.Class, dflt.Title = "warn", "the next boot runs "+*lane.Flag+" ONCE; the boot after it the default again" } var parts []string if kl.LastOutcome != "" { parts = append(parts, fmt.Sprintf("%s %s %s", kl.LastKernel, kl.LastOutcome, ago(kl.LastAt, now))) } switch { case kl.Due != "" && kl.Tonight: parts = append(parts, "due "+kl.Due+" — household told "+ago(kl.NoticeAt, now)+": TONIGHT") case kl.Due != "": parts = append(parts, "due "+kl.Due+" — not told yet (mails 09–20 h)") } if lane.Phase != "" && lane.Phase != "none" { parts = append(parts, "phase "+lane.Phase) } if len(parts) == 0 { parts = append(parts, "—") } step = cell{Text: strings.Join(parts, " · "), Title: strings.TrimSpace(kl.Why + " " + kl.LastReason)} switch kl.LastOutcome { case "fell_back", "health_failed", "self_reverted", "refused", "failed": step.Class = "warn" case "revert_failed": step.Class = "bad" } if len(lane.SetupProblems) > 0 { step.Class = "warn" step.Title = "cannot do a one-shot boot: " + strings.Join(lane.SetupProblems, "; ") } return dflt, step } // agentCell is the "Agent" cell (R-530, hub v0.135.0): the box's agent against the vouched one, how far behind and // since when. Amber while behind; red from the alarm's wait on (the operator alarm fires then). An unreadable // version is "unknown", never a guess; nothing vouched → the version alone. func agentCell(boxAgent, vouched string, since time.Time, after time.Duration, now time.Time) cell { if !semver.Valid(boxAgent) { return unknownCell("") } c := cell{Text: boxAgent} if !semver.Valid(vouched) { c.Title = "no vouched agent to compare with" return c } if semver.Compare(boxAgent, vouched) >= 0 { c.Title = "current (vouched " + vouched + ")" return c } c.Class = "warn" c.Text = boxAgent + " → " + vouched c.Title = osupdates.ReleasesBehind(boxAgent, vouched) + " — sign an agent_update for this box" if !since.IsZero() { c.Text += " (since " + since.UTC().Format("2006-01-02") + ")" if now.Sub(since) >= after { c.Class = "bad" } } return c } // floorRow is one line of the System page's "Version floors" table (R-604). type floorRow struct { CustomerID, CustomerName, Version string Age cell HeldBack bool // the override is BELOW the global floor: the global does not move this box } func buildFloorRows(ovs []store.CustomerFloorOverride, global string, now time.Time) []floorRow { var out []floorRow for _, o := range ovs { r := floorRow{CustomerID: o.CustomerID, CustomerName: o.CustomerName, Version: o.Version} if o.SetAt.IsZero() { r.Age = cell{Text: "unknown", Class: "warn", Title: "set before hub v0.135.0 — the hub did not record when"} } else { r.Age = plain(ago(o.SetAt, now) + " (" + o.SetAt.UTC().Format("2006-01-02") + ")") } if semver.Valid(global) && semver.Valid(o.Version) && semver.Compare(global, o.Version) > 0 { r.HeldBack = true } out = append(out, r) } return out } func plain(s string) cell { return cell{Text: s} } // trimStaleAfter is when a guest's last SUCCESSFUL disk trim reads amber: the boxes trim weekly (`09` §3 decision // 139), so two missed weeks are worth a look. const trimStaleAfter = 14 * 24 * time.Hour // trimCell is the "Last disk trim" cell (R-444): per guest, the last successful `pct fstrim` (from `last_ok_at`) and what // the newest attempt freed. Amber when the newest attempt failed (its error is shown) or the last success is older than // 14 days — judged on `last_ok_at`, never on the attempt time (an attempt is not a result). "—" when the agent sends no // stanza (no trimmer); an unreadable time is "unknown", never a guess. func trimCell(dt *sysfacts.DiskTrim, now time.Time) cell { if dt == nil { return cell{Text: "—", Title: "the agent reports no disk trim (an agent without the weekly trim)"} } if len(dt.Guests) == 0 { return cell{Text: "none yet", Title: "the weekly trim runs (" + dt.Schedule + ") and has not trimmed a guest yet"} } out := cell{Title: "schedule: " + dt.Schedule} var texts []string for _, g := range dt.Guests { t, warn, why := guestTrimText(g, now) if len(dt.Guests) > 1 { t = fmt.Sprintf("%d: %s", g.VMID, t) } texts = append(texts, t) if warn { out.Class = "warn" out.Title = fmt.Sprintf("guest %d: %s — %s", g.VMID, why, out.Title) } } out.Text = strings.Join(texts, " / ") return out } func guestTrimText(g sysfacts.GuestTrim, now time.Time) (text string, warn bool, why string) { var okAt time.Time if g.LastOKAt != "" { t, err := time.Parse(time.RFC3339, g.LastOKAt) if err != nil { return "unknown", true, fmt.Sprintf("unreadable last_ok_at %q", g.LastOKAt) } okAt = t } attAt, aerr := time.Parse(time.RFC3339, g.LastAttemptAt) if aerr != nil { return "unknown", true, fmt.Sprintf("unreadable last_attempt_at %q", g.LastAttemptAt) } if g.OK { text = fmt.Sprintf("%s · %.1f GiB", ago(attAt, now), float64(g.BytesTrimmed)/(1<<30)) if !okAt.IsZero() { text = fmt.Sprintf("%s · %.1f GiB", ago(okAt, now), float64(g.BytesTrimmed)/(1<<30)) } else { okAt = attAt } } else { last := "never ok" if !okAt.IsZero() { last = "last ok " + ago(okAt, now) } errText := g.Error if errText == "" { errText = "no error text" } text = fmt.Sprintf("FAILED %s: %s (%s)", ago(attAt, now), errText, last) warn, why = true, "the newest trim failed" } if !okAt.IsZero() && now.Sub(okAt) > trimStaleAfter { warn = true why = strings.TrimPrefix(why+"; ", "; ") + "the last successful trim is older than 14 days (the boxes trim weekly)" } return text, warn, why } // bundleCell is the "Root files" cell (R-840): the box's config bundle against the vouched agent's. Amber while behind, // red from the alarm's wait on, amber when a file was changed by hand (drift); "unknown" is never coloured as a fact. func bundleCell(f sysfacts.System, vouchedAgent, vouchedSHA string, since time.Time, after time.Duration, now time.Time) cell { b := f.Bundle if !f.Present || b.Version == "" || b.Version == sysfacts.Unknown { return unknownCell("") } c := cell{Text: b.Version} switch { case vouchedSHA == "": c.Title = "no vouched bundle to compare with (the vouched agent carries none)" case b.BundleSHA256 != vouchedSHA: c.Class, c.Title = "warn", "behind the vouched agent "+vouchedAgent+"'s bundle — send it with a signed agent_config_update" if !since.IsZero() { c.Title += " (behind since " + since.UTC().Format("2006-01-02 15:04") + " UTC)" if now.Sub(since) >= after { c.Class = "bad" } } } if len(b.Drift) > 0 { c.Text += " (changed by hand)" c.Class = "warn" c.Title = "files differ from the installed bundle: " + strings.Join(b.Drift, ", ") } return c } func unknownCell(s string) cell { if s == "" || s == sysfacts.Unknown { return cell{Text: "unknown", Class: "warn", Title: "the box could not read it (agent older than v0.142.0, or the guest is down)"} } return cell{Text: s} } func ago(t time.Time, now time.Time) string { if t.IsZero() { return "never" } d := now.Sub(t) switch { case d < time.Hour: return fmt.Sprintf("%d min ago", int(d.Minutes())) case d < 48*time.Hour: return fmt.Sprintf("%d h ago", int(d.Hours())) } return fmt.Sprintf("%d days ago", int(d.Hours()/24)) } // buildSystemRows is pure (the render test feeds it directly). func buildSystemRows(lines []osupdates.FleetLine, facts map[string]sysfacts.System, names map[string]string, stale, reboot, notCov time.Duration, now time.Time) []systemRow { var rows []systemRow for _, l := range lines { f := facts[l.HostID] r := systemRow{HostID: l.HostID, CustomerName: names[l.HostID], Ring: l.Ring, Enabled: l.Enabled, HasFacts: f.Present} switch l.Tunnel { case "running": r.Tunnel = plain("running") case "not_running", "inactive": r.Tunnel = cell{Text: l.Tunnel, Class: "bad"} default: r.Tunnel = cell{Text: l.Tunnel, Class: "warn"} } if !f.Present { r.FactsNote = "no versions reported (agent older than v0.142.0)" } else if f.FactsError != "" { r.FactsNote = "partial: " + f.FactsError } r.PVE = unknownCell(sysfacts.ShortPVE(f.PVEVersion)) r.KernelRunning = unknownCell(f.Host.KernelRunning) r.KernelNextBoot = unknownCell(f.Host.KernelNextBoot) if f.NextBootDiffers() { r.KernelNextBoot.Class, r.KernelNextBoot.Title = "warn", "the next boot changes the kernel ("+f.Host.KernelNextBootSource+")" } r.HostDebian = unknownCell(f.Host.Debian) r.HostRelease = plain(orDash(l.Host.ReleaseID)) r.HostPending = plain(fmt.Sprint(l.Host.Pending)) r.HostNotCovered = plain(fmt.Sprint(l.Host.NotCoveredFast)) if l.Host.NotCoveredFast > 0 { r.HostNotCovered.Class = "warn" } switch { case f.Host.Held == nil: r.Held = unknownCell("") case len(f.Host.Held) == 0: r.Held = plain("none") default: r.Held = cell{Text: strings.Join(f.Host.Held, ", "), Class: "warn", Title: "held by hand (an undo) — the hub cannot see it otherwise (R-848)"} } if l.Host.RebootNeededSince.IsZero() { r.RebootSince = plain("no") } else { r.RebootSince = cell{Text: "since " + l.Host.RebootNeededSince.UTC().Format("2006-01-02"), Class: "warn"} if now.Sub(l.Host.RebootNeededSince) >= reboot { r.RebootSince.Class = "bad" } } if f.Host.KernelPanic != nil { r.KernelPanic = plain(fmt.Sprintf("%d s", *f.Host.KernelPanic)) if *f.Host.KernelPanic == 0 { r.KernelPanic = cell{Text: "0 (stays off)", Class: "warn"} } } else { r.KernelPanic = unknownCell("") } r.Oops = plain("no") if f.Host.OopsThisBoot != nil && *f.Host.OopsThisBoot { r.Oops = cell{Text: "yes", Class: "warn", Title: "a kernel oops this boot"} } if cg := f.Host.CrashGuard; cg != nil { r.CrashRestarts24h = plain(fmt.Sprint(cg.In24h)) if cg.In24h > 0 { r.CrashRestarts24h.Class = "warn" } if cg.Tripped { r.Guard = cell{Text: "TRIPPED " + cg.TrippedAt, Class: "bad", Title: cg.TrippedReason} } else { r.Guard = plain("armed") } } else { r.CrashRestarts24h, r.Guard = unknownCell(""), cell{Text: "not installed", Class: "warn"} } r.GuestDebian = unknownCell(f.Guest.Debian) r.GuestRelease = plain(orDash(l.Guest.ReleaseID)) r.GuestPending = plain(fmt.Sprint(l.Guest.Pending)) r.GuestRestart = plain(fmt.Sprint(l.Guest.RestartNeeded)) r.Trim = trimCell(f.DiskTrim, now) r.Engine, r.Containerd = unknownCell(f.Guest.DockerEngine), unknownCell(f.Guest.Containerd) r.LiveRestore = unknownCell(f.Guest.LiveRestore) if f.Guest.LiveRestore == "off" { r.LiveRestore.Class, r.LiveRestore.Title = "warn", "a Docker step is refused until it is on (decision 87)" } r.DockerRelease = plain(orDash(l.Docker.ReleaseID)) last := l.Guest if l.Host.LastAt.After(last.LastAt) { last = l.Host } if l.Docker.LastAt.After(last.LastAt) { last = l.Docker } ok := l.Guest.LastSuccessfulLeg r.LastLeg = cell{Text: fmt.Sprintf("%s · %s · %.0f s", ago(last.LastAt, now), orDash(last.LastOutcome), last.WrapperPassSeconds), Title: "last successful leg: " + ago(ok, now)} if l.Enabled && !ok.IsZero() && now.Sub(ok) >= stale { r.LastLeg.Class = "bad" } else if last.LastOutcome == "health_failed" || last.LastOutcome == "failed" || last.LastOutcome == "refused" { r.LastLeg.Class = "warn" } rows = append(rows, r) } sort.Slice(rows, func(i, j int) bool { return rows[i].HostID < rows[j].HostID }) return rows } func orDash(s string) string { if s == "" { return "—" } return s } func (s *Server) handleSystem(w http.ResponseWriter, r *http.Request) { view, ok := s.osUpdates.(OSSystemView) if s.osUpdates == nil || !ok { http.Error(w, "os updates not configured", http.StatusServiceUnavailable) return } lines, err := view.Fleet() if err != nil { s.logger.Printf("[ERROR] system page: fleet: %v", err) http.Error(w, "Internal error", http.StatusInternalServerError) return } hosts, _ := s.store.ListHosts() facts, names := map[string]sysfacts.System{}, map[string]string{} for _, h := range hosts { names[h.HostID] = s.customerName(h.CustomerID) if rj, _ := s.store.GetLatestHostReportJSON(h.CustomerID); rj != "" { facts[h.HostID] = sysfacts.Parse(rj) } } stale, reboot, notCov := view.Thresholds() rows := buildSystemRows(lines, facts, names, stale, reboot, notCov, time.Now()) man := s.store.GetArtifactManifest() agents := map[string]string{} for _, h := range hosts { agents[h.HostID] = h.AgentVersion } for i := range rows { rows[i].KernelDefault, rows[i].KernelStep = kernelCells(facts[rows[i].HostID], view.KernelLineFor(rows[i].HostID), time.Now()) rows[i].Bundle = bundleCell(facts[rows[i].HostID], man.AgentVersion, man.BundleSHA256, s.store.BundleBehindSince(rows[i].HostID), view.BundleThreshold(), time.Now()) rows[i].Agent = agentCell(agents[rows[i].HostID], man.AgentVersion, s.store.AgentBehindSince(rows[i].HostID), view.AgentThreshold(), time.Now()) } global := s.store.GetGlobalMinControllerVersion() ovs, oerr := s.store.CustomerFloorOverrides() if oerr != nil { s.logger.Printf("[ERROR] system page: floor overrides: %v", oerr) } data := map[string]interface{}{ "Rows": rows, "GlobalFloor": global, "VouchedAgent": man.AgentVersion, "Floors": buildFloorRows(ovs, global, time.Now()), "Releases": view.Releases(), "Cancelled": view.CancelledReleases(), "Candidates": view.Candidates(), "Flash": r.URL.Query().Get("flash"), "FlashErr": r.URL.Query().Get("err"), "CSRFToken": s.getCSRFToken(r), } if err := s.templates.ExecuteTemplate(w, "system.html", data); err != nil { s.logger.Printf("[ERROR] system.html template: %v", err) } }