package web import ( "fmt" "net/http" "sort" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/osupdates" "gitea.dooplex.hu/admin/felhom-hub/internal/semver" "gitea.dooplex.hu/admin/felhom-hub/internal/store" "gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts" ) // The System page (hub v0.132.0; R-852, `09` decision 89, `11` §5.7): every box's real versions and its OS-update // state, with the operator's buttons (ring, switch, approve now, approve the Docker set). Read from the hub's own // records only: the newest host report's `system` stanza (agent ≥ v0.142.0) and the OS fleet lines. // cell is one value with its colour: "" plain, "warn" amber, "bad" red (red = an alarm would fire). type cell struct { Text string Class string Title string } type systemRow struct { HostID, CustomerName string Ring int Enabled bool Tunnel cell HasFacts bool FactsNote string // host PVE, KernelRunning, KernelNextBoot, HostDebian cell HostRelease, HostPending, HostNotCovered cell Held, RebootSince, KernelPanic, Oops cell CrashRestarts24h, Guard cell Bundle cell // R-840: the root-owned config bundle Agent cell // R-530: the box's agent against the vouched one // guest GuestDebian, GuestRelease, GuestPending, GuestRestart cell // docker Engine, Containerd, LiveRestore, DockerRelease cell // last leg LastLeg cell } // OSSystemView is what the System page needs beyond OSUpdateAdmin (implemented by *osupdates.Service). type OSSystemView interface { Fleet() ([]osupdates.FleetLine, error) Releases() []osupdates.ReleaseInfo CancelledReleases() []osupdates.ReleaseInfo Candidates() []osupdates.Status Thresholds() (stale, reboot, notCovered time.Duration) BundleThreshold() time.Duration AgentThreshold() time.Duration ApproveDocker() (string, error) } // agentCell is the "Agent" cell (R-530, hub v0.135.0): the box's agent against the vouched one, how far behind and // since when. Amber while behind; red from the alarm's wait on (the operator alarm fires then). An unreadable // version is "unknown", never a guess; nothing vouched → the version alone. func agentCell(boxAgent, vouched string, since time.Time, after time.Duration, now time.Time) cell { if !semver.Valid(boxAgent) { return unknownCell("") } c := cell{Text: boxAgent} if !semver.Valid(vouched) { c.Title = "no vouched agent to compare with" return c } if semver.Compare(boxAgent, vouched) >= 0 { c.Title = "current (vouched " + vouched + ")" return c } c.Class = "warn" c.Text = boxAgent + " → " + vouched c.Title = osupdates.ReleasesBehind(boxAgent, vouched) + " — sign an agent_update for this box" if !since.IsZero() { c.Text += " (since " + since.UTC().Format("2006-01-02") + ")" if now.Sub(since) >= after { c.Class = "bad" } } return c } // floorRow is one line of the System page's "Version floors" table (R-604). type floorRow struct { CustomerID, CustomerName, Version string Age cell HeldBack bool // the override is BELOW the global floor: the global does not move this box } func buildFloorRows(ovs []store.CustomerFloorOverride, global string, now time.Time) []floorRow { var out []floorRow for _, o := range ovs { r := floorRow{CustomerID: o.CustomerID, CustomerName: o.CustomerName, Version: o.Version} if o.SetAt.IsZero() { r.Age = cell{Text: "unknown", Class: "warn", Title: "set before hub v0.135.0 — the hub did not record when"} } else { r.Age = plain(ago(o.SetAt, now) + " (" + o.SetAt.UTC().Format("2006-01-02") + ")") } if semver.Valid(global) && semver.Valid(o.Version) && semver.Compare(global, o.Version) > 0 { r.HeldBack = true } out = append(out, r) } return out } func plain(s string) cell { return cell{Text: s} } // bundleCell is the "Root files" cell (R-840): the box's config bundle against the vouched agent's. Amber while behind, // red from the alarm's wait on, amber when a file was changed by hand (drift); "unknown" is never coloured as a fact. func bundleCell(f sysfacts.System, vouchedAgent, vouchedSHA string, since time.Time, after time.Duration, now time.Time) cell { b := f.Bundle if !f.Present || b.Version == "" || b.Version == sysfacts.Unknown { return unknownCell("") } c := cell{Text: b.Version} switch { case vouchedSHA == "": c.Title = "no vouched bundle to compare with (the vouched agent carries none)" case b.BundleSHA256 != vouchedSHA: c.Class, c.Title = "warn", "behind the vouched agent "+vouchedAgent+"'s bundle — send it with a signed agent_config_update" if !since.IsZero() { c.Title += " (behind since " + since.UTC().Format("2006-01-02 15:04") + " UTC)" if now.Sub(since) >= after { c.Class = "bad" } } } if len(b.Drift) > 0 { c.Text += " (changed by hand)" c.Class = "warn" c.Title = "files differ from the installed bundle: " + strings.Join(b.Drift, ", ") } return c } func unknownCell(s string) cell { if s == "" || s == sysfacts.Unknown { return cell{Text: "unknown", Class: "warn", Title: "the box could not read it (agent older than v0.142.0, or the guest is down)"} } return cell{Text: s} } func ago(t time.Time, now time.Time) string { if t.IsZero() { return "never" } d := now.Sub(t) switch { case d < time.Hour: return fmt.Sprintf("%d min ago", int(d.Minutes())) case d < 48*time.Hour: return fmt.Sprintf("%d h ago", int(d.Hours())) } return fmt.Sprintf("%d days ago", int(d.Hours()/24)) } // buildSystemRows is pure (the render test feeds it directly). func buildSystemRows(lines []osupdates.FleetLine, facts map[string]sysfacts.System, names map[string]string, stale, reboot, notCov time.Duration, now time.Time) []systemRow { var rows []systemRow for _, l := range lines { f := facts[l.HostID] r := systemRow{HostID: l.HostID, CustomerName: names[l.HostID], Ring: l.Ring, Enabled: l.Enabled, HasFacts: f.Present} switch l.Tunnel { case "running": r.Tunnel = plain("running") case "not_running", "inactive": r.Tunnel = cell{Text: l.Tunnel, Class: "bad"} default: r.Tunnel = cell{Text: l.Tunnel, Class: "warn"} } if !f.Present { r.FactsNote = "no versions reported (agent older than v0.142.0)" } else if f.FactsError != "" { r.FactsNote = "partial: " + f.FactsError } r.PVE = unknownCell(sysfacts.ShortPVE(f.PVEVersion)) r.KernelRunning = unknownCell(f.Host.KernelRunning) r.KernelNextBoot = unknownCell(f.Host.KernelNextBoot) if f.NextBootDiffers() { r.KernelNextBoot.Class, r.KernelNextBoot.Title = "warn", "the next boot changes the kernel ("+f.Host.KernelNextBootSource+")" } r.HostDebian = unknownCell(f.Host.Debian) r.HostRelease = plain(orDash(l.Host.ReleaseID)) r.HostPending = plain(fmt.Sprint(l.Host.Pending)) r.HostNotCovered = plain(fmt.Sprint(l.Host.NotCoveredFast)) if l.Host.NotCoveredFast > 0 { r.HostNotCovered.Class = "warn" } switch { case f.Host.Held == nil: r.Held = unknownCell("") case len(f.Host.Held) == 0: r.Held = plain("none") default: r.Held = cell{Text: strings.Join(f.Host.Held, ", "), Class: "warn", Title: "held by hand (an undo) — the hub cannot see it otherwise (R-848)"} } if l.Host.RebootNeededSince.IsZero() { r.RebootSince = plain("no") } else { r.RebootSince = cell{Text: "since " + l.Host.RebootNeededSince.UTC().Format("2006-01-02"), Class: "warn"} if now.Sub(l.Host.RebootNeededSince) >= reboot { r.RebootSince.Class = "bad" } } if f.Host.KernelPanic != nil { r.KernelPanic = plain(fmt.Sprintf("%d s", *f.Host.KernelPanic)) if *f.Host.KernelPanic == 0 { r.KernelPanic = cell{Text: "0 (stays off)", Class: "warn"} } } else { r.KernelPanic = unknownCell("") } r.Oops = plain("no") if f.Host.OopsThisBoot != nil && *f.Host.OopsThisBoot { r.Oops = cell{Text: "yes", Class: "warn", Title: "a kernel oops this boot"} } if cg := f.Host.CrashGuard; cg != nil { r.CrashRestarts24h = plain(fmt.Sprint(cg.In24h)) if cg.In24h > 0 { r.CrashRestarts24h.Class = "warn" } if cg.Tripped { r.Guard = cell{Text: "TRIPPED " + cg.TrippedAt, Class: "bad", Title: cg.TrippedReason} } else { r.Guard = plain("armed") } } else { r.CrashRestarts24h, r.Guard = unknownCell(""), cell{Text: "not installed", Class: "warn"} } r.GuestDebian = unknownCell(f.Guest.Debian) r.GuestRelease = plain(orDash(l.Guest.ReleaseID)) r.GuestPending = plain(fmt.Sprint(l.Guest.Pending)) r.GuestRestart = plain(fmt.Sprint(l.Guest.RestartNeeded)) r.Engine, r.Containerd = unknownCell(f.Guest.DockerEngine), unknownCell(f.Guest.Containerd) r.LiveRestore = unknownCell(f.Guest.LiveRestore) if f.Guest.LiveRestore == "off" { r.LiveRestore.Class, r.LiveRestore.Title = "warn", "a Docker step is refused until it is on (decision 87)" } r.DockerRelease = plain(orDash(l.Docker.ReleaseID)) last := l.Guest if l.Host.LastAt.After(last.LastAt) { last = l.Host } if l.Docker.LastAt.After(last.LastAt) { last = l.Docker } ok := l.Guest.LastSuccessfulLeg r.LastLeg = cell{Text: fmt.Sprintf("%s · %s · %.0f s", ago(last.LastAt, now), orDash(last.LastOutcome), last.WrapperPassSeconds), Title: "last successful leg: " + ago(ok, now)} if l.Enabled && !ok.IsZero() && now.Sub(ok) >= stale { r.LastLeg.Class = "bad" } else if last.LastOutcome == "health_failed" || last.LastOutcome == "failed" || last.LastOutcome == "refused" { r.LastLeg.Class = "warn" } rows = append(rows, r) } sort.Slice(rows, func(i, j int) bool { return rows[i].HostID < rows[j].HostID }) return rows } func orDash(s string) string { if s == "" { return "—" } return s } func (s *Server) handleSystem(w http.ResponseWriter, r *http.Request) { view, ok := s.osUpdates.(OSSystemView) if s.osUpdates == nil || !ok { http.Error(w, "os updates not configured", http.StatusServiceUnavailable) return } lines, err := view.Fleet() if err != nil { s.logger.Printf("[ERROR] system page: fleet: %v", err) http.Error(w, "Internal error", http.StatusInternalServerError) return } hosts, _ := s.store.ListHosts() facts, names := map[string]sysfacts.System{}, map[string]string{} for _, h := range hosts { names[h.HostID] = s.customerName(h.CustomerID) if rj, _ := s.store.GetLatestHostReportJSON(h.CustomerID); rj != "" { facts[h.HostID] = sysfacts.Parse(rj) } } stale, reboot, notCov := view.Thresholds() rows := buildSystemRows(lines, facts, names, stale, reboot, notCov, time.Now()) man := s.store.GetArtifactManifest() agents := map[string]string{} for _, h := range hosts { agents[h.HostID] = h.AgentVersion } for i := range rows { rows[i].Bundle = bundleCell(facts[rows[i].HostID], man.AgentVersion, man.BundleSHA256, s.store.BundleBehindSince(rows[i].HostID), view.BundleThreshold(), time.Now()) rows[i].Agent = agentCell(agents[rows[i].HostID], man.AgentVersion, s.store.AgentBehindSince(rows[i].HostID), view.AgentThreshold(), time.Now()) } global := s.store.GetGlobalMinControllerVersion() ovs, oerr := s.store.CustomerFloorOverrides() if oerr != nil { s.logger.Printf("[ERROR] system page: floor overrides: %v", oerr) } data := map[string]interface{}{ "Rows": rows, "GlobalFloor": global, "VouchedAgent": man.AgentVersion, "Floors": buildFloorRows(ovs, global, time.Now()), "Releases": view.Releases(), "Cancelled": view.CancelledReleases(), "Candidates": view.Candidates(), "Flash": r.URL.Query().Get("flash"), "FlashErr": r.URL.Query().Get("err"), "CSRFToken": s.getCSRFToken(r), } if err := s.templates.ExecuteTemplate(w, "system.html", data); err != nil { s.logger.Printf("[ERROR] system.html template: %v", err) } }