hub v0.132.0: the System page (versions + OS updates with the ring/switch/approve buttons, R-852), the Hosts Proxmox/kernel column, the operator-approved Docker engine release (2 healthy ring-0 nights), the crash-guard events (R-851); evidence audits/os-docker-crash-2026-10-04
gates / gates (push) Successful in 30s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-04 16:16:02 +02:00
parent bee277ffad
commit 175ecfcdd2
45 changed files with 2198 additions and 12 deletions
+104
View File
@@ -0,0 +1,104 @@
package api
import (
"encoding/json"
"fmt"
"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
)
// The crash-restart events (`09` decision 88, R-851, `11` §5.9, hub v0.132.0), read from the box's crash guard in the
// host report's `system` stanza. An unclean boot cannot be told from a power cut on these boxes (measured), so the
// texts say "an unexpected stop".
const (
EventHostCrashRestart = "host_crash_restart" // warning, operator: the box restarted after an unclean stop
EventHostRestartedAfterCrash = "host_restarted_after_crash" // info, the HOUSEHOLD's one line (never mailed)
EventCrashGuardTripped = "host_crash_guard_tripped" // error, operator: the next crash leaves the box off
EventCrashGuardRearmed = "host_crash_guard_rearmed" // info, operator
EventKernelOops = "host_kernel_oops" // warning, operator: a kernel oops this boot (no restart)
)
type crashSeen struct {
Boots []string `json:"boots"`
TrippedAt string `json:"tripped_at"`
RearmedAt string `json:"rearmed_at"`
OopsBoot string `json:"oops_boot"`
}
// checkCrash turns NEW crash-guard facts into events. The first report the hub sees from a box only records what is
// there (no flood of old history). Pinned by crash_test.go.
func (h *Handler) checkCrash(hostID, custID string, body []byte) {
sys := sysfacts.Parse(string(body))
cg := sys.Host.CrashGuard
if !sys.Present || cg == nil {
return
}
var seen crashSeen
raw := h.store.CrashSeen(hostID)
first := raw == ""
if !first {
_ = json.Unmarshal([]byte(raw), &seen)
}
known := map[string]bool{}
for _, b := range seen.Boots {
known[b] = true
}
emit := func(typ, sev, msg string, extra map[string]any) {
if first {
return
}
d := map[string]any{"host_id": hostID}
for k, v := range extra {
d[k] = v
}
dj, _ := json.Marshal(d)
h.logger.Printf("[WARN] host %s crash guard: %s", hostID, typ)
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
} else if h.dispatcher != nil {
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
}
}
for _, b := range cg.UncleanBoots {
if known[b] {
continue
}
emit(EventHostCrashRestart, "warning", fmt.Sprintf("%s restarted by itself after an unexpected stop (a kernel crash, a power cut or a hard reset) at %s — %d such restart(s) in 24 h. kernel.panic now %s.",
hostID, b, cg.In24h, intOr(cg.KernelPanic, "unknown")), map[string]any{"boot_at": b, "in_24h": cg.In24h})
emit(EventHostRestartedAfterCrash, "info", "Your box restarted by itself after an unexpected stop. You do not need to do anything.",
map[string]any{"boot_at": b})
seen.Boots = append(seen.Boots, b)
}
if cg.Tripped && cg.TrippedAt != "" && cg.TrippedAt != seen.TrippedAt {
emit(EventCrashGuardTripped, "error", fmt.Sprintf("The crash guard of %s TRIPPED at %s: %s. The next crash leaves the box OFF until someone switches it on. Re-arm: `felhom-crash-guard rearm` on the host, or wait 24 h of normal running.",
hostID, cg.TrippedAt, cg.TrippedReason), map[string]any{"tripped_at": cg.TrippedAt})
seen.TrippedAt = cg.TrippedAt
}
if cg.RearmedAt != "" && cg.RearmedAt != seen.RearmedAt {
emit(EventCrashGuardRearmed, "info", fmt.Sprintf("The crash guard of %s is armed again (%s, by %s).", hostID, cg.RearmedAt, cg.RearmedBy),
map[string]any{"rearmed_at": cg.RearmedAt})
seen.RearmedAt = cg.RearmedAt
}
if sys.Host.OopsThisBoot != nil && *sys.Host.OopsThisBoot && cg.BootID != "" && cg.BootID != seen.OopsBoot {
emit(EventKernelOops, "warning", fmt.Sprintf("%s logged a kernel oops this boot (an error the kernel survived). The box keeps running; read `journalctl -k -b` on the host.", hostID),
map[string]any{"boot_id": cg.BootID})
seen.OopsBoot = cg.BootID
}
if first {
seen.Boots, seen.TrippedAt, seen.RearmedAt = append([]string{}, cg.UncleanBoots...), cg.TrippedAt, cg.RearmedAt
}
if len(seen.Boots) > 200 {
seen.Boots = seen.Boots[len(seen.Boots)-200:]
}
sj, _ := json.Marshal(seen)
if err := h.store.SetCrashSeen(hostID, string(sj)); err != nil {
h.logger.Printf("[WARN] crash_seen save failed for %s: %v", hostID, err)
}
}
func intOr(p *int, d string) string {
if p == nil {
return d
}
return fmt.Sprint(*p)
}
+67
View File
@@ -0,0 +1,67 @@
package api
import (
"fmt"
"strings"
"testing"
"time"
)
func crashBody(boots []string, tripped bool, trippedAt, rearmedAt, bootID string, oops bool) []byte {
q := func(ss []string) string {
if len(ss) == 0 {
return "[]"
}
return `["` + strings.Join(ss, `","`) + `"]`
}
return []byte(fmt.Sprintf(`{"system":{"pve_version":"pve-manager/9.0.11/x","facts":{"host":{"oops_this_boot":%v,"kernel_panic":10,
"crash_guard":{"unclean_boots":%s,"unclean_boots_24h":%d,"tripped":%v,"tripped_at":%q,"tripped_reason":"2 unclean boots within 60 minutes",
"rearmed_at":%q,"rearmed_by":"operator","boot_id":%q}},"guest":{}}}}`, oops, q(boots), len(boots), tripped, trippedAt, rearmedAt, bootID))
}
func countEv(t *testing.T, h *Handler, cust, typ string) int {
t.Helper()
ev, err := h.store.GetEventsByType(cust, typ, time.Now().Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
return len(ev)
}
// The first report only records history (no flood); each NEW unclean boot is one operator event + one household line;
// the trip is one alarm; a repeat report is quiet. Red-proof: drop the `known[b]` skip and the repeat report emits again.
func TestCheckCrash_EventsOncePerFact(t *testing.T) {
h, st, _ := newTestHandler(t)
seedHost(t, st, "h1", "c1", "K1")
old := []string{"2026-10-01T10:00:00Z"}
h.checkCrash("h1", "c1", crashBody(old, false, "", "", "b0", false))
if n := countEv(t, h, "c1", EventHostCrashRestart); n != 0 {
t.Fatalf("first sighting flooded %d events", n)
}
two := append(old, "2026-10-04T15:40:00Z")
h.checkCrash("h1", "c1", crashBody(two, false, "", "", "b1", false))
h.checkCrash("h1", "c1", crashBody(two, false, "", "", "b1", false)) // the same report again
if countEv(t, h, "c1", EventHostCrashRestart) != 1 || countEv(t, h, "c1", EventHostRestartedAfterCrash) != 1 {
t.Fatalf("restart events: op=%d household=%d", countEv(t, h, "c1", EventHostCrashRestart), countEv(t, h, "c1", EventHostRestartedAfterCrash))
}
three := append(two, "2026-10-04T15:45:00Z")
h.checkCrash("h1", "c1", crashBody(three, true, "2026-10-04T15:45:30Z", "", "b2", false))
h.checkCrash("h1", "c1", crashBody(three, true, "2026-10-04T15:45:30Z", "", "b2", false))
if countEv(t, h, "c1", EventCrashGuardTripped) != 1 {
t.Fatalf("tripped events = %d", countEv(t, h, "c1", EventCrashGuardTripped))
}
h.checkCrash("h1", "c1", crashBody(three, false, "", "2026-10-04T16:10:00Z", "b2", true))
if countEv(t, h, "c1", EventCrashGuardRearmed) != 1 || countEv(t, h, "c1", EventKernelOops) != 1 {
t.Fatal("re-arm / oops not announced")
}
}
// An agent older than v0.142.0 (no system stanza) never produces a crash event.
func TestCheckCrash_NoStanzaIsQuiet(t *testing.T) {
h, st, _ := newTestHandler(t)
seedHost(t, st, "h1", "c1", "K1")
h.checkCrash("h1", "c1", []byte(`{"host":{}}`))
if st.CrashSeen("h1") != "" {
t.Fatal("a report without the stanza recorded crash state")
}
}
+1
View File
@@ -863,6 +863,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
}
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
h.checkCrash(hostID, custID, body)
for _, g := range rep.Guests {
status := g.Status
+1
View File
@@ -87,5 +87,6 @@
"bind.resend.button": "Send me a new link",
"bind.resent.lead": "Done.",
"bind.resent.body": "If this was a real link and your box is not linked yet, a new e-mail reaches your registered address within a few minutes. If it does not, contact support.",
"mail.event.host_restarted_after_crash": "Your box restarted by itself after an unexpected stop. You do not need to do anything.",
"mail.event.os_update_applied": "System security fixes were installed on your box. You do not need to do anything."
}
+1
View File
@@ -87,5 +87,6 @@
"bind.resend.button": "Új linket kérek",
"bind.resent.lead": "Kész.",
"bind.resent.body": "Ha ez egy valódi hivatkozás volt, és a dobozod még nincs összekötve, néhány percen belül új e-mailt kapsz a regisztrált címedre. Ha nem jön, szólj az ügyfélszolgálatnak.",
"mail.event.host_restarted_after_crash": "A dobozod egy váratlan leállás után magától újraindult. Ehhez nem kell semmit tenned.",
"mail.event.os_update_applied": "Rendszerbiztonsági javítások telepítve a dobozodra. Ehhez nem kell semmit tenned."
}
+5
View File
@@ -695,6 +695,11 @@ var operatorOnlyEvents = map[string]bool{
"os_reboot_needed": true,
"os_ring0_stalled": true,
"os_not_covered": true,
// R-851 (hub v0.132.0): the crash guard. host_restarted_after_crash is deliberately NOT here — the household's line.
"host_crash_restart": true,
"host_crash_guard_tripped": true,
"host_crash_guard_rearmed": true,
"host_kernel_oops": true,
// R-197 (v0.93.0). "The sealed offsite repository key changed" is a custody fact about escrow
// blobs. A customer can take no action on it — the remedy is the operator's inspection of the
// off-site tier — and the text is operator-grade English naming host ids and retained-blob
@@ -12,11 +12,15 @@ import (
func TestOSUpdateEvents_OperatorOnlyExceptApplied(t *testing.T) {
for _, e := range []string{osupdates.EventFailed, osupdates.EventHealthFailed, osupdates.EventReleaseApprove,
osupdates.EventApprovedNow, osupdates.EventSettings, osupdates.EventStale, osupdates.EventRebootNeeded,
osupdates.EventRing0Stalled, osupdates.EventNotCovered, "tunnel_down", "tunnel_recovered"} {
osupdates.EventRing0Stalled, osupdates.EventNotCovered, "tunnel_down", "tunnel_recovered",
"host_crash_restart", "host_crash_guard_tripped", "host_crash_guard_rearmed", "host_kernel_oops"} {
if !operatorOnlyEvents[e] {
t.Errorf("%s is not operator-only", e)
}
}
if operatorOnlyEvents["host_restarted_after_crash"] {
t.Errorf("host_restarted_after_crash is the household's line and must NOT be operator-only")
}
if operatorOnlyEvents[osupdates.EventApplied] {
t.Errorf("%s is the household's line and must NOT be operator-only", osupdates.EventApplied)
}
@@ -0,0 +1,18 @@
SUBJECT: [Felhom] Warning: Your box restarted by itself after an unexpected stop. You do not need to do anything.
---
Dear Customer,
Your Felhom system sent the following notification:
Your box restarted by itself after an unexpected stop. You do not need to do anything.
Details:
- Server: demo-fixture
- Time: 2026-01-15 10:30
- Level: Warning
- Type: host_restarted_after_crash
If you have any questions, contact your operator.
Best regards,
Felhom.eu monitoring
@@ -0,0 +1,18 @@
SUBJECT: [Felhom] Figyelmeztetés: A dobozod egy váratlan leállás után magától újraindult. Ehhez nem kell semmit tenned.
---
Kedves Ügyfél!
A Felhom rendszered a következő értesítést küldte:
A dobozod egy váratlan leállás után magától újraindult. Ehhez nem kell semmit tenned.
Részletek:
- Szerver: demo-fixture
- Időpont: 2026-01-15 10:30
- Szint: Figyelmeztetés
- Típus: host_restarted_after_crash
Ha kérdésed van, vedd fel a kapcsolatot az üzemeltetővel.
Üdvözlettel,
Felhom.eu monitoring
+79
View File
@@ -0,0 +1,79 @@
package osupdates
import (
"strings"
"testing"
"time"
)
var engineSet = []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker"},
{Name: "containerd.io", Version: "2.3.6-1~debian.13~trixie", Origin: "Docker"}}
func (f *fix) dockerNight(t *testing.T, host string, healthy bool) {
t.Helper()
out := "nothing"
if !healthy {
out = "health_failed"
}
f.ingest(t, host, Report{Layer: LayerDocker, Trigger: "night", Mode: "apply", Outcome: out, Healthy: healthy,
Installed: append([]Package{pk("libc6", "x")}, engineSet...)})
}
// The Docker set is NEVER approved automatically (`11` §5.8: the operator approves it). Red-proof: add LayerDocker to
// Layers (the auto-approved list) and this fails.
func TestDocker_NeverAutoApproved(t *testing.T) {
f := newFix(t)
for i := 0; i < 3; i++ {
f.dockerNight(t, "hp", true)
f.dockerNight(t, "n100", true)
f.now = f.now.Add(25 * time.Hour)
f.s.Evaluate()
}
if rel, _ := f.s.Store.LatestOSRelease(LayerDocker); rel != nil {
t.Fatalf("a Docker set was auto-approved: %+v", rel)
}
}
// The operator's button works only after every ring-0 box ran the set 2 healthy nights; an unhealthy step blocks it.
// The candidate is the six engine packages only. Red-proof: compare nights against 1 instead of DockerNights and the
// one-night step approves.
func TestDocker_ApproveNeedsTwoHealthyNightsOnEveryRing0Box(t *testing.T) {
f := newFix(t)
f.dockerNight(t, "hp", true)
f.dockerNight(t, "n100", true)
if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "1 of 2") {
t.Fatalf("approved after one night: %v", err)
}
f.now = f.now.Add(24 * time.Hour)
f.dockerNight(t, "hp", true)
if _, err := f.s.ApproveDocker(); err == nil {
t.Fatal("approved while n100 had only one night")
}
f.dockerNight(t, "n100", true)
id, err := f.s.ApproveDocker()
if err != nil || !strings.HasPrefix(id, "os-docker-") {
t.Fatalf("%q %v", id, err)
}
rel, _ := f.s.Store.LatestOSRelease(LayerDocker)
if rel.ApprovedBy != "operator" || !strings.Contains(rel.PackagesJSON, "docker-ce") || strings.Contains(rel.PackagesJSON, "libc6") {
t.Fatalf("release %+v", rel)
}
if len(f.bumps) != 0 {
t.Fatalf("a Docker approval must nudge no box (ring 1 takes it by a signed job): %v", f.bumps)
}
if b := f.s.DesiredBlock("cust1"); b.Release != nil || b.HostRelease != nil {
t.Fatalf("the Docker set leaked into the desired block: %+v", b)
}
}
func TestDocker_UnhealthyStepBlocksTheButton(t *testing.T) {
f := newFix(t)
f.dockerNight(t, "hp", true)
f.dockerNight(t, "n100", true)
f.now = f.now.Add(24 * time.Hour)
f.dockerNight(t, "hp", false)
f.dockerNight(t, "n100", true)
if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "health_failed") {
t.Fatalf("an unhealthy Docker step did not block: %v", err)
}
}
+168 -11
View File
@@ -33,13 +33,21 @@ import (
// Layers.
const (
LayerGuest = "guest"
LayerHost = "host"
LayerGuest = "guest"
LayerHost = "host"
LayerDocker = "docker" // the guest's Docker engine set — slow lane, OPERATOR-approved only (`11` §5.8, hub v0.132.0)
)
// Layers lists every layer the hub approves.
// Layers lists the layers the hub approves AUTOMATICALLY (the fast lane).
var Layers = []string{LayerGuest, LayerHost}
// AllLayers adds the Docker engine set (approved only by the operator's button, ApproveDocker).
var AllLayers = []string{LayerGuest, LayerHost, LayerDocker}
// dockerNames are the six packages of the Docker engine set (the agent wrapper's DOCKER_NAMES).
var dockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true,
"docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true}
// hostSlowRE mirrors the wrapper's HOST_SLOW_RE (felhom-agent configs/felhom-os-apply): kernel, boot and firmware
// packages are the host's slow lane and never enter a host release (the wrapper would refuse the whole plan, R14).
// Pinned by TestCandidate_HostLeavesOutKernelBootFirmware.
@@ -91,6 +99,9 @@ type Report struct {
RebootScanned bool `json:"reboot_scanned,omitempty"` // agent ≥ 0.141.1: the pass looked (host: every pass)
Refused json.RawMessage `json:"refused,omitempty"`
PassSeconds float64 `json:"pass_seconds,omitempty"`
DockerEngine string `json:"docker_engine,omitempty"` // docker layer (agent v0.142.0)
Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed
Undo bool `json:"undo,omitempty"` // docker layer: a signed undo
}
// PendingPkg is one update the sources offer.
@@ -145,6 +156,10 @@ type Service struct {
Emit func(customerID, eventType, severity, message, details, source string)
ApproveAfter time.Duration
NightsRequired int
// DockerNights is how many healthy ring-0 night runs of the Docker step an engine set needs before the operator's
// button may approve it (`11` §5.8: 2; 0 means 2; negative means none). OS_DOCKER_APPROVE_NIGHTS overrides it for
// a TEST only, logged.
DockerNights int
// Alarm thresholds (`11` §8.3; decided by CC unattended — operator may reverse). Zero = the default.
StaleAfter time.Duration // 7 d
RebootAfter time.Duration // 14 d
@@ -191,8 +206,9 @@ func (s *Service) event(customerID, typ, sev, msg string, details any) {
}
func layerOf(r Report) string {
if r.Layer == LayerHost {
return LayerHost
switch r.Layer {
case LayerHost, LayerDocker:
return r.Layer
}
return LayerGuest
}
@@ -215,8 +231,11 @@ func (s *Service) Ingest(hostID string, r Report) error {
details := map[string]any{"host_id": hostID, "layer": layer, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome,
"upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason}
where := "the box"
if layer == LayerHost {
switch layer {
case LayerHost:
where = "the box's base system"
case LayerDocker:
where = "the box's app engine (Docker " + r.DockerEngine + ")"
}
switch r.Outcome {
case "applied", "health_failed":
@@ -224,8 +243,11 @@ func (s *Service) Ingest(hostID string, r Report) error {
fmt.Sprintf("System security fixes installed on %s (%d package(s)).", where, len(r.Upgraded)), details)
if !r.Healthy || r.Outcome == "health_failed" {
undo := "last night's whole-guest backup is the undo (restore by hand, decision 81)"
if layer == LayerHost {
switch layer {
case LayerHost:
undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)"
case LayerDocker:
undo = "sign an os_docker_step with undo for the previous engine set (runbook `os-updates-docker-undo.md`)"
}
s.event(h.CustomerID, EventHealthFailed, "error",
fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.",
@@ -254,7 +276,11 @@ func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, e
return nil, err
}
for _, p := range r.Installed {
if p.Origin != "Debian" && p.Origin != "Debian-Security" {
if layer == LayerDocker {
if !dockerNames[p.Name] {
continue
}
} else if p.Origin != "Debian" && p.Origin != "Debian-Security" {
continue
}
if layer == LayerHost && hostSlowRE.MatchString(p.Name) {
@@ -322,6 +348,7 @@ type Status struct {
FirstSeen time.Time
Packages int
Waiting string // why not approved yet ("" = approved or nothing to do)
Approved string // the release id when this set is already approved
}
// Evaluate checks the approval rule of every layer and approves when it holds. Called every minute.
@@ -337,7 +364,12 @@ func (s *Service) Evaluate() ([]Status, error) {
return out, nil
}
func (s *Service) evaluateLayer(layer string) (Status, error) {
func (s *Service) evaluateLayer(layer string) (Status, error) { return s.layerStatus(layer, true) }
// LayerStatus is the read-only view of a fast-lane layer's candidate (the System page): it never approves.
func (s *Service) LayerStatus(layer string) (Status, error) { return s.layerStatus(layer, false) }
func (s *Service) layerStatus(layer string, approve bool) (Status, error) {
ring0, err := s.ring0Hosts()
if err != nil || len(ring0) == 0 {
return Status{Layer: layer, Waiting: "no ring-0 box"}, err
@@ -354,6 +386,7 @@ func (s *Service) evaluateLayer(layer string) (Status, error) {
}
st := Status{Layer: layer, Fingerprint: fp, FirstSeen: first, Packages: len(list)}
if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp {
st.Approved = rel.ID
return st, nil // already approved
}
if age := s.now().Sub(first); age < s.ApproveAfter {
@@ -380,9 +413,53 @@ func (s *Service) evaluateLayer(layer string) (Status, error) {
return st, nil
}
}
if !approve {
st.Waiting = "ready — approves at the next minute's check"
return st, nil
}
return st, s.approve(layer, fp, list, "auto")
}
// ReleaseInfo is one layer's newest approved release, for the System page.
type ReleaseInfo struct {
Layer string
ID string
ApprovedAt time.Time
ApprovedBy string
Packages int
}
// Releases lists the newest release of every layer (guest, host, Docker); a layer with none is absent.
func (s *Service) Releases() []ReleaseInfo {
var out []ReleaseInfo
for _, layer := range AllLayers {
rel, _ := s.Store.LatestOSRelease(layer)
if rel == nil {
continue
}
var list []Package
_ = json.Unmarshal([]byte(rel.PackagesJSON), &list)
out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list)})
}
return out
}
// Candidates is the read-only candidate status of every layer (the Docker one by its own rule).
func (s *Service) Candidates() []Status {
var out []Status
for _, layer := range Layers {
st, _ := s.LayerStatus(layer)
out = append(out, st)
}
d, _ := s.DockerStatus()
return append(out, d)
}
// Thresholds are the alarm numbers the System page colours by (the same values the alarms use).
func (s *Service) Thresholds() (stale, reboot, notCovered time.Duration) {
return dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour)
}
// ApproveNow approves every layer's current candidate at once (operator, urgent fix). Returns the release ids.
func (s *Service) ApproveNow() (string, error) {
ring0, err := s.ring0Hosts()
@@ -418,6 +495,84 @@ func (s *Service) ApproveNow() (string, error) {
return strings.Join(ids, ","), nil
}
// DockerStatus is the Docker engine set ring 0 runs now and whether the operator's button may approve it: every ring-0
// box has run it in DockerNights healthy night Docker steps since it was first seen, and none unhealthy.
func (s *Service) DockerStatus() (Status, error) {
st := Status{Layer: LayerDocker}
ring0, err := s.ring0Hosts()
if err != nil || len(ring0) == 0 {
st.Waiting = "no ring-0 box"
return st, err
}
cand, err := s.candidate(LayerDocker, ring0)
if err != nil || len(cand) == 0 {
st.Waiting = "a ring-0 box has not reported a Docker step"
return st, err
}
fp, list := fingerprint(LayerDocker, cand)
pj, _ := json.Marshal(list)
first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
if err != nil {
return st, err
}
st.Fingerprint, st.FirstSeen, st.Packages = fp, first, len(list)
if rel, _ := s.Store.LatestOSRelease(LayerDocker); rel != nil && rel.Fingerprint == fp {
st.Approved, st.Waiting = rel.ID, "already approved"
return st, nil
}
need := s.DockerNights // 0 = the default 2; negative = none (a TEST override only, logged at start)
switch {
case need == 0:
need = 2
case need < 0:
need = 0
}
for _, h := range ring0 {
reps, err := s.Store.OSReportsSince(h, LayerDocker, first)
if err != nil {
return st, err
}
nights := 0
for _, r := range reps {
if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" {
st.Waiting = fmt.Sprintf("%s reported a Docker step %s (healthy=%v) at %s", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339))
return st, nil
}
if r.Trigger == "night" {
nights++
}
}
if nights < need {
st.Waiting = fmt.Sprintf("%s has %d of %d healthy night Docker step(s) with this set", h, nights, need)
return st, nil
}
}
return st, nil
}
// ApproveDocker is the operator's button: it approves the Docker engine set ring 0 runs, only when DockerStatus allows.
// A ring-1 box then takes it only through a signed operator job (`11` §5.8) — approval alone installs nothing.
func (s *Service) ApproveDocker() (string, error) {
st, err := s.DockerStatus()
if err != nil {
return "", err
}
if st.Waiting != "" {
return "", fmt.Errorf("osupdates: the Docker set cannot be approved yet: %s", st.Waiting)
}
ring0, _ := s.ring0Hosts()
cand, err := s.candidate(LayerDocker, ring0)
if err != nil {
return "", err
}
fp, list := fingerprint(LayerDocker, cand)
if err := s.approve(LayerDocker, fp, list, "operator"); err != nil {
return "", err
}
rel, _ := s.Store.LatestOSRelease(LayerDocker)
return rel.ID, nil
}
func (s *Service) approve(layer, fp string, list []Package, by string) error {
at := s.now().UTC().Truncate(time.Second)
id := "os-" + layer + "-" + at.Format("20060102-150405")
@@ -428,7 +583,7 @@ func (s *Service) approve(layer, fp string, list []Package, by string) error {
s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)", id, layer, by, len(list), fp)
s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages).", id, layer, by, len(list)),
map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp})
if s.Bump != nil {
if s.Bump != nil && layer != LayerDocker { // a Docker set reaches ring 1 only by a signed job, not the desired state
hosts, _ := s.Store.ListHosts()
for _, h := range hosts {
if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled {
@@ -518,6 +673,7 @@ type FleetLine struct {
Tunnel string
Guest LayerLine
Host LayerLine
Docker LayerLine
}
func (s *Service) layerLine(hostID, layer string, ring int) LayerLine {
@@ -605,6 +761,7 @@ func (s *Service) Fleet() ([]FleetLine, error) {
}
l.Guest = s.layerLine(h.HostID, LayerGuest, st.Ring)
l.Host = s.layerLine(h.HostID, LayerHost, st.Ring)
l.Docker = s.layerLine(h.HostID, LayerDocker, st.Ring)
out = append(out, l)
}
return out, nil
@@ -617,7 +774,7 @@ func (s *Service) FleetJSON() (any, error) {
return nil, err
}
out := map[string]any{"boxes": lines}
for _, layer := range Layers {
for _, layer := range AllLayers {
if rel, _ := s.Store.LatestOSRelease(layer); rel != nil {
out["latest_"+layer+"_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy}
}
+6
View File
@@ -313,3 +313,9 @@ func (s *Store) FirstOSReport(layer string, hosts []string) (*OSReport, error) {
}
return &rs[0], nil
}
// CrashSeen / SetCrashSeen keep, per host, what the hub already announced from the box's crash guard (hub v0.132.0):
// the unclean boots, the trip and the re-arm it has turned into events, so a report repeating them stays quiet.
func (s *Store) CrashSeen(hostID string) string { return s.getSetting("crash_seen:" + hostID) }
func (s *Store) SetCrashSeen(hostID, v string) error { return s.setSetting("crash_seen:"+hostID, v) }
+128
View File
@@ -0,0 +1,128 @@
// Package sysfacts reads the agent's `system` stanza (agent v0.142.0, R-852, `09` decision 89): the box's Proxmox,
// kernel, Debian and Docker versions and its crash guard. One parser for the System page, the Hosts page and the
// crash events. A value the box could not read stays "unknown" — never empty, never guessed. Pinned by sysfacts_test.go.
package sysfacts
import (
"encoding/json"
"strings"
)
// Unknown is what a field reads when nobody could read it.
const Unknown = "unknown"
// CrashGuard is the box's felhom-crash-guard state (`11` §5.9).
type CrashGuard struct {
Armed bool `json:"armed"`
Tripped bool `json:"tripped"`
TrippedAt string `json:"tripped_at"`
TrippedReason string `json:"tripped_reason"`
UncleanBoots []string `json:"unclean_boots"`
InWindow int `json:"unclean_boots_in_window"`
In24h int `json:"unclean_boots_24h"`
LastBootAt string `json:"last_boot_at"`
LastBootUnclean bool `json:"last_boot_unclean"`
KernelPanic *int `json:"kernel_panic"`
BootID string `json:"boot_id"`
RearmedAt string `json:"rearmed_at"`
RearmedBy string `json:"rearmed_by"`
}
// Host is the Proxmox host's half.
type Host struct {
Debian string `json:"debian"`
KernelRunning string `json:"kernel_running"`
KernelNextBoot string `json:"kernel_next_boot"`
KernelNextBootSource string `json:"kernel_next_boot_source"`
Held []string `json:"held"` // nil = unknown
Tainted *int `json:"tainted"`
OopsThisBoot *bool `json:"oops_this_boot"`
KernelPanic *int `json:"kernel_panic"`
CrashGuard *CrashGuard `json:"crash_guard"`
}
// Guest is the customer guest's half.
type Guest struct {
Debian string `json:"debian"`
DockerEngine string `json:"docker_engine"`
Containerd string `json:"containerd"`
LiveRestore string `json:"live_restore"` // on | off | unknown
UnknownReason string `json:"unknown_reason"`
}
// System is the whole stanza. Present is false for a report from an agent older than v0.142.0.
type System struct {
Present bool
PVEVersion string
KernelVersion string
VMID int
FactsError string
ReadAt string
Host Host
Guest Guest
}
type wire struct {
System *struct {
PVEVersion string `json:"pve_version"`
KernelVersion string `json:"kernel_version"`
VMID int `json:"vmid"`
Facts json.RawMessage `json:"facts"`
FactsError string `json:"facts_error"`
ReadAt string `json:"read_at"`
} `json:"system"`
}
func orUnknown(s string) string {
if strings.TrimSpace(s) == "" {
return Unknown
}
return s
}
// Parse reads the stanza from a host-report body. Never fails: an absent or broken stanza is Present=false or unknowns.
func Parse(reportJSON string) System {
out := System{PVEVersion: Unknown, KernelVersion: Unknown,
Host: Host{Debian: Unknown, KernelRunning: Unknown, KernelNextBoot: Unknown},
Guest: Guest{Debian: Unknown, DockerEngine: Unknown, Containerd: Unknown, LiveRestore: Unknown}}
var w wire
if json.Unmarshal([]byte(reportJSON), &w) != nil || w.System == nil {
return out
}
out.Present = true
out.PVEVersion, out.KernelVersion = orUnknown(w.System.PVEVersion), orUnknown(w.System.KernelVersion)
out.VMID, out.FactsError, out.ReadAt = w.System.VMID, w.System.FactsError, w.System.ReadAt
var f struct {
Host Host `json:"host"`
Guest Guest `json:"guest"`
}
if len(w.System.Facts) > 0 && json.Unmarshal(w.System.Facts, &f) == nil {
out.Host, out.Guest = f.Host, f.Guest
}
out.Host.Debian, out.Host.KernelRunning = orUnknown(out.Host.Debian), orUnknown(out.Host.KernelRunning)
out.Host.KernelNextBoot = orUnknown(out.Host.KernelNextBoot)
out.Guest.Debian, out.Guest.DockerEngine = orUnknown(out.Guest.Debian), orUnknown(out.Guest.DockerEngine)
out.Guest.Containerd, out.Guest.LiveRestore = orUnknown(out.Guest.Containerd), orUnknown(out.Guest.LiveRestore)
return out
}
// ShortPVE turns "pve-manager/9.0.11/abc123" into "9.0.11".
func ShortPVE(s string) string {
if p := strings.Split(s, "/"); len(p) >= 2 && p[0] == "pve-manager" {
return p[1]
}
return s
}
// ShortKernel turns "Linux 7.0.14-20-pve #1 SMP …" into "7.0.14-20-pve".
func ShortKernel(s string) string {
if f := strings.Fields(s); len(f) >= 2 && f[0] == "Linux" {
return f[1]
}
return s
}
// NextBootDiffers is true when the box will boot a kernel other than the one it runs (both known).
func (s System) NextBootDiffers() bool {
return s.Host.KernelRunning != Unknown && s.Host.KernelNextBoot != Unknown && s.Host.KernelRunning != s.Host.KernelNextBoot
}
+43
View File
@@ -0,0 +1,43 @@
package sysfacts
import "testing"
const full = `{"system":{"pve_version":"pve-manager/9.0.11/abc","kernel_version":"Linux 7.0.14-20-pve #1 SMP","vmid":9201,
"read_at":"2026-10-04T15:00:00Z","facts":{"host":{"debian":"13.7","kernel_running":"7.0.14-20-pve","kernel_next_boot":"7.0.2-6-pve",
"held":["tzdata"],"tainted":4225,"oops_this_boot":true,"kernel_panic":10,"crash_guard":{"armed":false,"tripped":true,
"tripped_at":"2026-10-04T16:00:00Z","unclean_boots":["2026-10-04T15:40:00Z"],"unclean_boots_24h":1}},
"guest":{"debian":"13.7","docker_engine":"29.7.2","containerd":"2.3.3-1","live_restore":"on"}}}}`
func TestParse_Full(t *testing.T) {
s := Parse(full)
if !s.Present || ShortPVE(s.PVEVersion) != "9.0.11" || ShortKernel(s.KernelVersion) != "7.0.14-20-pve" || s.VMID != 9201 {
t.Fatalf("%+v", s)
}
if s.Host.Debian != "13.7" || len(s.Host.Held) != 1 || !*s.Host.OopsThisBoot || !s.Host.CrashGuard.Tripped || !s.NextBootDiffers() {
t.Fatalf("host %+v", s.Host)
}
if s.Guest.DockerEngine != "29.7.2" || s.Guest.LiveRestore != "on" {
t.Fatalf("guest %+v", s.Guest)
}
}
// An agent older than v0.142.0, or a stanza whose facts failed: unknown everywhere, never empty.
// Red-proof: drop the orUnknown on the guest fields and the partial case fails.
func TestParse_AbsentAndPartialAreUnknown(t *testing.T) {
s := Parse(`{"host":{}}`)
if s.Present || s.PVEVersion != Unknown || s.Guest.DockerEngine != Unknown || s.Host.Held != nil {
t.Fatalf("absent: %+v", s)
}
s = Parse(`{"system":{"pve_version":"pve-manager/9.0.11/x","facts_error":"no running customer guest"}}`)
if !s.Present || s.Guest.DockerEngine != Unknown || s.Guest.LiveRestore != Unknown || s.Host.KernelRunning != Unknown || s.FactsError == "" {
t.Fatalf("partial: %+v", s)
}
// facts present but empty fields (a guest that answered nothing): still unknown, not ""
s = Parse(`{"system":{"pve_version":"x","facts":{"host":{},"guest":{"docker_engine":"","live_restore":"","containerd":""}}}}`)
if s.Guest.DockerEngine != Unknown || s.Guest.LiveRestore != Unknown || s.Guest.Containerd != Unknown || s.Host.Debian != Unknown {
t.Fatalf("empty facts fields: %+v", s)
}
if s.NextBootDiffers() {
t.Fatal("unknown kernels must not read as 'differs'")
}
}
+9
View File
@@ -11,6 +11,7 @@ import (
"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
)
// agentOrUnknown renders an agent version for operator text, mapping the empty (never-reported)
@@ -502,6 +503,8 @@ func parseHostStorageTargets(reportJSON string) []storageTargetView {
// hostListRow is the per-host view model for the fleet list.
type hostListRow struct {
PVEVersion string // R-852 (hub v0.132.0): from the report's system stanza; "unknown" when not reported
KernelRunning string
HostID string
CustomerID string
CustomerName string
@@ -590,8 +593,14 @@ func (s *Server) handleHostsList(w http.ResponseWriter, r *http.Request) {
}
// Vitals from the latest report body.
row.PVEVersion, row.KernelRunning = sysfacts.Unknown, sysfacts.Unknown
if reportJSON, _ := s.store.GetLatestHostReportJSON(h.CustomerID); reportJSON != "" {
row.Vitals = parseHostVitals(reportJSON)
sf := sysfacts.Parse(reportJSON)
row.PVEVersion, row.KernelRunning = sysfacts.ShortPVE(sf.PVEVersion), sf.Host.KernelRunning
if row.KernelRunning == sysfacts.Unknown {
row.KernelRunning = sysfacts.ShortKernel(sf.KernelVersion)
}
}
if wf, ok := worstFill[h.HostID]; ok {
+24
View File
@@ -2,7 +2,9 @@ package web
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
)
@@ -12,13 +14,27 @@ import (
// POST /os/ring/<host_id> ring=0|1
// POST /os/enabled/<host_id> on=1|0
// POST /os/approve-now approve the current ring-0 set at once (an operator event)
// POST /os/approve-docker approve the Docker engine set ring 0 ran 2 healthy nights (`11` §5.8)
// (a form field return=/system makes any POST answer with a redirect to the System page)
// GET /os/fleet one line per box (JSON)
func (s *Server) handleOSAdmin(w http.ResponseWriter, r *http.Request, path string) {
if s.osUpdates == nil {
http.Error(w, "os updates not configured", http.StatusServiceUnavailable)
return
}
// A button on the System page posts return=/system: answer with a redirect and a flash, never JSON.
fromPage := r.Method == http.MethodPost && r.FormValue("return") == "/system"
reply := func(v any, err error) {
if fromPage {
q := "flash=done"
if err != nil {
q = "err=" + url.QueryEscape(err.Error())
} else if m, ok := v.(map[string]string); ok && m["release_id"] != "" {
q = "flash=" + url.QueryEscape("approved "+m["release_id"])
}
http.Redirect(w, r, "/system?"+q, http.StatusSeeOther)
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@@ -46,6 +62,14 @@ func (s *Server) handleOSAdmin(w http.ResponseWriter, r *http.Request, path stri
case r.Method == http.MethodPost && path == "/os/approve-now":
id, err := s.osUpdates.ApproveNow()
reply(map[string]string{"release_id": id}, err)
case r.Method == http.MethodPost && path == "/os/approve-docker":
view, ok := s.osUpdates.(OSSystemView)
if !ok {
reply(nil, fmt.Errorf("docker approval not available"))
return
}
id, err := view.ApproveDocker()
reply(map[string]string{"release_id": id}, err)
default:
http.Error(w, "not found", http.StatusNotFound)
}
+3
View File
@@ -457,6 +457,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Hosts — read-only fleet view (audit F-M1) + the v0.46.0 log-bundle actions.
case path == "/hosts" || path == "/hosts/":
s.handleHostsList(w, r)
// System — every box's versions + OS updates, with the operator's buttons (hub v0.132.0, R-852).
case path == "/system":
s.handleSystem(w, r)
// R-21 slice C — unclaimed-appliance operator actions (bind/discard). POST only.
case strings.HasPrefix(path, "/appliances/") && strings.HasSuffix(path, "/bind"):
if id, ok := parseApplianceID(path, "bind"); ok && r.Method == http.MethodPost {
+221
View File
@@ -0,0 +1,221 @@
package web
import (
"fmt"
"net/http"
"sort"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/osupdates"
"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
)
// The System page (hub v0.132.0; R-852, `09` decision 89, `11` §5.7): every box's real versions and its OS-update
// state, with the operator's buttons (ring, switch, approve now, approve the Docker set). Read from the hub's own
// records only: the newest host report's `system` stanza (agent ≥ v0.142.0) and the OS fleet lines.
// cell is one value with its colour: "" plain, "warn" amber, "bad" red (red = an alarm would fire).
type cell struct {
Text string
Class string
Title string
}
type systemRow struct {
HostID, CustomerName string
Ring int
Enabled bool
Tunnel cell
HasFacts bool
FactsNote string
// host
PVE, KernelRunning, KernelNextBoot, HostDebian cell
HostRelease, HostPending, HostNotCovered cell
Held, RebootSince, KernelPanic, Oops cell
CrashRestarts24h, Guard cell
// guest
GuestDebian, GuestRelease, GuestPending, GuestRestart cell
// docker
Engine, Containerd, LiveRestore, DockerRelease cell
// last leg
LastLeg cell
}
// OSSystemView is what the System page needs beyond OSUpdateAdmin (implemented by *osupdates.Service).
type OSSystemView interface {
Fleet() ([]osupdates.FleetLine, error)
Releases() []osupdates.ReleaseInfo
Candidates() []osupdates.Status
Thresholds() (stale, reboot, notCovered time.Duration)
ApproveDocker() (string, error)
}
func plain(s string) cell { return cell{Text: s} }
func unknownCell(s string) cell {
if s == "" || s == sysfacts.Unknown {
return cell{Text: "unknown", Class: "warn", Title: "the box could not read it (agent older than v0.142.0, or the guest is down)"}
}
return cell{Text: s}
}
func ago(t time.Time, now time.Time) string {
if t.IsZero() {
return "never"
}
d := now.Sub(t)
switch {
case d < time.Hour:
return fmt.Sprintf("%d min ago", int(d.Minutes()))
case d < 48*time.Hour:
return fmt.Sprintf("%d h ago", int(d.Hours()))
}
return fmt.Sprintf("%d days ago", int(d.Hours()/24))
}
// buildSystemRows is pure (the render test feeds it directly).
func buildSystemRows(lines []osupdates.FleetLine, facts map[string]sysfacts.System, names map[string]string,
stale, reboot, notCov time.Duration, now time.Time) []systemRow {
var rows []systemRow
for _, l := range lines {
f := facts[l.HostID]
r := systemRow{HostID: l.HostID, CustomerName: names[l.HostID], Ring: l.Ring, Enabled: l.Enabled, HasFacts: f.Present}
switch l.Tunnel {
case "running":
r.Tunnel = plain("running")
case "not_running", "inactive":
r.Tunnel = cell{Text: l.Tunnel, Class: "bad"}
default:
r.Tunnel = cell{Text: l.Tunnel, Class: "warn"}
}
if !f.Present {
r.FactsNote = "no versions reported (agent older than v0.142.0)"
} else if f.FactsError != "" {
r.FactsNote = "partial: " + f.FactsError
}
r.PVE = unknownCell(sysfacts.ShortPVE(f.PVEVersion))
r.KernelRunning = unknownCell(f.Host.KernelRunning)
r.KernelNextBoot = unknownCell(f.Host.KernelNextBoot)
if f.NextBootDiffers() {
r.KernelNextBoot.Class, r.KernelNextBoot.Title = "warn", "the next boot changes the kernel ("+f.Host.KernelNextBootSource+")"
}
r.HostDebian = unknownCell(f.Host.Debian)
r.HostRelease = plain(orDash(l.Host.ReleaseID))
r.HostPending = plain(fmt.Sprint(l.Host.Pending))
r.HostNotCovered = plain(fmt.Sprint(l.Host.NotCoveredFast))
if l.Host.NotCoveredFast > 0 {
r.HostNotCovered.Class = "warn"
}
switch {
case f.Host.Held == nil:
r.Held = unknownCell("")
case len(f.Host.Held) == 0:
r.Held = plain("none")
default:
r.Held = cell{Text: strings.Join(f.Host.Held, ", "), Class: "warn", Title: "held by hand (an undo) — the hub cannot see it otherwise (R-848)"}
}
if l.Host.RebootNeededSince.IsZero() {
r.RebootSince = plain("no")
} else {
r.RebootSince = cell{Text: "since " + l.Host.RebootNeededSince.UTC().Format("2006-01-02"), Class: "warn"}
if now.Sub(l.Host.RebootNeededSince) >= reboot {
r.RebootSince.Class = "bad"
}
}
if f.Host.KernelPanic != nil {
r.KernelPanic = plain(fmt.Sprintf("%d s", *f.Host.KernelPanic))
if *f.Host.KernelPanic == 0 {
r.KernelPanic = cell{Text: "0 (stays off)", Class: "warn"}
}
} else {
r.KernelPanic = unknownCell("")
}
r.Oops = plain("no")
if f.Host.OopsThisBoot != nil && *f.Host.OopsThisBoot {
r.Oops = cell{Text: "yes", Class: "warn", Title: "a kernel oops this boot"}
}
if cg := f.Host.CrashGuard; cg != nil {
r.CrashRestarts24h = plain(fmt.Sprint(cg.In24h))
if cg.In24h > 0 {
r.CrashRestarts24h.Class = "warn"
}
if cg.Tripped {
r.Guard = cell{Text: "TRIPPED " + cg.TrippedAt, Class: "bad", Title: cg.TrippedReason}
} else {
r.Guard = plain("armed")
}
} else {
r.CrashRestarts24h, r.Guard = unknownCell(""), cell{Text: "not installed", Class: "warn"}
}
r.GuestDebian = unknownCell(f.Guest.Debian)
r.GuestRelease = plain(orDash(l.Guest.ReleaseID))
r.GuestPending = plain(fmt.Sprint(l.Guest.Pending))
r.GuestRestart = plain(fmt.Sprint(l.Guest.RestartNeeded))
r.Engine, r.Containerd = unknownCell(f.Guest.DockerEngine), unknownCell(f.Guest.Containerd)
r.LiveRestore = unknownCell(f.Guest.LiveRestore)
if f.Guest.LiveRestore == "off" {
r.LiveRestore.Class, r.LiveRestore.Title = "warn", "a Docker step is refused until it is on (decision 87)"
}
r.DockerRelease = plain(orDash(l.Docker.ReleaseID))
last := l.Guest
if l.Host.LastAt.After(last.LastAt) {
last = l.Host
}
if l.Docker.LastAt.After(last.LastAt) {
last = l.Docker
}
ok := l.Guest.LastSuccessfulLeg
r.LastLeg = cell{Text: fmt.Sprintf("%s · %s · %.0f s", ago(last.LastAt, now), orDash(last.LastOutcome), last.WrapperPassSeconds),
Title: "last successful leg: " + ago(ok, now)}
if l.Enabled && !ok.IsZero() && now.Sub(ok) >= stale {
r.LastLeg.Class = "bad"
} else if last.LastOutcome == "health_failed" || last.LastOutcome == "failed" || last.LastOutcome == "refused" {
r.LastLeg.Class = "warn"
}
rows = append(rows, r)
}
sort.Slice(rows, func(i, j int) bool { return rows[i].HostID < rows[j].HostID })
return rows
}
func orDash(s string) string {
if s == "" {
return "—"
}
return s
}
func (s *Server) handleSystem(w http.ResponseWriter, r *http.Request) {
view, ok := s.osUpdates.(OSSystemView)
if s.osUpdates == nil || !ok {
http.Error(w, "os updates not configured", http.StatusServiceUnavailable)
return
}
lines, err := view.Fleet()
if err != nil {
s.logger.Printf("[ERROR] system page: fleet: %v", err)
http.Error(w, "Internal error", http.StatusInternalServerError)
return
}
hosts, _ := s.store.ListHosts()
facts, names := map[string]sysfacts.System{}, map[string]string{}
for _, h := range hosts {
names[h.HostID] = s.customerName(h.CustomerID)
if rj, _ := s.store.GetLatestHostReportJSON(h.CustomerID); rj != "" {
facts[h.HostID] = sysfacts.Parse(rj)
}
}
stale, reboot, notCov := view.Thresholds()
data := map[string]interface{}{
"Rows": buildSystemRows(lines, facts, names, stale, reboot, notCov, time.Now()),
"Releases": view.Releases(),
"Candidates": view.Candidates(),
"Flash": r.URL.Query().Get("flash"),
"FlashErr": r.URL.Query().Get("err"),
"CSRFToken": s.getCSRFToken(r),
}
if err := s.templates.ExecuteTemplate(w, "system.html", data); err != nil {
s.logger.Printf("[ERROR] system.html template: %v", err)
}
}
+155
View File
@@ -0,0 +1,155 @@
package web
import (
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/osupdates"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
"golang.org/x/crypto/bcrypt"
)
const sysFull = `{"host":{"cpu_percent":1},"cloudflared":{"status":"running"},"system":{"pve_version":"pve-manager/9.0.11/abc",
"kernel_version":"Linux 7.0.14-20-pve #1","vmid":9201,"facts":{"host":{"debian":"13.7","kernel_running":"7.0.14-20-pve",
"kernel_next_boot":"7.0.2-6-pve","kernel_next_boot_source":"saved default","held":["tzdata"],"kernel_panic":0,"oops_this_boot":false,
"crash_guard":{"armed":false,"tripped":true,"tripped_at":"2026-10-04T16:00:00Z","tripped_reason":"2 unclean boots within 60 minutes","unclean_boots_24h":2}},
"guest":{"debian":"13.7","docker_engine":"29.8.2","containerd":"2.3.6-1~debian.13~trixie","live_restore":"on"}}}}`
const sysPartial = `{"host":{"cpu_percent":1},"system":{"pve_version":"pve-manager/9.0.10/x","kernel_version":"Linux 7.0.2-6-pve #1","facts_error":"no running customer guest"}}`
const sysOld = `{"host":{"cpu_percent":1}}`
func systemServer(t *testing.T) (*Server, *store.Store, *osupdates.Service) {
s, st := newTestServer(t)
for _, h := range []struct{ id, cust, body string }{{"full-1", "c-full", sysFull}, {"part-1", "c-part", sysPartial}, {"old-1", "c-old", sysOld}} {
if err := st.UpsertHost(&store.Host{HostID: h.id, CustomerID: h.cust, APIKey: "k-" + h.id}); err != nil {
t.Fatal(err)
}
if err := st.SaveHostReport(h.id, h.cust, []byte(h.body), store.HostReportDenorm{AgentVersion: "0.142.0", CloudflaredStatus: "running"}); err != nil {
t.Fatal(err)
}
}
svc := &osupdates.Service{Store: st, ApproveAfter: 24 * time.Hour, NightsRequired: 1}
s.SetOSUpdateAdmin(svc)
return s, st, svc
}
func getSystem(t *testing.T, s *Server) string {
t.Helper()
rr := httptest.NewRecorder()
s.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/system", nil))
if rr.Code != 200 {
t.Fatalf("GET /system = %d %s", rr.Code, rr.Body.String())
}
return rr.Body.String()
}
// Full, partial and old-agent boxes render; an unreadable value says "unknown", never empty or guessed.
// Red-proof: render KernelRunning with no unknownCell and the partial box shows an empty cell.
func TestSystemPage_FullPartialUnknown(t *testing.T) {
s, _, _ := systemServer(t)
b := getSystem(t, s)
for _, want := range []string{"9.0.11", "7.0.14-20-pve", "29.8.2", "tzdata", "TRIPPED", "0 (stays off)",
"9.0.10", "partial: no running customer guest", "no versions reported (agent older than v0.142.0)",
`action="/os/ring/full-1"`, `action="/os/enabled/part-1"`, `action="/os/approve-now"`} {
if !strings.Contains(b, want) {
t.Errorf("System page lacks %q", want)
}
}
if strings.Count(b, ">unknown<") < 6 {
t.Errorf("the partial and old boxes must read unknown, got %d unknown cells", strings.Count(b, ">unknown<"))
}
if strings.Count(b, `class="c-warn" title="the box could not read it`) < 6 {
t.Errorf("an unknown value must be shown amber with its reason, got %d", strings.Count(b, `class="c-warn" title="the box could not read it`))
}
if !strings.Contains(b, `class="c-bad" title="2 unclean boots`) {
t.Error("a tripped crash guard must be red")
}
}
// The "Approve Docker set" button appears ONLY when the rule allows it (seam rule: one render test per branch).
func TestSystemPage_DockerButtonOnlyWhenReady(t *testing.T) {
s, st, svc := systemServer(t)
if strings.Contains(getSystem(t, s), `action="/os/approve-docker"`) {
t.Fatal("button shown with no Docker candidate")
}
_ = st.SetOSRing("full-1", 0)
night := func() {
if err := svc.Ingest("full-1", osupdates.Report{RunID: time.Now().String(), Layer: "docker", Trigger: "night", Mode: "apply",
Outcome: "nothing", Healthy: true, Installed: []osupdates.Package{{Name: "docker-ce", Version: "5:29.8.2-1", Origin: "Docker"}}}); err != nil {
t.Fatal(err)
}
}
night()
if strings.Contains(getSystem(t, s), `action="/os/approve-docker"`) {
t.Fatal("button shown after ONE night")
}
night()
if !strings.Contains(getSystem(t, s), `action="/os/approve-docker"`) {
t.Fatal("button missing after two healthy nights")
}
}
// Every button needs the operator login; a box's own API key is not one. Red-proof: route /os/ outside RequireAuth.
func TestSystemButtons_NeedTheOperatorLogin(t *testing.T) {
s, st, _ := systemServer(t)
h, _ := bcrypt.GenerateFromPassword([]byte("operator-pw"), bcrypt.MinCost)
s.configPasswordHash = string(h)
for _, tc := range []struct{ method, path, body string }{
{http.MethodGet, "/system", ""},
{http.MethodPost, "/os/ring/full-1", "ring=0&return=%2Fsystem"},
{http.MethodPost, "/os/enabled/full-1", "on=0&return=%2Fsystem"},
{http.MethodPost, "/os/approve-now", "return=%2Fsystem"},
{http.MethodPost, "/os/approve-docker", "return=%2Fsystem"},
} {
for _, auth := range []string{"", "Bearer k-full-1"} {
req := httptest.NewRequest(tc.method, tc.path, strings.NewReader(tc.body))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("X-Requested-With", "XMLHttpRequest")
if auth != "" {
req.Header.Set("Authorization", auth)
}
rr := httptest.NewRecorder()
s.RequireAuth(http.HandlerFunc(s.ServeHTTP)).ServeHTTP(rr, req)
if rr.Code == http.StatusOK || rr.Code == http.StatusSeeOther {
t.Errorf("%s %s with auth %q = %d — must be refused", tc.method, tc.path, auth, rr.Code)
}
}
}
if st.GetOSHostSettings("full-1").Enabled != true || st.GetOSHostSettings("full-1").Ring != 1 {
t.Fatal("an unauthenticated POST changed a box")
}
}
// With the operator's session a page button changes the box and returns to the page.
func TestSystemButtons_RedirectBackToThePage(t *testing.T) {
s, st, _ := systemServer(t)
h, _ := bcrypt.GenerateFromPassword([]byte("operator-pw"), bcrypt.MinCost)
s.configPasswordHash = string(h)
cookie, csrf := newRevealSession(t, s)
form := url.Values{"on": {"0"}, "return": {"/system"}, "_csrf": {csrf}}
req := httptest.NewRequest(http.MethodPost, "/os/enabled/full-1", strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.AddCookie(cookie)
rr := httptest.NewRecorder()
s.RequireAuth(http.HandlerFunc(s.ServeHTTP)).ServeHTTP(rr, req)
if rr.Code != http.StatusSeeOther || !strings.HasPrefix(rr.Header().Get("Location"), "/system?flash=") {
t.Fatalf("= %d %q", rr.Code, rr.Header().Get("Location"))
}
if st.GetOSHostSettings("full-1").Enabled {
t.Fatal("the switch did not change")
}
}
// The Hosts page shows the Proxmox version and the running kernel (unknown for an old agent).
func TestHostsPage_ProxmoxKernelColumn(t *testing.T) {
s, _, _ := systemServer(t)
rr := httptest.NewRecorder()
s.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/hosts", nil))
b := rr.Body.String()
if !strings.Contains(b, "Proxmox / kernel") || !strings.Contains(b, "9.0.11") || !strings.Contains(b, "7.0.2-6-pve") {
t.Fatalf("hosts column missing:\n%s", b[:min(len(b), 400)])
}
}
@@ -18,6 +18,7 @@
<a href="/configs" class="nav-link">Customers</a>
<a href="/apps" class="nav-link active">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
+1
View File
@@ -16,6 +16,7 @@
<a href="/configs" class="nav-link">Customers</a>
<a href="/apps" class="nav-link active">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
@@ -17,6 +17,7 @@
<a href="/configs" class="nav-link active">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
+1
View File
@@ -16,6 +16,7 @@
<a href="/configs" class="nav-link active">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
@@ -16,6 +16,7 @@
<a href="/configs" class="nav-link">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link active">Configuration</a>
</nav>
@@ -18,6 +18,7 @@
<a href="/configs" class="nav-link active">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
@@ -17,6 +17,7 @@
<a href="/configs" class="nav-link">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
@@ -16,6 +16,7 @@
<a href="/configs" class="nav-link">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link active">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
+3
View File
@@ -17,6 +17,7 @@
<a href="/configs" class="nav-link">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link active">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
@@ -85,6 +86,7 @@
<th>Host</th>
<th>Customer</th>
<th>Agent</th>
<th>Proxmox / kernel</th>
<th>Status</th>
<th>Guests</th>
<th>CPU</th>
@@ -100,6 +102,7 @@
<td><a href="/hosts/{{.HostID}}">{{.HostID}}</a></td>
<td>{{if .CustomerName}}{{.CustomerName}}{{else}}{{.CustomerID}}{{end}}</td>
<td>{{if .AgentVersion}}<code>{{.AgentVersion}}</code>{{else}}—{{end}}{{if .FloorHeld}} <span class="status-badge status-warn" title="{{.HeldReason}}">floor held</span>{{else if eq .FloorSource "declared"}} <span class="status-badge" title="served above the vouched golden: the agent requirement was declared with the floor (R-472)">floor: declared MinAgent</span>{{end}}</td>
<td style="font-size: 0.85em;">{{.PVEVersion}}<br><span class="text-muted">{{.KernelRunning}}</span></td>
<td><span class="status-badge {{.StatusClass}}">{{.StatusLabel}}</span></td>
<td>{{if .HasReport}}{{.GuestRunning}}/{{.GuestTotal}}{{else}}—{{end}}</td>
<td>{{if .HasReport}}{{formatFloat .Vitals.CPUPercent}}%{{else}}—{{end}}</td>
+1
View File
@@ -16,6 +16,7 @@
<a href="/configs" class="nav-link active">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
+1
View File
@@ -16,6 +16,7 @@
<a href="/configs" class="nav-link">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link">System</a>
<a href="/offsite" class="nav-link active">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
+128
View File
@@ -0,0 +1,128 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>System — Felhom Hub</title>
<link rel="stylesheet" href="/style.css?v={{hubVersion}}">
<style>
.sys td, .sys th { white-space: nowrap; font-size: 0.82em; vertical-align: top; }
.sys .grp { border-left: 2px solid var(--border, #444); }
.c-warn { color: var(--warn); font-weight: 600; }
.c-bad { color: var(--danger, #e5534b); font-weight: 700; }
.sys form { display: inline; }
.rel-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr)); gap: 0.75rem; }
</style>
</head>
<body>
{{template "icon_sprite"}}
{{template "inline_confirm_js"}}
<div class="container">
<header>
<h1>Felhom <span>Hub</span></h1>
<nav class="nav-links">
<a href="/" class="nav-link">Dashboard</a>
<a href="/configs" class="nav-link">Customers</a>
<a href="/apps" class="nav-link">Apps</a>
<a href="/hosts" class="nav-link">Hosts</a>
<a href="/system" class="nav-link active">System</a>
<a href="/offsite" class="nav-link">Offsite</a>
<a href="/configuration" class="nav-link">Configuration</a>
</nav>
</header>
<h2 style="margin-bottom: 1rem;">System — versions and OS updates</h2>
{{if .Flash}}<div class="flash flash-success" style="margin-bottom: 1rem;">{{.Flash}}</div>{{end}}
{{if .FlashErr}}<div class="flash flash-error" style="margin-bottom: 1rem;">{{.FlashErr}}</div>{{end}}
<section class="card" style="margin-bottom: 1.5rem;">
<h3 style="margin-top: 0;">Approved releases</h3>
<div class="rel-grid">
{{range .Releases}}
<div><strong>{{.Layer}}</strong>: <code>{{.ID}}</code><br>
<span class="text-muted">{{.Packages}} packages · {{.ApprovedAt.UTC.Format "2006-01-02 15:04"}} UTC · by {{.ApprovedBy}}</span></div>
{{else}}<div class="text-muted">No release approved yet.</div>{{end}}
</div>
<h3>What ring 0 runs now</h3>
<div class="rel-grid">
{{range .Candidates}}
<div><strong>{{.Layer}}</strong>:
{{if .Fingerprint}}{{.Packages}} packages, first seen {{.FirstSeen.UTC.Format "2006-01-02 15:04"}} UTC{{else}}<span class="text-muted">—</span>{{end}}<br>
{{if .Approved}}<span class="text-muted">approved as {{.Approved}}</span>
{{else if .Waiting}}<span class="text-muted">{{.Waiting}}</span>{{end}}
{{if and (eq .Layer "docker") .Fingerprint (not .Approved) (eq .Waiting "")}}
<form method="POST" action="/os/approve-docker" style="margin-top: 0.3rem;">
<input type="hidden" name="_csrf" value="{{$.CSRFToken}}"><input type="hidden" name="return" value="/system">
<button type="submit" class="btn btn-sm" data-confirm="Approve this Docker engine set? Ring-1 boxes take it only through a signed operator job.">Approve Docker set</button>
</form>{{end}}
</div>
{{end}}
</div>
<form method="POST" action="/os/approve-now" style="margin-top: 0.8rem;">
<input type="hidden" name="_csrf" value="{{$.CSRFToken}}"><input type="hidden" name="return" value="/system">
<button type="submit" class="btn btn-sm btn-danger" data-confirm="Approve the guest and host sets ring 0 runs NOW, without the 24 h + 1 night wait? Every ring-1 box installs them at its next night run.">Approve now (guest + host)</button>
<span class="text-muted" style="font-size: 0.85em;">An urgent fix only — normally the hub approves after 24 h and one night.</span>
</form>
</section>
{{if .Rows}}
<section class="card" style="padding: 0; overflow-x: auto;">
<table class="data-table sys">
<thead>
<tr>
<th>Box</th><th>Ring / updates</th><th>Tunnel</th>
<th class="grp">Proxmox</th><th>Kernel (running)</th><th>Kernel (next boot)</th><th>Debian</th><th>Felhom release</th><th>Pending</th><th>Not covered</th><th>Held</th><th>Reboot needed</th><th>kernel.panic</th><th>Oops</th><th>Crash restarts 24 h</th><th>Crash guard</th>
<th class="grp">Guest Debian</th><th>Felhom release</th><th>Pending</th><th>Restart needed</th>
<th class="grp">Docker</th><th>containerd</th><th>live-restore</th><th>Docker release</th>
<th class="grp">Last OS leg</th>
</tr>
<tr class="text-muted"><th></th><th></th><th></th><th class="grp" colspan="13">host</th><th class="grp" colspan="4">guest</th><th class="grp" colspan="4">Docker engine</th><th class="grp"></th></tr>
</thead>
<tbody>
{{range .Rows}}
<tr>
<td><a href="/hosts/{{.HostID}}">{{.HostID}}</a>{{if .CustomerName}}<br><span class="text-muted">{{.CustomerName}}</span>{{end}}
{{if .FactsNote}}<br><span class="c-warn" style="font-weight: normal;">{{.FactsNote}}</span>{{end}}</td>
<td>
ring {{.Ring}}
<form method="POST" action="/os/ring/{{.HostID}}">
<input type="hidden" name="_csrf" value="{{$.CSRFToken}}"><input type="hidden" name="return" value="/system">
{{if eq .Ring 0}}<input type="hidden" name="ring" value="1"><button type="submit" class="btn btn-sm btn-outline" data-confirm="Make {{.HostID}} a normal (ring 1) box? It then installs only approved releases.">→ normal</button>
{{else}}<input type="hidden" name="ring" value="0"><button type="submit" class="btn btn-sm btn-outline" data-confirm="Make {{.HostID}} a DEMO (ring 0) box? It then installs every new fix first and takes unsigned Docker steps if its root-owned ring-0 mark allows.">→ demo</button>{{end}}
</form><br>
updates {{if .Enabled}}<strong>ON</strong>{{else}}<span class="c-warn">OFF</span>{{end}}
<form method="POST" action="/os/enabled/{{.HostID}}">
<input type="hidden" name="_csrf" value="{{$.CSRFToken}}"><input type="hidden" name="return" value="/system">
{{if .Enabled}}<input type="hidden" name="on" value="0"><button type="submit" class="btn btn-sm btn-outline" data-confirm="Switch OS updates OFF for {{.HostID}}? It keeps reporting and installs nothing.">switch off</button>
{{else}}<input type="hidden" name="on" value="1"><button type="submit" class="btn btn-sm btn-outline">switch on</button>{{end}}
</form>
</td>
{{template "sys_cell" .Tunnel}}
<td class="grp {{if .PVE.Class}}c-{{.PVE.Class}}{{end}}">{{.PVE.Text}}</td>
{{template "sys_cell" .KernelRunning}}{{template "sys_cell" .KernelNextBoot}}{{template "sys_cell" .HostDebian}}
{{template "sys_cell" .HostRelease}}{{template "sys_cell" .HostPending}}{{template "sys_cell" .HostNotCovered}}
{{template "sys_cell" .Held}}{{template "sys_cell" .RebootSince}}{{template "sys_cell" .KernelPanic}}{{template "sys_cell" .Oops}}
{{template "sys_cell" .CrashRestarts24h}}{{template "sys_cell" .Guard}}
<td class="grp {{if .GuestDebian.Class}}c-{{.GuestDebian.Class}}{{end}}">{{.GuestDebian.Text}}</td>
{{template "sys_cell" .GuestRelease}}{{template "sys_cell" .GuestPending}}{{template "sys_cell" .GuestRestart}}
<td class="grp {{if .Engine.Class}}c-{{.Engine.Class}}{{end}}">{{.Engine.Text}}</td>
{{template "sys_cell" .Containerd}}{{template "sys_cell" .LiveRestore}}{{template "sys_cell" .DockerRelease}}
<td class="grp {{if .LastLeg.Class}}c-{{.LastLeg.Class}}{{end}}" title="{{.LastLeg.Title}}">{{.LastLeg.Text}}</td>
</tr>
{{end}}
</tbody>
</table>
</section>
<p class="text-muted" style="font-size: 0.85em;">Amber: worth a look. Red: an operator alarm fires (`08` §6.3). "unknown": the box could not read the value — never a guess.</p>
{{else}}
<div class="empty-state"><p>No boxes yet.</p></div>
{{end}}
<footer style="margin-top: 2rem; color: var(--text-muted); font-size: 0.8rem; text-align: center;">
Felhom Hub <span style="font-family: var(--font-mono)">{{hubVersion}}</span>
</footer>
</div>
</body>
</html>
{{define "sys_cell"}}<td{{if .Class}} class="c-{{.Class}}"{{end}}{{if .Title}} title="{{.Title}}"{{end}}>{{.Text}}</td>{{end}}