hub v0.132.0: the System page (versions + OS updates with the ring/switch/approve buttons, R-852), the Hosts Proxmox/kernel column, the operator-approved Docker engine release (2 healthy ring-0 nights), the crash-guard events (R-851); evidence audits/os-docker-crash-2026-10-04
gates / gates (push) Successful in 30s
gates / gates (push) Successful in 30s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
|
||||
)
|
||||
|
||||
// The crash-restart events (`09` decision 88, R-851, `11` §5.9, hub v0.132.0), read from the box's crash guard in the
|
||||
// host report's `system` stanza. An unclean boot cannot be told from a power cut on these boxes (measured), so the
|
||||
// texts say "an unexpected stop".
|
||||
const (
|
||||
EventHostCrashRestart = "host_crash_restart" // warning, operator: the box restarted after an unclean stop
|
||||
EventHostRestartedAfterCrash = "host_restarted_after_crash" // info, the HOUSEHOLD's one line (never mailed)
|
||||
EventCrashGuardTripped = "host_crash_guard_tripped" // error, operator: the next crash leaves the box off
|
||||
EventCrashGuardRearmed = "host_crash_guard_rearmed" // info, operator
|
||||
EventKernelOops = "host_kernel_oops" // warning, operator: a kernel oops this boot (no restart)
|
||||
)
|
||||
|
||||
type crashSeen struct {
|
||||
Boots []string `json:"boots"`
|
||||
TrippedAt string `json:"tripped_at"`
|
||||
RearmedAt string `json:"rearmed_at"`
|
||||
OopsBoot string `json:"oops_boot"`
|
||||
}
|
||||
|
||||
// checkCrash turns NEW crash-guard facts into events. The first report the hub sees from a box only records what is
|
||||
// there (no flood of old history). Pinned by crash_test.go.
|
||||
func (h *Handler) checkCrash(hostID, custID string, body []byte) {
|
||||
sys := sysfacts.Parse(string(body))
|
||||
cg := sys.Host.CrashGuard
|
||||
if !sys.Present || cg == nil {
|
||||
return
|
||||
}
|
||||
var seen crashSeen
|
||||
raw := h.store.CrashSeen(hostID)
|
||||
first := raw == ""
|
||||
if !first {
|
||||
_ = json.Unmarshal([]byte(raw), &seen)
|
||||
}
|
||||
known := map[string]bool{}
|
||||
for _, b := range seen.Boots {
|
||||
known[b] = true
|
||||
}
|
||||
emit := func(typ, sev, msg string, extra map[string]any) {
|
||||
if first {
|
||||
return
|
||||
}
|
||||
d := map[string]any{"host_id": hostID}
|
||||
for k, v := range extra {
|
||||
d[k] = v
|
||||
}
|
||||
dj, _ := json.Marshal(d)
|
||||
h.logger.Printf("[WARN] host %s crash guard: %s", hostID, typ)
|
||||
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
|
||||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
|
||||
} else if h.dispatcher != nil {
|
||||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
|
||||
}
|
||||
}
|
||||
for _, b := range cg.UncleanBoots {
|
||||
if known[b] {
|
||||
continue
|
||||
}
|
||||
emit(EventHostCrashRestart, "warning", fmt.Sprintf("%s restarted by itself after an unexpected stop (a kernel crash, a power cut or a hard reset) at %s — %d such restart(s) in 24 h. kernel.panic now %s.",
|
||||
hostID, b, cg.In24h, intOr(cg.KernelPanic, "unknown")), map[string]any{"boot_at": b, "in_24h": cg.In24h})
|
||||
emit(EventHostRestartedAfterCrash, "info", "Your box restarted by itself after an unexpected stop. You do not need to do anything.",
|
||||
map[string]any{"boot_at": b})
|
||||
seen.Boots = append(seen.Boots, b)
|
||||
}
|
||||
if cg.Tripped && cg.TrippedAt != "" && cg.TrippedAt != seen.TrippedAt {
|
||||
emit(EventCrashGuardTripped, "error", fmt.Sprintf("The crash guard of %s TRIPPED at %s: %s. The next crash leaves the box OFF until someone switches it on. Re-arm: `felhom-crash-guard rearm` on the host, or wait 24 h of normal running.",
|
||||
hostID, cg.TrippedAt, cg.TrippedReason), map[string]any{"tripped_at": cg.TrippedAt})
|
||||
seen.TrippedAt = cg.TrippedAt
|
||||
}
|
||||
if cg.RearmedAt != "" && cg.RearmedAt != seen.RearmedAt {
|
||||
emit(EventCrashGuardRearmed, "info", fmt.Sprintf("The crash guard of %s is armed again (%s, by %s).", hostID, cg.RearmedAt, cg.RearmedBy),
|
||||
map[string]any{"rearmed_at": cg.RearmedAt})
|
||||
seen.RearmedAt = cg.RearmedAt
|
||||
}
|
||||
if sys.Host.OopsThisBoot != nil && *sys.Host.OopsThisBoot && cg.BootID != "" && cg.BootID != seen.OopsBoot {
|
||||
emit(EventKernelOops, "warning", fmt.Sprintf("%s logged a kernel oops this boot (an error the kernel survived). The box keeps running; read `journalctl -k -b` on the host.", hostID),
|
||||
map[string]any{"boot_id": cg.BootID})
|
||||
seen.OopsBoot = cg.BootID
|
||||
}
|
||||
if first {
|
||||
seen.Boots, seen.TrippedAt, seen.RearmedAt = append([]string{}, cg.UncleanBoots...), cg.TrippedAt, cg.RearmedAt
|
||||
}
|
||||
if len(seen.Boots) > 200 {
|
||||
seen.Boots = seen.Boots[len(seen.Boots)-200:]
|
||||
}
|
||||
sj, _ := json.Marshal(seen)
|
||||
if err := h.store.SetCrashSeen(hostID, string(sj)); err != nil {
|
||||
h.logger.Printf("[WARN] crash_seen save failed for %s: %v", hostID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func intOr(p *int, d string) string {
|
||||
if p == nil {
|
||||
return d
|
||||
}
|
||||
return fmt.Sprint(*p)
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func crashBody(boots []string, tripped bool, trippedAt, rearmedAt, bootID string, oops bool) []byte {
|
||||
q := func(ss []string) string {
|
||||
if len(ss) == 0 {
|
||||
return "[]"
|
||||
}
|
||||
return `["` + strings.Join(ss, `","`) + `"]`
|
||||
}
|
||||
return []byte(fmt.Sprintf(`{"system":{"pve_version":"pve-manager/9.0.11/x","facts":{"host":{"oops_this_boot":%v,"kernel_panic":10,
|
||||
"crash_guard":{"unclean_boots":%s,"unclean_boots_24h":%d,"tripped":%v,"tripped_at":%q,"tripped_reason":"2 unclean boots within 60 minutes",
|
||||
"rearmed_at":%q,"rearmed_by":"operator","boot_id":%q}},"guest":{}}}}`, oops, q(boots), len(boots), tripped, trippedAt, rearmedAt, bootID))
|
||||
}
|
||||
|
||||
func countEv(t *testing.T, h *Handler, cust, typ string) int {
|
||||
t.Helper()
|
||||
ev, err := h.store.GetEventsByType(cust, typ, time.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return len(ev)
|
||||
}
|
||||
|
||||
// The first report only records history (no flood); each NEW unclean boot is one operator event + one household line;
|
||||
// the trip is one alarm; a repeat report is quiet. Red-proof: drop the `known[b]` skip and the repeat report emits again.
|
||||
func TestCheckCrash_EventsOncePerFact(t *testing.T) {
|
||||
h, st, _ := newTestHandler(t)
|
||||
seedHost(t, st, "h1", "c1", "K1")
|
||||
old := []string{"2026-10-01T10:00:00Z"}
|
||||
h.checkCrash("h1", "c1", crashBody(old, false, "", "", "b0", false))
|
||||
if n := countEv(t, h, "c1", EventHostCrashRestart); n != 0 {
|
||||
t.Fatalf("first sighting flooded %d events", n)
|
||||
}
|
||||
two := append(old, "2026-10-04T15:40:00Z")
|
||||
h.checkCrash("h1", "c1", crashBody(two, false, "", "", "b1", false))
|
||||
h.checkCrash("h1", "c1", crashBody(two, false, "", "", "b1", false)) // the same report again
|
||||
if countEv(t, h, "c1", EventHostCrashRestart) != 1 || countEv(t, h, "c1", EventHostRestartedAfterCrash) != 1 {
|
||||
t.Fatalf("restart events: op=%d household=%d", countEv(t, h, "c1", EventHostCrashRestart), countEv(t, h, "c1", EventHostRestartedAfterCrash))
|
||||
}
|
||||
three := append(two, "2026-10-04T15:45:00Z")
|
||||
h.checkCrash("h1", "c1", crashBody(three, true, "2026-10-04T15:45:30Z", "", "b2", false))
|
||||
h.checkCrash("h1", "c1", crashBody(three, true, "2026-10-04T15:45:30Z", "", "b2", false))
|
||||
if countEv(t, h, "c1", EventCrashGuardTripped) != 1 {
|
||||
t.Fatalf("tripped events = %d", countEv(t, h, "c1", EventCrashGuardTripped))
|
||||
}
|
||||
h.checkCrash("h1", "c1", crashBody(three, false, "", "2026-10-04T16:10:00Z", "b2", true))
|
||||
if countEv(t, h, "c1", EventCrashGuardRearmed) != 1 || countEv(t, h, "c1", EventKernelOops) != 1 {
|
||||
t.Fatal("re-arm / oops not announced")
|
||||
}
|
||||
}
|
||||
|
||||
// An agent older than v0.142.0 (no system stanza) never produces a crash event.
|
||||
func TestCheckCrash_NoStanzaIsQuiet(t *testing.T) {
|
||||
h, st, _ := newTestHandler(t)
|
||||
seedHost(t, st, "h1", "c1", "K1")
|
||||
h.checkCrash("h1", "c1", []byte(`{"host":{}}`))
|
||||
if st.CrashSeen("h1") != "" {
|
||||
t.Fatal("a report without the stanza recorded crash state")
|
||||
}
|
||||
}
|
||||
@@ -863,6 +863,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
|
||||
h.checkCrash(hostID, custID, body)
|
||||
|
||||
for _, g := range rep.Guests {
|
||||
status := g.Status
|
||||
|
||||
@@ -87,5 +87,6 @@
|
||||
"bind.resend.button": "Send me a new link",
|
||||
"bind.resent.lead": "Done.",
|
||||
"bind.resent.body": "If this was a real link and your box is not linked yet, a new e-mail reaches your registered address within a few minutes. If it does not, contact support.",
|
||||
"mail.event.host_restarted_after_crash": "Your box restarted by itself after an unexpected stop. You do not need to do anything.",
|
||||
"mail.event.os_update_applied": "System security fixes were installed on your box. You do not need to do anything."
|
||||
}
|
||||
|
||||
@@ -87,5 +87,6 @@
|
||||
"bind.resend.button": "Új linket kérek",
|
||||
"bind.resent.lead": "Kész.",
|
||||
"bind.resent.body": "Ha ez egy valódi hivatkozás volt, és a dobozod még nincs összekötve, néhány percen belül új e-mailt kapsz a regisztrált címedre. Ha nem jön, szólj az ügyfélszolgálatnak.",
|
||||
"mail.event.host_restarted_after_crash": "A dobozod egy váratlan leállás után magától újraindult. Ehhez nem kell semmit tenned.",
|
||||
"mail.event.os_update_applied": "Rendszerbiztonsági javítások telepítve a dobozodra. Ehhez nem kell semmit tenned."
|
||||
}
|
||||
|
||||
@@ -695,6 +695,11 @@ var operatorOnlyEvents = map[string]bool{
|
||||
"os_reboot_needed": true,
|
||||
"os_ring0_stalled": true,
|
||||
"os_not_covered": true,
|
||||
// R-851 (hub v0.132.0): the crash guard. host_restarted_after_crash is deliberately NOT here — the household's line.
|
||||
"host_crash_restart": true,
|
||||
"host_crash_guard_tripped": true,
|
||||
"host_crash_guard_rearmed": true,
|
||||
"host_kernel_oops": true,
|
||||
// R-197 (v0.93.0). "The sealed offsite repository key changed" is a custody fact about escrow
|
||||
// blobs. A customer can take no action on it — the remedy is the operator's inspection of the
|
||||
// off-site tier — and the text is operator-grade English naming host ids and retained-blob
|
||||
|
||||
@@ -12,11 +12,15 @@ import (
|
||||
func TestOSUpdateEvents_OperatorOnlyExceptApplied(t *testing.T) {
|
||||
for _, e := range []string{osupdates.EventFailed, osupdates.EventHealthFailed, osupdates.EventReleaseApprove,
|
||||
osupdates.EventApprovedNow, osupdates.EventSettings, osupdates.EventStale, osupdates.EventRebootNeeded,
|
||||
osupdates.EventRing0Stalled, osupdates.EventNotCovered, "tunnel_down", "tunnel_recovered"} {
|
||||
osupdates.EventRing0Stalled, osupdates.EventNotCovered, "tunnel_down", "tunnel_recovered",
|
||||
"host_crash_restart", "host_crash_guard_tripped", "host_crash_guard_rearmed", "host_kernel_oops"} {
|
||||
if !operatorOnlyEvents[e] {
|
||||
t.Errorf("%s is not operator-only", e)
|
||||
}
|
||||
}
|
||||
if operatorOnlyEvents["host_restarted_after_crash"] {
|
||||
t.Errorf("host_restarted_after_crash is the household's line and must NOT be operator-only")
|
||||
}
|
||||
if operatorOnlyEvents[osupdates.EventApplied] {
|
||||
t.Errorf("%s is the household's line and must NOT be operator-only", osupdates.EventApplied)
|
||||
}
|
||||
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
SUBJECT: [Felhom] Warning: Your box restarted by itself after an unexpected stop. You do not need to do anything.
|
||||
---
|
||||
Dear Customer,
|
||||
|
||||
Your Felhom system sent the following notification:
|
||||
|
||||
Your box restarted by itself after an unexpected stop. You do not need to do anything.
|
||||
|
||||
Details:
|
||||
- Server: demo-fixture
|
||||
- Time: 2026-01-15 10:30
|
||||
- Level: Warning
|
||||
- Type: host_restarted_after_crash
|
||||
|
||||
If you have any questions, contact your operator.
|
||||
|
||||
Best regards,
|
||||
Felhom.eu monitoring
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
SUBJECT: [Felhom] Figyelmeztetés: A dobozod egy váratlan leállás után magától újraindult. Ehhez nem kell semmit tenned.
|
||||
---
|
||||
Kedves Ügyfél!
|
||||
|
||||
A Felhom rendszered a következő értesítést küldte:
|
||||
|
||||
A dobozod egy váratlan leállás után magától újraindult. Ehhez nem kell semmit tenned.
|
||||
|
||||
Részletek:
|
||||
- Szerver: demo-fixture
|
||||
- Időpont: 2026-01-15 10:30
|
||||
- Szint: Figyelmeztetés
|
||||
- Típus: host_restarted_after_crash
|
||||
|
||||
Ha kérdésed van, vedd fel a kapcsolatot az üzemeltetővel.
|
||||
|
||||
Üdvözlettel,
|
||||
Felhom.eu monitoring
|
||||
@@ -0,0 +1,79 @@
|
||||
package osupdates
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var engineSet = []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker"},
|
||||
{Name: "containerd.io", Version: "2.3.6-1~debian.13~trixie", Origin: "Docker"}}
|
||||
|
||||
func (f *fix) dockerNight(t *testing.T, host string, healthy bool) {
|
||||
t.Helper()
|
||||
out := "nothing"
|
||||
if !healthy {
|
||||
out = "health_failed"
|
||||
}
|
||||
f.ingest(t, host, Report{Layer: LayerDocker, Trigger: "night", Mode: "apply", Outcome: out, Healthy: healthy,
|
||||
Installed: append([]Package{pk("libc6", "x")}, engineSet...)})
|
||||
}
|
||||
|
||||
// The Docker set is NEVER approved automatically (`11` §5.8: the operator approves it). Red-proof: add LayerDocker to
|
||||
// Layers (the auto-approved list) and this fails.
|
||||
func TestDocker_NeverAutoApproved(t *testing.T) {
|
||||
f := newFix(t)
|
||||
for i := 0; i < 3; i++ {
|
||||
f.dockerNight(t, "hp", true)
|
||||
f.dockerNight(t, "n100", true)
|
||||
f.now = f.now.Add(25 * time.Hour)
|
||||
f.s.Evaluate()
|
||||
}
|
||||
if rel, _ := f.s.Store.LatestOSRelease(LayerDocker); rel != nil {
|
||||
t.Fatalf("a Docker set was auto-approved: %+v", rel)
|
||||
}
|
||||
}
|
||||
|
||||
// The operator's button works only after every ring-0 box ran the set 2 healthy nights; an unhealthy step blocks it.
|
||||
// The candidate is the six engine packages only. Red-proof: compare nights against 1 instead of DockerNights and the
|
||||
// one-night step approves.
|
||||
func TestDocker_ApproveNeedsTwoHealthyNightsOnEveryRing0Box(t *testing.T) {
|
||||
f := newFix(t)
|
||||
f.dockerNight(t, "hp", true)
|
||||
f.dockerNight(t, "n100", true)
|
||||
if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "1 of 2") {
|
||||
t.Fatalf("approved after one night: %v", err)
|
||||
}
|
||||
f.now = f.now.Add(24 * time.Hour)
|
||||
f.dockerNight(t, "hp", true)
|
||||
if _, err := f.s.ApproveDocker(); err == nil {
|
||||
t.Fatal("approved while n100 had only one night")
|
||||
}
|
||||
f.dockerNight(t, "n100", true)
|
||||
id, err := f.s.ApproveDocker()
|
||||
if err != nil || !strings.HasPrefix(id, "os-docker-") {
|
||||
t.Fatalf("%q %v", id, err)
|
||||
}
|
||||
rel, _ := f.s.Store.LatestOSRelease(LayerDocker)
|
||||
if rel.ApprovedBy != "operator" || !strings.Contains(rel.PackagesJSON, "docker-ce") || strings.Contains(rel.PackagesJSON, "libc6") {
|
||||
t.Fatalf("release %+v", rel)
|
||||
}
|
||||
if len(f.bumps) != 0 {
|
||||
t.Fatalf("a Docker approval must nudge no box (ring 1 takes it by a signed job): %v", f.bumps)
|
||||
}
|
||||
if b := f.s.DesiredBlock("cust1"); b.Release != nil || b.HostRelease != nil {
|
||||
t.Fatalf("the Docker set leaked into the desired block: %+v", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocker_UnhealthyStepBlocksTheButton(t *testing.T) {
|
||||
f := newFix(t)
|
||||
f.dockerNight(t, "hp", true)
|
||||
f.dockerNight(t, "n100", true)
|
||||
f.now = f.now.Add(24 * time.Hour)
|
||||
f.dockerNight(t, "hp", false)
|
||||
f.dockerNight(t, "n100", true)
|
||||
if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "health_failed") {
|
||||
t.Fatalf("an unhealthy Docker step did not block: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -33,13 +33,21 @@ import (
|
||||
|
||||
// Layers.
|
||||
const (
|
||||
LayerGuest = "guest"
|
||||
LayerHost = "host"
|
||||
LayerGuest = "guest"
|
||||
LayerHost = "host"
|
||||
LayerDocker = "docker" // the guest's Docker engine set — slow lane, OPERATOR-approved only (`11` §5.8, hub v0.132.0)
|
||||
)
|
||||
|
||||
// Layers lists every layer the hub approves.
|
||||
// Layers lists the layers the hub approves AUTOMATICALLY (the fast lane).
|
||||
var Layers = []string{LayerGuest, LayerHost}
|
||||
|
||||
// AllLayers adds the Docker engine set (approved only by the operator's button, ApproveDocker).
|
||||
var AllLayers = []string{LayerGuest, LayerHost, LayerDocker}
|
||||
|
||||
// dockerNames are the six packages of the Docker engine set (the agent wrapper's DOCKER_NAMES).
|
||||
var dockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true,
|
||||
"docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true}
|
||||
|
||||
// hostSlowRE mirrors the wrapper's HOST_SLOW_RE (felhom-agent configs/felhom-os-apply): kernel, boot and firmware
|
||||
// packages are the host's slow lane and never enter a host release (the wrapper would refuse the whole plan, R14).
|
||||
// Pinned by TestCandidate_HostLeavesOutKernelBootFirmware.
|
||||
@@ -91,6 +99,9 @@ type Report struct {
|
||||
RebootScanned bool `json:"reboot_scanned,omitempty"` // agent ≥ 0.141.1: the pass looked (host: every pass)
|
||||
Refused json.RawMessage `json:"refused,omitempty"`
|
||||
PassSeconds float64 `json:"pass_seconds,omitempty"`
|
||||
DockerEngine string `json:"docker_engine,omitempty"` // docker layer (agent v0.142.0)
|
||||
Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed
|
||||
Undo bool `json:"undo,omitempty"` // docker layer: a signed undo
|
||||
}
|
||||
|
||||
// PendingPkg is one update the sources offer.
|
||||
@@ -145,6 +156,10 @@ type Service struct {
|
||||
Emit func(customerID, eventType, severity, message, details, source string)
|
||||
ApproveAfter time.Duration
|
||||
NightsRequired int
|
||||
// DockerNights is how many healthy ring-0 night runs of the Docker step an engine set needs before the operator's
|
||||
// button may approve it (`11` §5.8: 2; 0 means 2; negative means none). OS_DOCKER_APPROVE_NIGHTS overrides it for
|
||||
// a TEST only, logged.
|
||||
DockerNights int
|
||||
// Alarm thresholds (`11` §8.3; decided by CC unattended — operator may reverse). Zero = the default.
|
||||
StaleAfter time.Duration // 7 d
|
||||
RebootAfter time.Duration // 14 d
|
||||
@@ -191,8 +206,9 @@ func (s *Service) event(customerID, typ, sev, msg string, details any) {
|
||||
}
|
||||
|
||||
func layerOf(r Report) string {
|
||||
if r.Layer == LayerHost {
|
||||
return LayerHost
|
||||
switch r.Layer {
|
||||
case LayerHost, LayerDocker:
|
||||
return r.Layer
|
||||
}
|
||||
return LayerGuest
|
||||
}
|
||||
@@ -215,8 +231,11 @@ func (s *Service) Ingest(hostID string, r Report) error {
|
||||
details := map[string]any{"host_id": hostID, "layer": layer, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome,
|
||||
"upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason}
|
||||
where := "the box"
|
||||
if layer == LayerHost {
|
||||
switch layer {
|
||||
case LayerHost:
|
||||
where = "the box's base system"
|
||||
case LayerDocker:
|
||||
where = "the box's app engine (Docker " + r.DockerEngine + ")"
|
||||
}
|
||||
switch r.Outcome {
|
||||
case "applied", "health_failed":
|
||||
@@ -224,8 +243,11 @@ func (s *Service) Ingest(hostID string, r Report) error {
|
||||
fmt.Sprintf("System security fixes installed on %s (%d package(s)).", where, len(r.Upgraded)), details)
|
||||
if !r.Healthy || r.Outcome == "health_failed" {
|
||||
undo := "last night's whole-guest backup is the undo (restore by hand, decision 81)"
|
||||
if layer == LayerHost {
|
||||
switch layer {
|
||||
case LayerHost:
|
||||
undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)"
|
||||
case LayerDocker:
|
||||
undo = "sign an os_docker_step with undo for the previous engine set (runbook `os-updates-docker-undo.md`)"
|
||||
}
|
||||
s.event(h.CustomerID, EventHealthFailed, "error",
|
||||
fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.",
|
||||
@@ -254,7 +276,11 @@ func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, e
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range r.Installed {
|
||||
if p.Origin != "Debian" && p.Origin != "Debian-Security" {
|
||||
if layer == LayerDocker {
|
||||
if !dockerNames[p.Name] {
|
||||
continue
|
||||
}
|
||||
} else if p.Origin != "Debian" && p.Origin != "Debian-Security" {
|
||||
continue
|
||||
}
|
||||
if layer == LayerHost && hostSlowRE.MatchString(p.Name) {
|
||||
@@ -322,6 +348,7 @@ type Status struct {
|
||||
FirstSeen time.Time
|
||||
Packages int
|
||||
Waiting string // why not approved yet ("" = approved or nothing to do)
|
||||
Approved string // the release id when this set is already approved
|
||||
}
|
||||
|
||||
// Evaluate checks the approval rule of every layer and approves when it holds. Called every minute.
|
||||
@@ -337,7 +364,12 @@ func (s *Service) Evaluate() ([]Status, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Service) evaluateLayer(layer string) (Status, error) {
|
||||
func (s *Service) evaluateLayer(layer string) (Status, error) { return s.layerStatus(layer, true) }
|
||||
|
||||
// LayerStatus is the read-only view of a fast-lane layer's candidate (the System page): it never approves.
|
||||
func (s *Service) LayerStatus(layer string) (Status, error) { return s.layerStatus(layer, false) }
|
||||
|
||||
func (s *Service) layerStatus(layer string, approve bool) (Status, error) {
|
||||
ring0, err := s.ring0Hosts()
|
||||
if err != nil || len(ring0) == 0 {
|
||||
return Status{Layer: layer, Waiting: "no ring-0 box"}, err
|
||||
@@ -354,6 +386,7 @@ func (s *Service) evaluateLayer(layer string) (Status, error) {
|
||||
}
|
||||
st := Status{Layer: layer, Fingerprint: fp, FirstSeen: first, Packages: len(list)}
|
||||
if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp {
|
||||
st.Approved = rel.ID
|
||||
return st, nil // already approved
|
||||
}
|
||||
if age := s.now().Sub(first); age < s.ApproveAfter {
|
||||
@@ -380,9 +413,53 @@ func (s *Service) evaluateLayer(layer string) (Status, error) {
|
||||
return st, nil
|
||||
}
|
||||
}
|
||||
if !approve {
|
||||
st.Waiting = "ready — approves at the next minute's check"
|
||||
return st, nil
|
||||
}
|
||||
return st, s.approve(layer, fp, list, "auto")
|
||||
}
|
||||
|
||||
// ReleaseInfo is one layer's newest approved release, for the System page.
|
||||
type ReleaseInfo struct {
|
||||
Layer string
|
||||
ID string
|
||||
ApprovedAt time.Time
|
||||
ApprovedBy string
|
||||
Packages int
|
||||
}
|
||||
|
||||
// Releases lists the newest release of every layer (guest, host, Docker); a layer with none is absent.
|
||||
func (s *Service) Releases() []ReleaseInfo {
|
||||
var out []ReleaseInfo
|
||||
for _, layer := range AllLayers {
|
||||
rel, _ := s.Store.LatestOSRelease(layer)
|
||||
if rel == nil {
|
||||
continue
|
||||
}
|
||||
var list []Package
|
||||
_ = json.Unmarshal([]byte(rel.PackagesJSON), &list)
|
||||
out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Candidates is the read-only candidate status of every layer (the Docker one by its own rule).
|
||||
func (s *Service) Candidates() []Status {
|
||||
var out []Status
|
||||
for _, layer := range Layers {
|
||||
st, _ := s.LayerStatus(layer)
|
||||
out = append(out, st)
|
||||
}
|
||||
d, _ := s.DockerStatus()
|
||||
return append(out, d)
|
||||
}
|
||||
|
||||
// Thresholds are the alarm numbers the System page colours by (the same values the alarms use).
|
||||
func (s *Service) Thresholds() (stale, reboot, notCovered time.Duration) {
|
||||
return dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour)
|
||||
}
|
||||
|
||||
// ApproveNow approves every layer's current candidate at once (operator, urgent fix). Returns the release ids.
|
||||
func (s *Service) ApproveNow() (string, error) {
|
||||
ring0, err := s.ring0Hosts()
|
||||
@@ -418,6 +495,84 @@ func (s *Service) ApproveNow() (string, error) {
|
||||
return strings.Join(ids, ","), nil
|
||||
}
|
||||
|
||||
// DockerStatus is the Docker engine set ring 0 runs now and whether the operator's button may approve it: every ring-0
|
||||
// box has run it in DockerNights healthy night Docker steps since it was first seen, and none unhealthy.
|
||||
func (s *Service) DockerStatus() (Status, error) {
|
||||
st := Status{Layer: LayerDocker}
|
||||
ring0, err := s.ring0Hosts()
|
||||
if err != nil || len(ring0) == 0 {
|
||||
st.Waiting = "no ring-0 box"
|
||||
return st, err
|
||||
}
|
||||
cand, err := s.candidate(LayerDocker, ring0)
|
||||
if err != nil || len(cand) == 0 {
|
||||
st.Waiting = "a ring-0 box has not reported a Docker step"
|
||||
return st, err
|
||||
}
|
||||
fp, list := fingerprint(LayerDocker, cand)
|
||||
pj, _ := json.Marshal(list)
|
||||
first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
|
||||
if err != nil {
|
||||
return st, err
|
||||
}
|
||||
st.Fingerprint, st.FirstSeen, st.Packages = fp, first, len(list)
|
||||
if rel, _ := s.Store.LatestOSRelease(LayerDocker); rel != nil && rel.Fingerprint == fp {
|
||||
st.Approved, st.Waiting = rel.ID, "already approved"
|
||||
return st, nil
|
||||
}
|
||||
need := s.DockerNights // 0 = the default 2; negative = none (a TEST override only, logged at start)
|
||||
switch {
|
||||
case need == 0:
|
||||
need = 2
|
||||
case need < 0:
|
||||
need = 0
|
||||
}
|
||||
for _, h := range ring0 {
|
||||
reps, err := s.Store.OSReportsSince(h, LayerDocker, first)
|
||||
if err != nil {
|
||||
return st, err
|
||||
}
|
||||
nights := 0
|
||||
for _, r := range reps {
|
||||
if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" {
|
||||
st.Waiting = fmt.Sprintf("%s reported a Docker step %s (healthy=%v) at %s", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339))
|
||||
return st, nil
|
||||
}
|
||||
if r.Trigger == "night" {
|
||||
nights++
|
||||
}
|
||||
}
|
||||
if nights < need {
|
||||
st.Waiting = fmt.Sprintf("%s has %d of %d healthy night Docker step(s) with this set", h, nights, need)
|
||||
return st, nil
|
||||
}
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// ApproveDocker is the operator's button: it approves the Docker engine set ring 0 runs, only when DockerStatus allows.
|
||||
// A ring-1 box then takes it only through a signed operator job (`11` §5.8) — approval alone installs nothing.
|
||||
func (s *Service) ApproveDocker() (string, error) {
|
||||
st, err := s.DockerStatus()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if st.Waiting != "" {
|
||||
return "", fmt.Errorf("osupdates: the Docker set cannot be approved yet: %s", st.Waiting)
|
||||
}
|
||||
ring0, _ := s.ring0Hosts()
|
||||
cand, err := s.candidate(LayerDocker, ring0)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
fp, list := fingerprint(LayerDocker, cand)
|
||||
if err := s.approve(LayerDocker, fp, list, "operator"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
rel, _ := s.Store.LatestOSRelease(LayerDocker)
|
||||
return rel.ID, nil
|
||||
}
|
||||
|
||||
func (s *Service) approve(layer, fp string, list []Package, by string) error {
|
||||
at := s.now().UTC().Truncate(time.Second)
|
||||
id := "os-" + layer + "-" + at.Format("20060102-150405")
|
||||
@@ -428,7 +583,7 @@ func (s *Service) approve(layer, fp string, list []Package, by string) error {
|
||||
s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)", id, layer, by, len(list), fp)
|
||||
s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages).", id, layer, by, len(list)),
|
||||
map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp})
|
||||
if s.Bump != nil {
|
||||
if s.Bump != nil && layer != LayerDocker { // a Docker set reaches ring 1 only by a signed job, not the desired state
|
||||
hosts, _ := s.Store.ListHosts()
|
||||
for _, h := range hosts {
|
||||
if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled {
|
||||
@@ -518,6 +673,7 @@ type FleetLine struct {
|
||||
Tunnel string
|
||||
Guest LayerLine
|
||||
Host LayerLine
|
||||
Docker LayerLine
|
||||
}
|
||||
|
||||
func (s *Service) layerLine(hostID, layer string, ring int) LayerLine {
|
||||
@@ -605,6 +761,7 @@ func (s *Service) Fleet() ([]FleetLine, error) {
|
||||
}
|
||||
l.Guest = s.layerLine(h.HostID, LayerGuest, st.Ring)
|
||||
l.Host = s.layerLine(h.HostID, LayerHost, st.Ring)
|
||||
l.Docker = s.layerLine(h.HostID, LayerDocker, st.Ring)
|
||||
out = append(out, l)
|
||||
}
|
||||
return out, nil
|
||||
@@ -617,7 +774,7 @@ func (s *Service) FleetJSON() (any, error) {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]any{"boxes": lines}
|
||||
for _, layer := range Layers {
|
||||
for _, layer := range AllLayers {
|
||||
if rel, _ := s.Store.LatestOSRelease(layer); rel != nil {
|
||||
out["latest_"+layer+"_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy}
|
||||
}
|
||||
|
||||
@@ -313,3 +313,9 @@ func (s *Store) FirstOSReport(layer string, hosts []string) (*OSReport, error) {
|
||||
}
|
||||
return &rs[0], nil
|
||||
}
|
||||
|
||||
// CrashSeen / SetCrashSeen keep, per host, what the hub already announced from the box's crash guard (hub v0.132.0):
|
||||
// the unclean boots, the trip and the re-arm it has turned into events, so a report repeating them stays quiet.
|
||||
func (s *Store) CrashSeen(hostID string) string { return s.getSetting("crash_seen:" + hostID) }
|
||||
|
||||
func (s *Store) SetCrashSeen(hostID, v string) error { return s.setSetting("crash_seen:"+hostID, v) }
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
// Package sysfacts reads the agent's `system` stanza (agent v0.142.0, R-852, `09` decision 89): the box's Proxmox,
|
||||
// kernel, Debian and Docker versions and its crash guard. One parser for the System page, the Hosts page and the
|
||||
// crash events. A value the box could not read stays "unknown" — never empty, never guessed. Pinned by sysfacts_test.go.
|
||||
package sysfacts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Unknown is what a field reads when nobody could read it.
|
||||
const Unknown = "unknown"
|
||||
|
||||
// CrashGuard is the box's felhom-crash-guard state (`11` §5.9).
|
||||
type CrashGuard struct {
|
||||
Armed bool `json:"armed"`
|
||||
Tripped bool `json:"tripped"`
|
||||
TrippedAt string `json:"tripped_at"`
|
||||
TrippedReason string `json:"tripped_reason"`
|
||||
UncleanBoots []string `json:"unclean_boots"`
|
||||
InWindow int `json:"unclean_boots_in_window"`
|
||||
In24h int `json:"unclean_boots_24h"`
|
||||
LastBootAt string `json:"last_boot_at"`
|
||||
LastBootUnclean bool `json:"last_boot_unclean"`
|
||||
KernelPanic *int `json:"kernel_panic"`
|
||||
BootID string `json:"boot_id"`
|
||||
RearmedAt string `json:"rearmed_at"`
|
||||
RearmedBy string `json:"rearmed_by"`
|
||||
}
|
||||
|
||||
// Host is the Proxmox host's half.
|
||||
type Host struct {
|
||||
Debian string `json:"debian"`
|
||||
KernelRunning string `json:"kernel_running"`
|
||||
KernelNextBoot string `json:"kernel_next_boot"`
|
||||
KernelNextBootSource string `json:"kernel_next_boot_source"`
|
||||
Held []string `json:"held"` // nil = unknown
|
||||
Tainted *int `json:"tainted"`
|
||||
OopsThisBoot *bool `json:"oops_this_boot"`
|
||||
KernelPanic *int `json:"kernel_panic"`
|
||||
CrashGuard *CrashGuard `json:"crash_guard"`
|
||||
}
|
||||
|
||||
// Guest is the customer guest's half.
|
||||
type Guest struct {
|
||||
Debian string `json:"debian"`
|
||||
DockerEngine string `json:"docker_engine"`
|
||||
Containerd string `json:"containerd"`
|
||||
LiveRestore string `json:"live_restore"` // on | off | unknown
|
||||
UnknownReason string `json:"unknown_reason"`
|
||||
}
|
||||
|
||||
// System is the whole stanza. Present is false for a report from an agent older than v0.142.0.
|
||||
type System struct {
|
||||
Present bool
|
||||
PVEVersion string
|
||||
KernelVersion string
|
||||
VMID int
|
||||
FactsError string
|
||||
ReadAt string
|
||||
Host Host
|
||||
Guest Guest
|
||||
}
|
||||
|
||||
type wire struct {
|
||||
System *struct {
|
||||
PVEVersion string `json:"pve_version"`
|
||||
KernelVersion string `json:"kernel_version"`
|
||||
VMID int `json:"vmid"`
|
||||
Facts json.RawMessage `json:"facts"`
|
||||
FactsError string `json:"facts_error"`
|
||||
ReadAt string `json:"read_at"`
|
||||
} `json:"system"`
|
||||
}
|
||||
|
||||
func orUnknown(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return Unknown
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Parse reads the stanza from a host-report body. Never fails: an absent or broken stanza is Present=false or unknowns.
|
||||
func Parse(reportJSON string) System {
|
||||
out := System{PVEVersion: Unknown, KernelVersion: Unknown,
|
||||
Host: Host{Debian: Unknown, KernelRunning: Unknown, KernelNextBoot: Unknown},
|
||||
Guest: Guest{Debian: Unknown, DockerEngine: Unknown, Containerd: Unknown, LiveRestore: Unknown}}
|
||||
var w wire
|
||||
if json.Unmarshal([]byte(reportJSON), &w) != nil || w.System == nil {
|
||||
return out
|
||||
}
|
||||
out.Present = true
|
||||
out.PVEVersion, out.KernelVersion = orUnknown(w.System.PVEVersion), orUnknown(w.System.KernelVersion)
|
||||
out.VMID, out.FactsError, out.ReadAt = w.System.VMID, w.System.FactsError, w.System.ReadAt
|
||||
var f struct {
|
||||
Host Host `json:"host"`
|
||||
Guest Guest `json:"guest"`
|
||||
}
|
||||
if len(w.System.Facts) > 0 && json.Unmarshal(w.System.Facts, &f) == nil {
|
||||
out.Host, out.Guest = f.Host, f.Guest
|
||||
}
|
||||
out.Host.Debian, out.Host.KernelRunning = orUnknown(out.Host.Debian), orUnknown(out.Host.KernelRunning)
|
||||
out.Host.KernelNextBoot = orUnknown(out.Host.KernelNextBoot)
|
||||
out.Guest.Debian, out.Guest.DockerEngine = orUnknown(out.Guest.Debian), orUnknown(out.Guest.DockerEngine)
|
||||
out.Guest.Containerd, out.Guest.LiveRestore = orUnknown(out.Guest.Containerd), orUnknown(out.Guest.LiveRestore)
|
||||
return out
|
||||
}
|
||||
|
||||
// ShortPVE turns "pve-manager/9.0.11/abc123" into "9.0.11".
|
||||
func ShortPVE(s string) string {
|
||||
if p := strings.Split(s, "/"); len(p) >= 2 && p[0] == "pve-manager" {
|
||||
return p[1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// ShortKernel turns "Linux 7.0.14-20-pve #1 SMP …" into "7.0.14-20-pve".
|
||||
func ShortKernel(s string) string {
|
||||
if f := strings.Fields(s); len(f) >= 2 && f[0] == "Linux" {
|
||||
return f[1]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// NextBootDiffers is true when the box will boot a kernel other than the one it runs (both known).
|
||||
func (s System) NextBootDiffers() bool {
|
||||
return s.Host.KernelRunning != Unknown && s.Host.KernelNextBoot != Unknown && s.Host.KernelRunning != s.Host.KernelNextBoot
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package sysfacts
|
||||
|
||||
import "testing"
|
||||
|
||||
const full = `{"system":{"pve_version":"pve-manager/9.0.11/abc","kernel_version":"Linux 7.0.14-20-pve #1 SMP","vmid":9201,
|
||||
"read_at":"2026-10-04T15:00:00Z","facts":{"host":{"debian":"13.7","kernel_running":"7.0.14-20-pve","kernel_next_boot":"7.0.2-6-pve",
|
||||
"held":["tzdata"],"tainted":4225,"oops_this_boot":true,"kernel_panic":10,"crash_guard":{"armed":false,"tripped":true,
|
||||
"tripped_at":"2026-10-04T16:00:00Z","unclean_boots":["2026-10-04T15:40:00Z"],"unclean_boots_24h":1}},
|
||||
"guest":{"debian":"13.7","docker_engine":"29.7.2","containerd":"2.3.3-1","live_restore":"on"}}}}`
|
||||
|
||||
func TestParse_Full(t *testing.T) {
|
||||
s := Parse(full)
|
||||
if !s.Present || ShortPVE(s.PVEVersion) != "9.0.11" || ShortKernel(s.KernelVersion) != "7.0.14-20-pve" || s.VMID != 9201 {
|
||||
t.Fatalf("%+v", s)
|
||||
}
|
||||
if s.Host.Debian != "13.7" || len(s.Host.Held) != 1 || !*s.Host.OopsThisBoot || !s.Host.CrashGuard.Tripped || !s.NextBootDiffers() {
|
||||
t.Fatalf("host %+v", s.Host)
|
||||
}
|
||||
if s.Guest.DockerEngine != "29.7.2" || s.Guest.LiveRestore != "on" {
|
||||
t.Fatalf("guest %+v", s.Guest)
|
||||
}
|
||||
}
|
||||
|
||||
// An agent older than v0.142.0, or a stanza whose facts failed: unknown everywhere, never empty.
|
||||
// Red-proof: drop the orUnknown on the guest fields and the partial case fails.
|
||||
func TestParse_AbsentAndPartialAreUnknown(t *testing.T) {
|
||||
s := Parse(`{"host":{}}`)
|
||||
if s.Present || s.PVEVersion != Unknown || s.Guest.DockerEngine != Unknown || s.Host.Held != nil {
|
||||
t.Fatalf("absent: %+v", s)
|
||||
}
|
||||
s = Parse(`{"system":{"pve_version":"pve-manager/9.0.11/x","facts_error":"no running customer guest"}}`)
|
||||
if !s.Present || s.Guest.DockerEngine != Unknown || s.Guest.LiveRestore != Unknown || s.Host.KernelRunning != Unknown || s.FactsError == "" {
|
||||
t.Fatalf("partial: %+v", s)
|
||||
}
|
||||
// facts present but empty fields (a guest that answered nothing): still unknown, not ""
|
||||
s = Parse(`{"system":{"pve_version":"x","facts":{"host":{},"guest":{"docker_engine":"","live_restore":"","containerd":""}}}}`)
|
||||
if s.Guest.DockerEngine != Unknown || s.Guest.LiveRestore != Unknown || s.Guest.Containerd != Unknown || s.Host.Debian != Unknown {
|
||||
t.Fatalf("empty facts fields: %+v", s)
|
||||
}
|
||||
if s.NextBootDiffers() {
|
||||
t.Fatal("unknown kernels must not read as 'differs'")
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
|
||||
)
|
||||
|
||||
// agentOrUnknown renders an agent version for operator text, mapping the empty (never-reported)
|
||||
@@ -502,6 +503,8 @@ func parseHostStorageTargets(reportJSON string) []storageTargetView {
|
||||
|
||||
// hostListRow is the per-host view model for the fleet list.
|
||||
type hostListRow struct {
|
||||
PVEVersion string // R-852 (hub v0.132.0): from the report's system stanza; "unknown" when not reported
|
||||
KernelRunning string
|
||||
HostID string
|
||||
CustomerID string
|
||||
CustomerName string
|
||||
@@ -590,8 +593,14 @@ func (s *Server) handleHostsList(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Vitals from the latest report body.
|
||||
row.PVEVersion, row.KernelRunning = sysfacts.Unknown, sysfacts.Unknown
|
||||
if reportJSON, _ := s.store.GetLatestHostReportJSON(h.CustomerID); reportJSON != "" {
|
||||
row.Vitals = parseHostVitals(reportJSON)
|
||||
sf := sysfacts.Parse(reportJSON)
|
||||
row.PVEVersion, row.KernelRunning = sysfacts.ShortPVE(sf.PVEVersion), sf.Host.KernelRunning
|
||||
if row.KernelRunning == sysfacts.Unknown {
|
||||
row.KernelRunning = sysfacts.ShortKernel(sf.KernelVersion)
|
||||
}
|
||||
}
|
||||
|
||||
if wf, ok := worstFill[h.HostID]; ok {
|
||||
|
||||
@@ -2,7 +2,9 @@ package web
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
@@ -12,13 +14,27 @@ import (
|
||||
// POST /os/ring/<host_id> ring=0|1
|
||||
// POST /os/enabled/<host_id> on=1|0
|
||||
// POST /os/approve-now approve the current ring-0 set at once (an operator event)
|
||||
// POST /os/approve-docker approve the Docker engine set ring 0 ran 2 healthy nights (`11` §5.8)
|
||||
// (a form field return=/system makes any POST answer with a redirect to the System page)
|
||||
// GET /os/fleet one line per box (JSON)
|
||||
func (s *Server) handleOSAdmin(w http.ResponseWriter, r *http.Request, path string) {
|
||||
if s.osUpdates == nil {
|
||||
http.Error(w, "os updates not configured", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
// A button on the System page posts return=/system: answer with a redirect and a flash, never JSON.
|
||||
fromPage := r.Method == http.MethodPost && r.FormValue("return") == "/system"
|
||||
reply := func(v any, err error) {
|
||||
if fromPage {
|
||||
q := "flash=done"
|
||||
if err != nil {
|
||||
q = "err=" + url.QueryEscape(err.Error())
|
||||
} else if m, ok := v.(map[string]string); ok && m["release_id"] != "" {
|
||||
q = "flash=" + url.QueryEscape("approved "+m["release_id"])
|
||||
}
|
||||
http.Redirect(w, r, "/system?"+q, http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
@@ -46,6 +62,14 @@ func (s *Server) handleOSAdmin(w http.ResponseWriter, r *http.Request, path stri
|
||||
case r.Method == http.MethodPost && path == "/os/approve-now":
|
||||
id, err := s.osUpdates.ApproveNow()
|
||||
reply(map[string]string{"release_id": id}, err)
|
||||
case r.Method == http.MethodPost && path == "/os/approve-docker":
|
||||
view, ok := s.osUpdates.(OSSystemView)
|
||||
if !ok {
|
||||
reply(nil, fmt.Errorf("docker approval not available"))
|
||||
return
|
||||
}
|
||||
id, err := view.ApproveDocker()
|
||||
reply(map[string]string{"release_id": id}, err)
|
||||
default:
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
@@ -457,6 +457,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Hosts — read-only fleet view (audit F-M1) + the v0.46.0 log-bundle actions.
|
||||
case path == "/hosts" || path == "/hosts/":
|
||||
s.handleHostsList(w, r)
|
||||
// System — every box's versions + OS updates, with the operator's buttons (hub v0.132.0, R-852).
|
||||
case path == "/system":
|
||||
s.handleSystem(w, r)
|
||||
// R-21 slice C — unclaimed-appliance operator actions (bind/discard). POST only.
|
||||
case strings.HasPrefix(path, "/appliances/") && strings.HasSuffix(path, "/bind"):
|
||||
if id, ok := parseApplianceID(path, "bind"); ok && r.Method == http.MethodPost {
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/osupdates"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
|
||||
)
|
||||
|
||||
// The System page (hub v0.132.0; R-852, `09` decision 89, `11` §5.7): every box's real versions and its OS-update
|
||||
// state, with the operator's buttons (ring, switch, approve now, approve the Docker set). Read from the hub's own
|
||||
// records only: the newest host report's `system` stanza (agent ≥ v0.142.0) and the OS fleet lines.
|
||||
|
||||
// cell is one value with its colour: "" plain, "warn" amber, "bad" red (red = an alarm would fire).
|
||||
type cell struct {
|
||||
Text string
|
||||
Class string
|
||||
Title string
|
||||
}
|
||||
|
||||
type systemRow struct {
|
||||
HostID, CustomerName string
|
||||
Ring int
|
||||
Enabled bool
|
||||
Tunnel cell
|
||||
HasFacts bool
|
||||
FactsNote string
|
||||
// host
|
||||
PVE, KernelRunning, KernelNextBoot, HostDebian cell
|
||||
HostRelease, HostPending, HostNotCovered cell
|
||||
Held, RebootSince, KernelPanic, Oops cell
|
||||
CrashRestarts24h, Guard cell
|
||||
// guest
|
||||
GuestDebian, GuestRelease, GuestPending, GuestRestart cell
|
||||
// docker
|
||||
Engine, Containerd, LiveRestore, DockerRelease cell
|
||||
// last leg
|
||||
LastLeg cell
|
||||
}
|
||||
|
||||
// OSSystemView is what the System page needs beyond OSUpdateAdmin (implemented by *osupdates.Service).
|
||||
type OSSystemView interface {
|
||||
Fleet() ([]osupdates.FleetLine, error)
|
||||
Releases() []osupdates.ReleaseInfo
|
||||
Candidates() []osupdates.Status
|
||||
Thresholds() (stale, reboot, notCovered time.Duration)
|
||||
ApproveDocker() (string, error)
|
||||
}
|
||||
|
||||
func plain(s string) cell { return cell{Text: s} }
|
||||
|
||||
func unknownCell(s string) cell {
|
||||
if s == "" || s == sysfacts.Unknown {
|
||||
return cell{Text: "unknown", Class: "warn", Title: "the box could not read it (agent older than v0.142.0, or the guest is down)"}
|
||||
}
|
||||
return cell{Text: s}
|
||||
}
|
||||
|
||||
func ago(t time.Time, now time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "never"
|
||||
}
|
||||
d := now.Sub(t)
|
||||
switch {
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("%d min ago", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%d h ago", int(d.Hours()))
|
||||
}
|
||||
return fmt.Sprintf("%d days ago", int(d.Hours()/24))
|
||||
}
|
||||
|
||||
// buildSystemRows is pure (the render test feeds it directly).
|
||||
func buildSystemRows(lines []osupdates.FleetLine, facts map[string]sysfacts.System, names map[string]string,
|
||||
stale, reboot, notCov time.Duration, now time.Time) []systemRow {
|
||||
var rows []systemRow
|
||||
for _, l := range lines {
|
||||
f := facts[l.HostID]
|
||||
r := systemRow{HostID: l.HostID, CustomerName: names[l.HostID], Ring: l.Ring, Enabled: l.Enabled, HasFacts: f.Present}
|
||||
switch l.Tunnel {
|
||||
case "running":
|
||||
r.Tunnel = plain("running")
|
||||
case "not_running", "inactive":
|
||||
r.Tunnel = cell{Text: l.Tunnel, Class: "bad"}
|
||||
default:
|
||||
r.Tunnel = cell{Text: l.Tunnel, Class: "warn"}
|
||||
}
|
||||
if !f.Present {
|
||||
r.FactsNote = "no versions reported (agent older than v0.142.0)"
|
||||
} else if f.FactsError != "" {
|
||||
r.FactsNote = "partial: " + f.FactsError
|
||||
}
|
||||
r.PVE = unknownCell(sysfacts.ShortPVE(f.PVEVersion))
|
||||
r.KernelRunning = unknownCell(f.Host.KernelRunning)
|
||||
r.KernelNextBoot = unknownCell(f.Host.KernelNextBoot)
|
||||
if f.NextBootDiffers() {
|
||||
r.KernelNextBoot.Class, r.KernelNextBoot.Title = "warn", "the next boot changes the kernel ("+f.Host.KernelNextBootSource+")"
|
||||
}
|
||||
r.HostDebian = unknownCell(f.Host.Debian)
|
||||
r.HostRelease = plain(orDash(l.Host.ReleaseID))
|
||||
r.HostPending = plain(fmt.Sprint(l.Host.Pending))
|
||||
r.HostNotCovered = plain(fmt.Sprint(l.Host.NotCoveredFast))
|
||||
if l.Host.NotCoveredFast > 0 {
|
||||
r.HostNotCovered.Class = "warn"
|
||||
}
|
||||
switch {
|
||||
case f.Host.Held == nil:
|
||||
r.Held = unknownCell("")
|
||||
case len(f.Host.Held) == 0:
|
||||
r.Held = plain("none")
|
||||
default:
|
||||
r.Held = cell{Text: strings.Join(f.Host.Held, ", "), Class: "warn", Title: "held by hand (an undo) — the hub cannot see it otherwise (R-848)"}
|
||||
}
|
||||
if l.Host.RebootNeededSince.IsZero() {
|
||||
r.RebootSince = plain("no")
|
||||
} else {
|
||||
r.RebootSince = cell{Text: "since " + l.Host.RebootNeededSince.UTC().Format("2006-01-02"), Class: "warn"}
|
||||
if now.Sub(l.Host.RebootNeededSince) >= reboot {
|
||||
r.RebootSince.Class = "bad"
|
||||
}
|
||||
}
|
||||
if f.Host.KernelPanic != nil {
|
||||
r.KernelPanic = plain(fmt.Sprintf("%d s", *f.Host.KernelPanic))
|
||||
if *f.Host.KernelPanic == 0 {
|
||||
r.KernelPanic = cell{Text: "0 (stays off)", Class: "warn"}
|
||||
}
|
||||
} else {
|
||||
r.KernelPanic = unknownCell("")
|
||||
}
|
||||
r.Oops = plain("no")
|
||||
if f.Host.OopsThisBoot != nil && *f.Host.OopsThisBoot {
|
||||
r.Oops = cell{Text: "yes", Class: "warn", Title: "a kernel oops this boot"}
|
||||
}
|
||||
if cg := f.Host.CrashGuard; cg != nil {
|
||||
r.CrashRestarts24h = plain(fmt.Sprint(cg.In24h))
|
||||
if cg.In24h > 0 {
|
||||
r.CrashRestarts24h.Class = "warn"
|
||||
}
|
||||
if cg.Tripped {
|
||||
r.Guard = cell{Text: "TRIPPED " + cg.TrippedAt, Class: "bad", Title: cg.TrippedReason}
|
||||
} else {
|
||||
r.Guard = plain("armed")
|
||||
}
|
||||
} else {
|
||||
r.CrashRestarts24h, r.Guard = unknownCell(""), cell{Text: "not installed", Class: "warn"}
|
||||
}
|
||||
r.GuestDebian = unknownCell(f.Guest.Debian)
|
||||
r.GuestRelease = plain(orDash(l.Guest.ReleaseID))
|
||||
r.GuestPending = plain(fmt.Sprint(l.Guest.Pending))
|
||||
r.GuestRestart = plain(fmt.Sprint(l.Guest.RestartNeeded))
|
||||
r.Engine, r.Containerd = unknownCell(f.Guest.DockerEngine), unknownCell(f.Guest.Containerd)
|
||||
r.LiveRestore = unknownCell(f.Guest.LiveRestore)
|
||||
if f.Guest.LiveRestore == "off" {
|
||||
r.LiveRestore.Class, r.LiveRestore.Title = "warn", "a Docker step is refused until it is on (decision 87)"
|
||||
}
|
||||
r.DockerRelease = plain(orDash(l.Docker.ReleaseID))
|
||||
last := l.Guest
|
||||
if l.Host.LastAt.After(last.LastAt) {
|
||||
last = l.Host
|
||||
}
|
||||
if l.Docker.LastAt.After(last.LastAt) {
|
||||
last = l.Docker
|
||||
}
|
||||
ok := l.Guest.LastSuccessfulLeg
|
||||
r.LastLeg = cell{Text: fmt.Sprintf("%s · %s · %.0f s", ago(last.LastAt, now), orDash(last.LastOutcome), last.WrapperPassSeconds),
|
||||
Title: "last successful leg: " + ago(ok, now)}
|
||||
if l.Enabled && !ok.IsZero() && now.Sub(ok) >= stale {
|
||||
r.LastLeg.Class = "bad"
|
||||
} else if last.LastOutcome == "health_failed" || last.LastOutcome == "failed" || last.LastOutcome == "refused" {
|
||||
r.LastLeg.Class = "warn"
|
||||
}
|
||||
rows = append(rows, r)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].HostID < rows[j].HostID })
|
||||
return rows
|
||||
}
|
||||
|
||||
func orDash(s string) string {
|
||||
if s == "" {
|
||||
return "—"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *Server) handleSystem(w http.ResponseWriter, r *http.Request) {
|
||||
view, ok := s.osUpdates.(OSSystemView)
|
||||
if s.osUpdates == nil || !ok {
|
||||
http.Error(w, "os updates not configured", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
lines, err := view.Fleet()
|
||||
if err != nil {
|
||||
s.logger.Printf("[ERROR] system page: fleet: %v", err)
|
||||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
hosts, _ := s.store.ListHosts()
|
||||
facts, names := map[string]sysfacts.System{}, map[string]string{}
|
||||
for _, h := range hosts {
|
||||
names[h.HostID] = s.customerName(h.CustomerID)
|
||||
if rj, _ := s.store.GetLatestHostReportJSON(h.CustomerID); rj != "" {
|
||||
facts[h.HostID] = sysfacts.Parse(rj)
|
||||
}
|
||||
}
|
||||
stale, reboot, notCov := view.Thresholds()
|
||||
data := map[string]interface{}{
|
||||
"Rows": buildSystemRows(lines, facts, names, stale, reboot, notCov, time.Now()),
|
||||
"Releases": view.Releases(),
|
||||
"Candidates": view.Candidates(),
|
||||
"Flash": r.URL.Query().Get("flash"),
|
||||
"FlashErr": r.URL.Query().Get("err"),
|
||||
"CSRFToken": s.getCSRFToken(r),
|
||||
}
|
||||
if err := s.templates.ExecuteTemplate(w, "system.html", data); err != nil {
|
||||
s.logger.Printf("[ERROR] system.html template: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/osupdates"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
const sysFull = `{"host":{"cpu_percent":1},"cloudflared":{"status":"running"},"system":{"pve_version":"pve-manager/9.0.11/abc",
|
||||
"kernel_version":"Linux 7.0.14-20-pve #1","vmid":9201,"facts":{"host":{"debian":"13.7","kernel_running":"7.0.14-20-pve",
|
||||
"kernel_next_boot":"7.0.2-6-pve","kernel_next_boot_source":"saved default","held":["tzdata"],"kernel_panic":0,"oops_this_boot":false,
|
||||
"crash_guard":{"armed":false,"tripped":true,"tripped_at":"2026-10-04T16:00:00Z","tripped_reason":"2 unclean boots within 60 minutes","unclean_boots_24h":2}},
|
||||
"guest":{"debian":"13.7","docker_engine":"29.8.2","containerd":"2.3.6-1~debian.13~trixie","live_restore":"on"}}}}`
|
||||
const sysPartial = `{"host":{"cpu_percent":1},"system":{"pve_version":"pve-manager/9.0.10/x","kernel_version":"Linux 7.0.2-6-pve #1","facts_error":"no running customer guest"}}`
|
||||
const sysOld = `{"host":{"cpu_percent":1}}`
|
||||
|
||||
func systemServer(t *testing.T) (*Server, *store.Store, *osupdates.Service) {
|
||||
s, st := newTestServer(t)
|
||||
for _, h := range []struct{ id, cust, body string }{{"full-1", "c-full", sysFull}, {"part-1", "c-part", sysPartial}, {"old-1", "c-old", sysOld}} {
|
||||
if err := st.UpsertHost(&store.Host{HostID: h.id, CustomerID: h.cust, APIKey: "k-" + h.id}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SaveHostReport(h.id, h.cust, []byte(h.body), store.HostReportDenorm{AgentVersion: "0.142.0", CloudflaredStatus: "running"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
svc := &osupdates.Service{Store: st, ApproveAfter: 24 * time.Hour, NightsRequired: 1}
|
||||
s.SetOSUpdateAdmin(svc)
|
||||
return s, st, svc
|
||||
}
|
||||
|
||||
func getSystem(t *testing.T, s *Server) string {
|
||||
t.Helper()
|
||||
rr := httptest.NewRecorder()
|
||||
s.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/system", nil))
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("GET /system = %d %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
return rr.Body.String()
|
||||
}
|
||||
|
||||
// Full, partial and old-agent boxes render; an unreadable value says "unknown", never empty or guessed.
|
||||
// Red-proof: render KernelRunning with no unknownCell and the partial box shows an empty cell.
|
||||
func TestSystemPage_FullPartialUnknown(t *testing.T) {
|
||||
s, _, _ := systemServer(t)
|
||||
b := getSystem(t, s)
|
||||
for _, want := range []string{"9.0.11", "7.0.14-20-pve", "29.8.2", "tzdata", "TRIPPED", "0 (stays off)",
|
||||
"9.0.10", "partial: no running customer guest", "no versions reported (agent older than v0.142.0)",
|
||||
`action="/os/ring/full-1"`, `action="/os/enabled/part-1"`, `action="/os/approve-now"`} {
|
||||
if !strings.Contains(b, want) {
|
||||
t.Errorf("System page lacks %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Count(b, ">unknown<") < 6 {
|
||||
t.Errorf("the partial and old boxes must read unknown, got %d unknown cells", strings.Count(b, ">unknown<"))
|
||||
}
|
||||
if strings.Count(b, `class="c-warn" title="the box could not read it`) < 6 {
|
||||
t.Errorf("an unknown value must be shown amber with its reason, got %d", strings.Count(b, `class="c-warn" title="the box could not read it`))
|
||||
}
|
||||
if !strings.Contains(b, `class="c-bad" title="2 unclean boots`) {
|
||||
t.Error("a tripped crash guard must be red")
|
||||
}
|
||||
}
|
||||
|
||||
// The "Approve Docker set" button appears ONLY when the rule allows it (seam rule: one render test per branch).
|
||||
func TestSystemPage_DockerButtonOnlyWhenReady(t *testing.T) {
|
||||
s, st, svc := systemServer(t)
|
||||
if strings.Contains(getSystem(t, s), `action="/os/approve-docker"`) {
|
||||
t.Fatal("button shown with no Docker candidate")
|
||||
}
|
||||
_ = st.SetOSRing("full-1", 0)
|
||||
night := func() {
|
||||
if err := svc.Ingest("full-1", osupdates.Report{RunID: time.Now().String(), Layer: "docker", Trigger: "night", Mode: "apply",
|
||||
Outcome: "nothing", Healthy: true, Installed: []osupdates.Package{{Name: "docker-ce", Version: "5:29.8.2-1", Origin: "Docker"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
night()
|
||||
if strings.Contains(getSystem(t, s), `action="/os/approve-docker"`) {
|
||||
t.Fatal("button shown after ONE night")
|
||||
}
|
||||
night()
|
||||
if !strings.Contains(getSystem(t, s), `action="/os/approve-docker"`) {
|
||||
t.Fatal("button missing after two healthy nights")
|
||||
}
|
||||
}
|
||||
|
||||
// Every button needs the operator login; a box's own API key is not one. Red-proof: route /os/ outside RequireAuth.
|
||||
func TestSystemButtons_NeedTheOperatorLogin(t *testing.T) {
|
||||
s, st, _ := systemServer(t)
|
||||
h, _ := bcrypt.GenerateFromPassword([]byte("operator-pw"), bcrypt.MinCost)
|
||||
s.configPasswordHash = string(h)
|
||||
for _, tc := range []struct{ method, path, body string }{
|
||||
{http.MethodGet, "/system", ""},
|
||||
{http.MethodPost, "/os/ring/full-1", "ring=0&return=%2Fsystem"},
|
||||
{http.MethodPost, "/os/enabled/full-1", "on=0&return=%2Fsystem"},
|
||||
{http.MethodPost, "/os/approve-now", "return=%2Fsystem"},
|
||||
{http.MethodPost, "/os/approve-docker", "return=%2Fsystem"},
|
||||
} {
|
||||
for _, auth := range []string{"", "Bearer k-full-1"} {
|
||||
req := httptest.NewRequest(tc.method, tc.path, strings.NewReader(tc.body))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("X-Requested-With", "XMLHttpRequest")
|
||||
if auth != "" {
|
||||
req.Header.Set("Authorization", auth)
|
||||
}
|
||||
rr := httptest.NewRecorder()
|
||||
s.RequireAuth(http.HandlerFunc(s.ServeHTTP)).ServeHTTP(rr, req)
|
||||
if rr.Code == http.StatusOK || rr.Code == http.StatusSeeOther {
|
||||
t.Errorf("%s %s with auth %q = %d — must be refused", tc.method, tc.path, auth, rr.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
if st.GetOSHostSettings("full-1").Enabled != true || st.GetOSHostSettings("full-1").Ring != 1 {
|
||||
t.Fatal("an unauthenticated POST changed a box")
|
||||
}
|
||||
}
|
||||
|
||||
// With the operator's session a page button changes the box and returns to the page.
|
||||
func TestSystemButtons_RedirectBackToThePage(t *testing.T) {
|
||||
s, st, _ := systemServer(t)
|
||||
h, _ := bcrypt.GenerateFromPassword([]byte("operator-pw"), bcrypt.MinCost)
|
||||
s.configPasswordHash = string(h)
|
||||
cookie, csrf := newRevealSession(t, s)
|
||||
form := url.Values{"on": {"0"}, "return": {"/system"}, "_csrf": {csrf}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/os/enabled/full-1", strings.NewReader(form.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.AddCookie(cookie)
|
||||
rr := httptest.NewRecorder()
|
||||
s.RequireAuth(http.HandlerFunc(s.ServeHTTP)).ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusSeeOther || !strings.HasPrefix(rr.Header().Get("Location"), "/system?flash=") {
|
||||
t.Fatalf("= %d %q", rr.Code, rr.Header().Get("Location"))
|
||||
}
|
||||
if st.GetOSHostSettings("full-1").Enabled {
|
||||
t.Fatal("the switch did not change")
|
||||
}
|
||||
}
|
||||
|
||||
// The Hosts page shows the Proxmox version and the running kernel (unknown for an old agent).
|
||||
func TestHostsPage_ProxmoxKernelColumn(t *testing.T) {
|
||||
s, _, _ := systemServer(t)
|
||||
rr := httptest.NewRecorder()
|
||||
s.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/hosts", nil))
|
||||
b := rr.Body.String()
|
||||
if !strings.Contains(b, "Proxmox / kernel") || !strings.Contains(b, "9.0.11") || !strings.Contains(b, "7.0.2-6-pve") {
|
||||
t.Fatalf("hosts column missing:\n%s", b[:min(len(b), 400)])
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@
|
||||
<a href="/configs" class="nav-link">Customers</a>
|
||||
<a href="/apps" class="nav-link active">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
<a href="/configs" class="nav-link">Customers</a>
|
||||
<a href="/apps" class="nav-link active">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
<a href="/configs" class="nav-link active">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
<a href="/configs" class="nav-link active">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
<a href="/configs" class="nav-link">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link active">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
<a href="/configs" class="nav-link active">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
<a href="/configs" class="nav-link">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
<a href="/configs" class="nav-link">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link active">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
<a href="/configs" class="nav-link">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link active">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
@@ -85,6 +86,7 @@
|
||||
<th>Host</th>
|
||||
<th>Customer</th>
|
||||
<th>Agent</th>
|
||||
<th>Proxmox / kernel</th>
|
||||
<th>Status</th>
|
||||
<th>Guests</th>
|
||||
<th>CPU</th>
|
||||
@@ -100,6 +102,7 @@
|
||||
<td><a href="/hosts/{{.HostID}}">{{.HostID}}</a></td>
|
||||
<td>{{if .CustomerName}}{{.CustomerName}}{{else}}{{.CustomerID}}{{end}}</td>
|
||||
<td>{{if .AgentVersion}}<code>{{.AgentVersion}}</code>{{else}}—{{end}}{{if .FloorHeld}} <span class="status-badge status-warn" title="{{.HeldReason}}">floor held</span>{{else if eq .FloorSource "declared"}} <span class="status-badge" title="served above the vouched golden: the agent requirement was declared with the floor (R-472)">floor: declared MinAgent</span>{{end}}</td>
|
||||
<td style="font-size: 0.85em;">{{.PVEVersion}}<br><span class="text-muted">{{.KernelRunning}}</span></td>
|
||||
<td><span class="status-badge {{.StatusClass}}">{{.StatusLabel}}</span></td>
|
||||
<td>{{if .HasReport}}{{.GuestRunning}}/{{.GuestTotal}}{{else}}—{{end}}</td>
|
||||
<td>{{if .HasReport}}{{formatFloat .Vitals.CPUPercent}}%{{else}}—{{end}}</td>
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
<a href="/configs" class="nav-link active">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
<a href="/configs" class="nav-link">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link">System</a>
|
||||
<a href="/offsite" class="nav-link active">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>System — Felhom Hub</title>
|
||||
<link rel="stylesheet" href="/style.css?v={{hubVersion}}">
|
||||
<style>
|
||||
.sys td, .sys th { white-space: nowrap; font-size: 0.82em; vertical-align: top; }
|
||||
.sys .grp { border-left: 2px solid var(--border, #444); }
|
||||
.c-warn { color: var(--warn); font-weight: 600; }
|
||||
.c-bad { color: var(--danger, #e5534b); font-weight: 700; }
|
||||
.sys form { display: inline; }
|
||||
.rel-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr)); gap: 0.75rem; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
{{template "icon_sprite"}}
|
||||
{{template "inline_confirm_js"}}
|
||||
<div class="container">
|
||||
<header>
|
||||
<h1>Felhom <span>Hub</span></h1>
|
||||
<nav class="nav-links">
|
||||
<a href="/" class="nav-link">Dashboard</a>
|
||||
<a href="/configs" class="nav-link">Customers</a>
|
||||
<a href="/apps" class="nav-link">Apps</a>
|
||||
<a href="/hosts" class="nav-link">Hosts</a>
|
||||
<a href="/system" class="nav-link active">System</a>
|
||||
<a href="/offsite" class="nav-link">Offsite</a>
|
||||
<a href="/configuration" class="nav-link">Configuration</a>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<h2 style="margin-bottom: 1rem;">System — versions and OS updates</h2>
|
||||
|
||||
{{if .Flash}}<div class="flash flash-success" style="margin-bottom: 1rem;">{{.Flash}}</div>{{end}}
|
||||
{{if .FlashErr}}<div class="flash flash-error" style="margin-bottom: 1rem;">{{.FlashErr}}</div>{{end}}
|
||||
|
||||
<section class="card" style="margin-bottom: 1.5rem;">
|
||||
<h3 style="margin-top: 0;">Approved releases</h3>
|
||||
<div class="rel-grid">
|
||||
{{range .Releases}}
|
||||
<div><strong>{{.Layer}}</strong>: <code>{{.ID}}</code><br>
|
||||
<span class="text-muted">{{.Packages}} packages · {{.ApprovedAt.UTC.Format "2006-01-02 15:04"}} UTC · by {{.ApprovedBy}}</span></div>
|
||||
{{else}}<div class="text-muted">No release approved yet.</div>{{end}}
|
||||
</div>
|
||||
<h3>What ring 0 runs now</h3>
|
||||
<div class="rel-grid">
|
||||
{{range .Candidates}}
|
||||
<div><strong>{{.Layer}}</strong>:
|
||||
{{if .Fingerprint}}{{.Packages}} packages, first seen {{.FirstSeen.UTC.Format "2006-01-02 15:04"}} UTC{{else}}<span class="text-muted">—</span>{{end}}<br>
|
||||
{{if .Approved}}<span class="text-muted">approved as {{.Approved}}</span>
|
||||
{{else if .Waiting}}<span class="text-muted">{{.Waiting}}</span>{{end}}
|
||||
{{if and (eq .Layer "docker") .Fingerprint (not .Approved) (eq .Waiting "")}}
|
||||
<form method="POST" action="/os/approve-docker" style="margin-top: 0.3rem;">
|
||||
<input type="hidden" name="_csrf" value="{{$.CSRFToken}}"><input type="hidden" name="return" value="/system">
|
||||
<button type="submit" class="btn btn-sm" data-confirm="Approve this Docker engine set? Ring-1 boxes take it only through a signed operator job.">Approve Docker set</button>
|
||||
</form>{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
<form method="POST" action="/os/approve-now" style="margin-top: 0.8rem;">
|
||||
<input type="hidden" name="_csrf" value="{{$.CSRFToken}}"><input type="hidden" name="return" value="/system">
|
||||
<button type="submit" class="btn btn-sm btn-danger" data-confirm="Approve the guest and host sets ring 0 runs NOW, without the 24 h + 1 night wait? Every ring-1 box installs them at its next night run.">Approve now (guest + host)</button>
|
||||
<span class="text-muted" style="font-size: 0.85em;">An urgent fix only — normally the hub approves after 24 h and one night.</span>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
{{if .Rows}}
|
||||
<section class="card" style="padding: 0; overflow-x: auto;">
|
||||
<table class="data-table sys">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Box</th><th>Ring / updates</th><th>Tunnel</th>
|
||||
<th class="grp">Proxmox</th><th>Kernel (running)</th><th>Kernel (next boot)</th><th>Debian</th><th>Felhom release</th><th>Pending</th><th>Not covered</th><th>Held</th><th>Reboot needed</th><th>kernel.panic</th><th>Oops</th><th>Crash restarts 24 h</th><th>Crash guard</th>
|
||||
<th class="grp">Guest Debian</th><th>Felhom release</th><th>Pending</th><th>Restart needed</th>
|
||||
<th class="grp">Docker</th><th>containerd</th><th>live-restore</th><th>Docker release</th>
|
||||
<th class="grp">Last OS leg</th>
|
||||
</tr>
|
||||
<tr class="text-muted"><th></th><th></th><th></th><th class="grp" colspan="13">host</th><th class="grp" colspan="4">guest</th><th class="grp" colspan="4">Docker engine</th><th class="grp"></th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{range .Rows}}
|
||||
<tr>
|
||||
<td><a href="/hosts/{{.HostID}}">{{.HostID}}</a>{{if .CustomerName}}<br><span class="text-muted">{{.CustomerName}}</span>{{end}}
|
||||
{{if .FactsNote}}<br><span class="c-warn" style="font-weight: normal;">{{.FactsNote}}</span>{{end}}</td>
|
||||
<td>
|
||||
ring {{.Ring}}
|
||||
<form method="POST" action="/os/ring/{{.HostID}}">
|
||||
<input type="hidden" name="_csrf" value="{{$.CSRFToken}}"><input type="hidden" name="return" value="/system">
|
||||
{{if eq .Ring 0}}<input type="hidden" name="ring" value="1"><button type="submit" class="btn btn-sm btn-outline" data-confirm="Make {{.HostID}} a normal (ring 1) box? It then installs only approved releases.">→ normal</button>
|
||||
{{else}}<input type="hidden" name="ring" value="0"><button type="submit" class="btn btn-sm btn-outline" data-confirm="Make {{.HostID}} a DEMO (ring 0) box? It then installs every new fix first and takes unsigned Docker steps if its root-owned ring-0 mark allows.">→ demo</button>{{end}}
|
||||
</form><br>
|
||||
updates {{if .Enabled}}<strong>ON</strong>{{else}}<span class="c-warn">OFF</span>{{end}}
|
||||
<form method="POST" action="/os/enabled/{{.HostID}}">
|
||||
<input type="hidden" name="_csrf" value="{{$.CSRFToken}}"><input type="hidden" name="return" value="/system">
|
||||
{{if .Enabled}}<input type="hidden" name="on" value="0"><button type="submit" class="btn btn-sm btn-outline" data-confirm="Switch OS updates OFF for {{.HostID}}? It keeps reporting and installs nothing.">switch off</button>
|
||||
{{else}}<input type="hidden" name="on" value="1"><button type="submit" class="btn btn-sm btn-outline">switch on</button>{{end}}
|
||||
</form>
|
||||
</td>
|
||||
{{template "sys_cell" .Tunnel}}
|
||||
<td class="grp {{if .PVE.Class}}c-{{.PVE.Class}}{{end}}">{{.PVE.Text}}</td>
|
||||
{{template "sys_cell" .KernelRunning}}{{template "sys_cell" .KernelNextBoot}}{{template "sys_cell" .HostDebian}}
|
||||
{{template "sys_cell" .HostRelease}}{{template "sys_cell" .HostPending}}{{template "sys_cell" .HostNotCovered}}
|
||||
{{template "sys_cell" .Held}}{{template "sys_cell" .RebootSince}}{{template "sys_cell" .KernelPanic}}{{template "sys_cell" .Oops}}
|
||||
{{template "sys_cell" .CrashRestarts24h}}{{template "sys_cell" .Guard}}
|
||||
<td class="grp {{if .GuestDebian.Class}}c-{{.GuestDebian.Class}}{{end}}">{{.GuestDebian.Text}}</td>
|
||||
{{template "sys_cell" .GuestRelease}}{{template "sys_cell" .GuestPending}}{{template "sys_cell" .GuestRestart}}
|
||||
<td class="grp {{if .Engine.Class}}c-{{.Engine.Class}}{{end}}">{{.Engine.Text}}</td>
|
||||
{{template "sys_cell" .Containerd}}{{template "sys_cell" .LiveRestore}}{{template "sys_cell" .DockerRelease}}
|
||||
<td class="grp {{if .LastLeg.Class}}c-{{.LastLeg.Class}}{{end}}" title="{{.LastLeg.Title}}">{{.LastLeg.Text}}</td>
|
||||
</tr>
|
||||
{{end}}
|
||||
</tbody>
|
||||
</table>
|
||||
</section>
|
||||
<p class="text-muted" style="font-size: 0.85em;">Amber: worth a look. Red: an operator alarm fires (`08` §6.3). "unknown": the box could not read the value — never a guess.</p>
|
||||
{{else}}
|
||||
<div class="empty-state"><p>No boxes yet.</p></div>
|
||||
{{end}}
|
||||
|
||||
<footer style="margin-top: 2rem; color: var(--text-muted); font-size: 0.8rem; text-align: center;">
|
||||
Felhom Hub <span style="font-family: var(--font-mono)">{{hubVersion}}</span>
|
||||
</footer>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
{{define "sys_cell"}}<td{{if .Class}} class="c-{{.Class}}"{{end}}{{if .Title}} title="{{.Title}}"{{end}}>{{.Text}}</td>{{end}}
|
||||
Reference in New Issue
Block a user