From 36c72138f16ca2fbfa6acfae855c829c441c472c Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 13 Jul 2026 14:52:04 +0200 Subject: [PATCH] hub: dead-host roll-up honesty - customer status folds worst expected host Customer status (dashboard row, /configs list, detail header + strip) is now worst(controllerDerived, hostStatusOf(each expected host)) via the ONE staleness definition (Server.hostStatus, hosts.go - shared with the HostStalenessChecker; no second threshold). Any host down/stale caps the customer at WARN with a cause chip naming the host ("host down: "); pending (never-reported) hosts worsen only once the customer has reported (onboarding exclusion). The three previously-inlined controller-status chains collapse into controllerStatus() (rollup.go). Display + derivation only - checker alerting untouched. Live shape pinned (drill-1 / Peti cluster): host down 23h + controller report minutes old rendered a GREEN row - TestRollup_DeadHostMasking now fails that exact outcome. Red-proof: short-circuiting foldHostStatus to controller-only flips C + two D subtests red ("dashboard row is GREEN over a 23h-dead host"). --- hub/internal/web/configs.go | 43 ++-- hub/internal/web/render_test.go | 1 + hub/internal/web/rollup.go | 78 +++++++ hub/internal/web/rollup_test.go | 194 ++++++++++++++++++ hub/internal/web/server.go | 19 +- hub/internal/web/templates/configs.html | 1 + .../web/templates/customer_unified.html | 2 + hub/internal/web/templates/dashboard.html | 1 + 8 files changed, 305 insertions(+), 34 deletions(-) create mode 100644 hub/internal/web/rollup.go create mode 100644 hub/internal/web/rollup_test.go diff --git a/hub/internal/web/configs.go b/hub/internal/web/configs.go index 0fa94b1..1744ccf 100644 --- a/hub/internal/web/configs.go +++ b/hub/internal/web/configs.go @@ -53,6 +53,7 @@ type customerListEntry struct { HasConfig bool IsBlocked bool OverallStatus string // ok, warn, down, disabled, pending, "" if no reports + HostCause string // v0.53.0 roll-up: "" or "host down|stale|pending: " ControllerVersion string TimeSinceReport time.Duration ConfigCreatedAt time.Time @@ -95,20 +96,13 @@ func (s *Server) handleConfigList(w http.ResponseWriter, r *http.Request) { } for _, c := range customers { - status := "ok" - if c.HealthStatus == "disabled" { - status = "disabled" - } else if c.TimeSinceReport > time.Hour { - status = "down" - } else if c.TimeSinceReport > 30*time.Minute || c.HealthStatus == "warn" { - status = "warn" - } else if c.HealthStatus == "fail" { - status = "down" - } + // Controller-derived status + the v0.53.0 dead-host roll-up (rollup.go). + status, hostCause := s.foldHostStatus(c.CustomerID, controllerStatus(&c), true) if entry, ok := merged[c.CustomerID]; ok { // Config exists — enrich with report data entry.OverallStatus = status + entry.HostCause = hostCause entry.ControllerVersion = c.ControllerVersion entry.TimeSinceReport = c.TimeSinceReport if entry.CustomerName == "" { @@ -120,12 +114,21 @@ func (s *Server) handleConfigList(w http.ResponseWriter, r *http.Request) { CustomerID: c.CustomerID, CustomerName: c.CustomerName, OverallStatus: status, + HostCause: hostCause, ControllerVersion: c.ControllerVersion, TimeSinceReport: c.TimeSinceReport, } } } + // Config-only customers (no reports yet): the roll-up still applies — a down/stale host + // must not hide behind the muted no-reports dash; only never-reported hosts are excluded. + for _, e := range merged { + if e.OverallStatus == "" { + e.OverallStatus, e.HostCause = s.foldHostStatus(e.CustomerID, "", false) + } + } + // Phase 2 floor: resolve each customer's effective floor (override else global) + below-floor flag. globalFloor := s.store.GetGlobalMinControllerVersion() for _, e := range merged { @@ -198,21 +201,15 @@ func (s *Server) handleCustomerUnified(w http.ResponseWriter, r *http.Request, c json.Unmarshal([]byte(cfg.ConfigJSON), &overrides) } - // Overall status + // Overall status: controller-derived + the v0.53.0 dead-host roll-up (rollup.go). The + // blocked override stays LAST (administrative state wins the token); the host cause chip + // renders regardless so the header says WHICH host is the problem. overallStatus := "pending" if customer != nil { - if customer.HealthStatus == "disabled" { - overallStatus = "disabled" - } else if customer.TimeSinceReport > time.Hour { - overallStatus = "down" - } else if customer.TimeSinceReport > 30*time.Minute || customer.HealthStatus == "warn" { - overallStatus = "warn" - } else if customer.HealthStatus == "fail" { - overallStatus = "down" - } else { - overallStatus = "ok" - } + overallStatus = controllerStatus(customer) } + var hostCause string + overallStatus, hostCause = s.foldHostStatus(customerID, overallStatus, customer != nil) if cfg != nil && cfg.Status == "blocked" { overallStatus = "blocked" } @@ -290,6 +287,7 @@ func (s *Server) handleCustomerUnified(w http.ResponseWriter, r *http.Request, c Customer *store.CustomerSummary Report map[string]interface{} OverallStatus string + HostCause string // v0.53.0 roll-up: "" or "host down|stale|pending: " LatestVersion string UpdateAvailable bool @@ -386,6 +384,7 @@ func (s *Server) handleCustomerUnified(w http.ResponseWriter, r *http.Request, c Customer: customer, Report: report, OverallStatus: overallStatus, + HostCause: hostCause, LatestVersion: latestVersion, UpdateAvailable: updateAvailable, diff --git a/hub/internal/web/render_test.go b/hub/internal/web/render_test.go index b44fe11..d5ba7ed 100644 --- a/hub/internal/web/render_test.go +++ b/hub/internal/web/render_test.go @@ -86,6 +86,7 @@ func TestTemplates_DashboardCriticalBadge(t *testing.T) { type dashboardCustomer struct { store.CustomerSummary OverallStatus string + HostCause string BackupAge string EventCriticals int EventErrors int diff --git a/hub/internal/web/rollup.go b/hub/internal/web/rollup.go new file mode 100644 index 0000000..b82e554 --- /dev/null +++ b/hub/internal/web/rollup.go @@ -0,0 +1,78 @@ +package web + +// Dead-host roll-up honesty (v0.53.0, drill-1 observation; operator ruling 2026-07-13): a +// customer's status may never look better than its worst expected host. The customer roll-up +// derives from CONTROLLER reports, which reach the hub independently of the host agent — so a +// host DOWN for 23 hours hid behind a green customer row as long as the guest kept reporting +// (the live Peti-cluster shape: proxmox1 down 23h, fresh reports through proxmox2). +// +// foldHostStatus worsens the controller-derived status with per-host staleness via +// (*Server).hostStatus — THE single staleness definition (hosts.go; the same thresholds the +// HostStalenessChecker alerts on — no second definition anywhere). Display + derivation only: +// checker alerting is untouched. + +import ( + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// controllerStatus is the controller-report-derived customer status — the pre-roll-up chain the +// dashboard, the /configs list and the customer detail all inlined verbatim; this is now the +// ONE copy. Behavior-preserving: the branch order (incl. fail-after-warn) is the historical one. +func controllerStatus(c *store.CustomerSummary) string { + switch { + case c.HealthStatus == "disabled": + return "disabled" + case c.TimeSinceReport > time.Hour: + return "down" + case c.TimeSinceReport > 30*time.Minute || c.HealthStatus == "warn": + return "warn" + case c.HealthStatus == "fail": + return "down" + default: + return "ok" + } +} + +// hostFoldRank orders host states by badness for the worst-host pick. "ok" ranks 0 (never folds). +var hostFoldRank = map[string]int{"down": 3, "stale": 2, "pending": 1} + +// hostFoldLabel is the operator-facing cause chip prefix per worst-host state. +var hostFoldLabel = map[string]string{"down": "host down", "stale": "host stale", "pending": "host pending"} + +// foldHostStatus folds the customer's expected hosts into a controller-derived status: +// worst(controllerDerived, hostStatusOf(each host)). Any host down/stale caps the customer at +// WARN (a green row over a dead host is the masking bug); the returned cause names the state +// AND the host ("host down: ") so the detail header says WHICH host. "pending" hosts +// (enrolled, never reported) worsen only after initial onboarding — customerHasReported=false +// (the customer has never reported) excludes them, a half-installed box is not an incident. +// Statuses worse than warn (down) and administrative ones (disabled/blocked) keep their own +// token; the cause chip still surfaces the host signal. Read errors degrade to the unfolded +// status — the page must render. +func (s *Server) foldHostStatus(customerID, base string, customerHasReported bool) (status, cause string) { + hosts, err := s.store.ListHostsByCustomer(customerID) + if err != nil { + s.logger.Printf("[ERROR] roll-up: ListHostsByCustomer %s: %v", customerID, err) + return base, "" + } + worst, worstHost := "", "" + for i := range hosts { + hs := s.hostStatus(hosts[i].LastReportAt) + if hs == "pending" && !customerHasReported { + continue + } + if hostFoldRank[hs] > hostFoldRank[worst] { + worst, worstHost = hs, hosts[i].HostID + } + } + if worst == "" { + return base, "" + } + cause = hostFoldLabel[worst] + ": " + worstHost + // The fold worsens, never improves: ok / pending / no-report ("") cap at warn. + if base == "ok" || base == "pending" || base == "" { + return "warn", cause + } + return base, cause +} diff --git a/hub/internal/web/rollup_test.go b/hub/internal/web/rollup_test.go new file mode 100644 index 0000000..65789ac --- /dev/null +++ b/hub/internal/web/rollup_test.go @@ -0,0 +1,194 @@ +package web + +// v0.53.0 dead-host roll-up honesty (drill-1 masking observation; operator ruling 2026-07-13). +// Scenario C is the LIVE Peti-cluster shape: a host down 23h while the controller keeps +// reporting through the internet — pre-fix the customer row rendered GREEN (the wrong outcome +// these tests pin). Scenario D bounds the fold: all-ok is a no-regression pass-through, a stale +// host warns with its own chip, and pending hosts worsen only after onboarding. +// RED-PROOF (§10 C): short-circuiting foldHostStatus to `return base, ""` (controller-only +// derivation) → the C assertions fail with the green row / missing chip visible. + +import ( + "database/sql" + "fmt" + "io" + "log" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// newRollupServer builds a server whose store DB path is known, so tests can backdate host +// reports over a second connection (the monitor host_staleness_test pattern). +func newRollupServer(t *testing.T) (*Server, *store.Store, *sql.DB) { + t.Helper() + path := filepath.Join(t.TempDir(), "t.db") + st, err := store.New(path, log.New(io.Discard, "", 0)) + if err != nil { + t.Fatalf("store.New: %v", err) + } + t.Cleanup(func() { st.Close() }) + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + t.Cleanup(func() { db.Close() }) + // staleThreshold 30m → host "stale" past 30m, "down" past 60m (the single definition). + s := New(st, "", "", "test", 30*time.Minute, log.New(io.Discard, "", 0)) + return s, st, db +} + +func backdateHost(t *testing.T, db *sql.DB, hostID string, minutesAgo int) { + t.Helper() + if _, err := db.Exec(`UPDATE hosts SET last_report_at = datetime('now', ?) WHERE host_id = ?`, + fmt.Sprintf("-%d minutes", minutesAgo), hostID); err != nil { + t.Fatal(err) + } +} + +func renderDashboard(t *testing.T, s *Server) string { + t.Helper() + rr := httptest.NewRecorder() + s.handleDashboard(rr, httptest.NewRequest("GET", "/", nil)) + if rr.Code != 200 { + t.Fatalf("dashboard = %d", rr.Code) + } + return rr.Body.String() +} + +func renderCustomer(t *testing.T, s *Server, customerID string) string { + t.Helper() + rr := httptest.NewRecorder() + s.handleCustomerUnified(rr, httptest.NewRequest("GET", "/customers/"+customerID, nil), customerID) + if rr.Code != 200 { + t.Fatalf("customer detail = %d", rr.Code) + } + return rr.Body.String() +} + +// Scenario C — the Peti shape: proxmox1 down 23h, fresh controller report → the customer row +// must read WARN with the cause chip naming the host; the detail header must say WHICH host. +// WRONG outcome (pre-fix): a green row. +func TestRollup_DeadHostMasking(t *testing.T) { + s, st, db := newRollupServer(t) + if err := st.SaveCustomerConfig(&store.CustomerConfig{ + CustomerID: "acme", CustomerName: "Acme", APIKey: "k", RetrievalPassword: "p", + }); err != nil { + t.Fatal(err) + } + // Fresh controller report (the guest reports hub-direct, independent of the host agent). + if err := st.SaveReport("acme", []byte(`{"customer_id":"acme","customer_name":"Acme"}`)); err != nil { + t.Fatal(err) + } + // The host: reported once, then silent for 23h → "down" by THE definition. + if err := st.UpsertHost(&store.Host{HostID: "proxmox1", CustomerID: "acme", APIKey: "hk"}); err != nil { + t.Fatal(err) + } + if err := st.SaveHostReport("proxmox1", "acme", []byte(`{}`), store.HostReportDenorm{}); err != nil { + t.Fatal(err) + } + backdateHost(t, db, "proxmox1", 23*60) + + body := renderDashboard(t, s) + if !strings.Contains(body, "host down: proxmox1") { + t.Error("dashboard row lacks the cause chip \"host down: proxmox1\"") + } + if strings.Contains(body, "status-badge-ok") { + t.Error("dashboard row is GREEN over a 23h-dead host — the exact masking bug") + } + if !strings.Contains(body, `status-badge-warn">host down: proxmox1`) { + t.Error("cause chip not rendered as a warn badge") + } + + // Detail header: WHICH host. + detail := renderCustomer(t, s, "acme") + if !strings.Contains(detail, "host down: proxmox1") { + t.Error("customer detail header does not name the down host") + } +} + +// Scenario D — boundaries: all-ok pass-through (no regression), stale-host warn chip, and the +// onboarding rule for pending hosts. +func TestRollup_Boundaries(t *testing.T) { + t.Run("all hosts ok leaves controller status untouched", func(t *testing.T) { + s, st, _ := newRollupServer(t) + if err := st.SaveReport("acme", []byte(`{"customer_id":"acme"}`)); err != nil { + t.Fatal(err) + } + if err := st.UpsertHost(&store.Host{HostID: "h-ok", CustomerID: "acme", APIKey: "hk"}); err != nil { + t.Fatal(err) + } + if err := st.SaveHostReport("h-ok", "acme", []byte(`{}`), store.HostReportDenorm{}); err != nil { + t.Fatal(err) + } + body := renderDashboard(t, s) + if !strings.Contains(body, "status-badge-ok") { + t.Error("healthy customer + healthy host must stay OK") + } + if strings.Contains(body, "host down") || strings.Contains(body, "host stale") || strings.Contains(body, "host pending") { + t.Error("cause chip rendered with every host ok") + } + }) + + t.Run("single stale host caps at warn with stale chip", func(t *testing.T) { + s, st, db := newRollupServer(t) + if err := st.SaveReport("acme", []byte(`{"customer_id":"acme"}`)); err != nil { + t.Fatal(err) + } + if err := st.UpsertHost(&store.Host{HostID: "h-stale", CustomerID: "acme", APIKey: "hk"}); err != nil { + t.Fatal(err) + } + if err := st.SaveHostReport("h-stale", "acme", []byte(`{}`), store.HostReportDenorm{}); err != nil { + t.Fatal(err) + } + backdateHost(t, db, "h-stale", 45) // between stale (30m) and down (60m) + body := renderDashboard(t, s) + if !strings.Contains(body, "host stale: h-stale") { + t.Error("stale-host cause chip missing") + } + if strings.Contains(body, "status-badge-ok") { + t.Error("customer stayed green over a stale host") + } + }) + + t.Run("pending host during onboarding does not worsen", func(t *testing.T) { + s, st, _ := newRollupServer(t) + // Config-only customer (never reported) + freshly-minted host (never reported). + if err := st.SaveCustomerConfig(&store.CustomerConfig{ + CustomerID: "newbie", CustomerName: "Newbie", APIKey: "k", RetrievalPassword: "p", + }); err != nil { + t.Fatal(err) + } + if err := st.UpsertHost(&store.Host{HostID: "h-new", CustomerID: "newbie", APIKey: "hk"}); err != nil { + t.Fatal(err) + } + body := renderDashboard(t, s) + if !strings.Contains(body, "status-badge-pending") { + t.Error("onboarding customer must render PENDING") + } + if strings.Contains(body, "host pending") { + t.Error("a never-reported host worsened a never-reported customer (onboarding exclusion violated)") + } + }) + + t.Run("pending host after onboarding worsens", func(t *testing.T) { + s, st, _ := newRollupServer(t) + if err := st.SaveReport("acme", []byte(`{"customer_id":"acme"}`)); err != nil { + t.Fatal(err) + } + if err := st.UpsertHost(&store.Host{HostID: "h-silent", CustomerID: "acme", APIKey: "hk"}); err != nil { + t.Fatal(err) + } + body := renderDashboard(t, s) + if !strings.Contains(body, "host pending: h-silent") { + t.Error("a never-reported host must worsen a LIVE customer (post-onboarding)") + } + if strings.Contains(body, "status-badge-ok") { + t.Error("customer stayed green over a never-reported host") + } + }) +} diff --git a/hub/internal/web/server.go b/hub/internal/web/server.go index aec0e8c..6ff52a4 100644 --- a/hub/internal/web/server.go +++ b/hub/internal/web/server.go @@ -611,6 +611,7 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) { type dashboardCustomer struct { store.CustomerSummary OverallStatus string // "ok", "warn", "down", "pending" + HostCause string // v0.53.0 roll-up: "" or "host down|stale|pending: " BackupAge string EventCriticals int EventErrors int @@ -629,18 +630,9 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) { seen[c.CustomerID] = true dc := dashboardCustomer{CustomerSummary: c} - // Determine overall status - if c.HealthStatus == "disabled" { - dc.OverallStatus = "disabled" - } else if c.TimeSinceReport > time.Hour { - dc.OverallStatus = "down" - } else if c.TimeSinceReport > 30*time.Minute || c.HealthStatus == "warn" { - dc.OverallStatus = "warn" - } else if c.HealthStatus == "fail" { - dc.OverallStatus = "down" - } else { - dc.OverallStatus = "ok" - } + // Controller-derived status + the v0.53.0 dead-host roll-up (rollup.go): a customer + // may never look better than its worst expected host. + dc.OverallStatus, dc.HostCause = s.foldHostStatus(c.CustomerID, controllerStatus(&c), true) // Backup age if c.BackupLastSnapshot != nil { @@ -672,6 +664,9 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) { OverallStatus: "pending", BackupAge: "–", } + // Roll-up for the never-reported customer too: a down/stale host worsens even during + // onboarding — only never-reported ("pending") hosts are excluded here. + dc.OverallStatus, dc.HostCause = s.foldHostStatus(cfg.CustomerID, dc.OverallStatus, false) data = append(data, dc) } diff --git a/hub/internal/web/templates/configs.html b/hub/internal/web/templates/configs.html index 9d9e75e..5a7ab75 100644 --- a/hub/internal/web/templates/configs.html +++ b/hub/internal/web/templates/configs.html @@ -63,6 +63,7 @@ {{if eq .OverallStatus "ok"}}OK{{else if eq .OverallStatus "warn"}}WARN{{else if eq .OverallStatus "down"}}DOWN{{else if eq .OverallStatus "disabled"}}PAUSED{{else if eq .OverallStatus "pending"}}PENDING{{else}}{{.OverallStatus}}{{end}} + {{if .HostCause}}{{.HostCause}}{{end}} {{else}} — {{end}} diff --git a/hub/internal/web/templates/customer_unified.html b/hub/internal/web/templates/customer_unified.html index 538f941..87cbad0 100644 --- a/hub/internal/web/templates/customer_unified.html +++ b/hub/internal/web/templates/customer_unified.html @@ -25,6 +25,7 @@

{{if .CustomerName}}{{.CustomerName}}{{else}}{{.CustomerID}}{{end}} + {{if .HostCause}}{{.HostCause}}{{end}}

{{if .HasReports}}

@@ -69,6 +70,7 @@

{{if .CustomerName}}{{.CustomerName}}{{else}}{{.CustomerID}}{{end}} {{.OverallStatus}} + {{if .HostCause}}{{.HostCause}}{{end}} {{if .HasReports}} Controller {{.Customer.ControllerVersion}} Last report {{timeAgo .Customer.ReceivedAt}} diff --git a/hub/internal/web/templates/dashboard.html b/hub/internal/web/templates/dashboard.html index 810e685..a0f7dde 100644 --- a/hub/internal/web/templates/dashboard.html +++ b/hub/internal/web/templates/dashboard.html @@ -54,6 +54,7 @@ {{if eq .OverallStatus "ok"}}OK{{else if eq .OverallStatus "warn"}}WARN{{else if eq .OverallStatus "disabled"}}PAUSED{{else if eq .OverallStatus "pending"}}PENDING{{else}}DOWN{{end}} + {{if .HostCause}}{{.HostCause}}{{end}} {{if eq .OverallStatus "pending"}}—{{else}}{{if gt (add (add .EventCriticals .EventErrors) .EventWarnings) 0}}{{if gt .EventCriticals 0}}{{.EventCriticals}}{{end}}{{if gt .EventErrors 0}}{{.EventErrors}}{{end}}{{if gt .EventWarnings 0}}{{.EventWarnings}}{{end}}{{else}}—{{end}}{{end}} {{if eq .OverallStatus "pending"}}—{{else}}{{timeAgo .ReceivedAt}}{{end}}