hub: dead-host roll-up honesty - customer status folds worst expected host

Customer status (dashboard row, /configs list, detail header + strip) is
now worst(controllerDerived, hostStatusOf(each expected host)) via the ONE
staleness definition (Server.hostStatus, hosts.go - shared with the
HostStalenessChecker; no second threshold). Any host down/stale caps the
customer at WARN with a cause chip naming the host ("host down: <id>");
pending (never-reported) hosts worsen only once the customer has reported
(onboarding exclusion). The three previously-inlined controller-status
chains collapse into controllerStatus() (rollup.go). Display + derivation
only - checker alerting untouched.

Live shape pinned (drill-1 / Peti cluster): host down 23h + controller
report minutes old rendered a GREEN row - TestRollup_DeadHostMasking now
fails that exact outcome. Red-proof: short-circuiting foldHostStatus to
controller-only flips C + two D subtests red ("dashboard row is GREEN
over a 23h-dead host").

hub/internal/web/configs.go

@@ -53,6 +53,7 @@ type customerListEntry struct {
 	HasConfig         bool
 	IsBlocked         bool
 	OverallStatus     string // ok, warn, down, disabled, pending, "" if no reports
+	HostCause         string // v0.53.0 roll-up: "" or "host down|stale|pending: <id>"
 	ControllerVersion string
 	TimeSinceReport   time.Duration
 	ConfigCreatedAt   time.Time
@@ -95,20 +96,13 @@ func (s *Server) handleConfigList(w http.ResponseWriter, r *http.Request) {
 	}
 
 	for _, c := range customers {
-		status := "ok"
-		if c.HealthStatus == "disabled" {
-			status = "disabled"
-		} else if c.TimeSinceReport > time.Hour {
-			status = "down"
-		} else if c.TimeSinceReport > 30*time.Minute || c.HealthStatus == "warn" {
-			status = "warn"
-		} else if c.HealthStatus == "fail" {
-			status = "down"
-		}
+		// Controller-derived status + the v0.53.0 dead-host roll-up (rollup.go).
+		status, hostCause := s.foldHostStatus(c.CustomerID, controllerStatus(&c), true)
 
 		if entry, ok := merged[c.CustomerID]; ok {
 			// Config exists — enrich with report data
 			entry.OverallStatus = status
+			entry.HostCause = hostCause
 			entry.ControllerVersion = c.ControllerVersion
 			entry.TimeSinceReport = c.TimeSinceReport
 			if entry.CustomerName == "" {
@@ -120,12 +114,21 @@ func (s *Server) handleConfigList(w http.ResponseWriter, r *http.Request) {
 				CustomerID:        c.CustomerID,
 				CustomerName:      c.CustomerName,
 				OverallStatus:     status,
+				HostCause:         hostCause,
 				ControllerVersion: c.ControllerVersion,
 				TimeSinceReport:   c.TimeSinceReport,
 			}
 		}
 	}
 
+	// Config-only customers (no reports yet): the roll-up still applies — a down/stale host
+	// must not hide behind the muted no-reports dash; only never-reported hosts are excluded.
+	for _, e := range merged {
+		if e.OverallStatus == "" {
+			e.OverallStatus, e.HostCause = s.foldHostStatus(e.CustomerID, "", false)
+		}
+	}
+
 	// Phase 2 floor: resolve each customer's effective floor (override else global) + below-floor flag.
 	globalFloor := s.store.GetGlobalMinControllerVersion()
 	for _, e := range merged {
@@ -198,21 +201,15 @@ func (s *Server) handleCustomerUnified(w http.ResponseWriter, r *http.Request, c
 		json.Unmarshal([]byte(cfg.ConfigJSON), &overrides)
 	}
 
-	// Overall status
+	// Overall status: controller-derived + the v0.53.0 dead-host roll-up (rollup.go). The
+	// blocked override stays LAST (administrative state wins the token); the host cause chip
+	// renders regardless so the header says WHICH host is the problem.
 	overallStatus := "pending"
 	if customer != nil {
-		if customer.HealthStatus == "disabled" {
-			overallStatus = "disabled"
-		} else if customer.TimeSinceReport > time.Hour {
-			overallStatus = "down"
-		} else if customer.TimeSinceReport > 30*time.Minute || customer.HealthStatus == "warn" {
-			overallStatus = "warn"
-		} else if customer.HealthStatus == "fail" {
-			overallStatus = "down"
-		} else {
-			overallStatus = "ok"
-		}
+		overallStatus = controllerStatus(customer)
 	}
+	var hostCause string
+	overallStatus, hostCause = s.foldHostStatus(customerID, overallStatus, customer != nil)
 	if cfg != nil && cfg.Status == "blocked" {
 		overallStatus = "blocked"
 	}
@@ -290,6 +287,7 @@ func (s *Server) handleCustomerUnified(w http.ResponseWriter, r *http.Request, c
 		Customer      *store.CustomerSummary
 		Report        map[string]interface{}
 		OverallStatus string
+		HostCause     string // v0.53.0 roll-up: "" or "host down|stale|pending: <id>"
 
 		LatestVersion   string
 		UpdateAvailable bool
@@ -386,6 +384,7 @@ func (s *Server) handleCustomerUnified(w http.ResponseWriter, r *http.Request, c
 		Customer:      customer,
 		Report:        report,
 		OverallStatus: overallStatus,
+		HostCause:     hostCause,
 
 		LatestVersion:   latestVersion,
 		UpdateAvailable: updateAvailable,

hub/internal/web/render_test.go

@@ -86,6 +86,7 @@ func TestTemplates_DashboardCriticalBadge(t *testing.T) {
 	type dashboardCustomer struct {
 		store.CustomerSummary
 		OverallStatus  string
+		HostCause      string
 		BackupAge      string
 		EventCriticals int
 		EventErrors    int

hub/internal/web/rollup.go

@@ -0,0 +1,78 @@
+package web
+
+// Dead-host roll-up honesty (v0.53.0, drill-1 observation; operator ruling 2026-07-13): a
+// customer's status may never look better than its worst expected host. The customer roll-up
+// derives from CONTROLLER reports, which reach the hub independently of the host agent — so a
+// host DOWN for 23 hours hid behind a green customer row as long as the guest kept reporting
+// (the live Peti-cluster shape: proxmox1 down 23h, fresh reports through proxmox2).
+//
+// foldHostStatus worsens the controller-derived status with per-host staleness via
+// (*Server).hostStatus — THE single staleness definition (hosts.go; the same thresholds the
+// HostStalenessChecker alerts on — no second definition anywhere). Display + derivation only:
+// checker alerting is untouched.
+
+import (
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// controllerStatus is the controller-report-derived customer status — the pre-roll-up chain the
+// dashboard, the /configs list and the customer detail all inlined verbatim; this is now the
+// ONE copy. Behavior-preserving: the branch order (incl. fail-after-warn) is the historical one.
+func controllerStatus(c *store.CustomerSummary) string {
+	switch {
+	case c.HealthStatus == "disabled":
+		return "disabled"
+	case c.TimeSinceReport > time.Hour:
+		return "down"
+	case c.TimeSinceReport > 30*time.Minute || c.HealthStatus == "warn":
+		return "warn"
+	case c.HealthStatus == "fail":
+		return "down"
+	default:
+		return "ok"
+	}
+}
+
+// hostFoldRank orders host states by badness for the worst-host pick. "ok" ranks 0 (never folds).
+var hostFoldRank = map[string]int{"down": 3, "stale": 2, "pending": 1}
+
+// hostFoldLabel is the operator-facing cause chip prefix per worst-host state.
+var hostFoldLabel = map[string]string{"down": "host down", "stale": "host stale", "pending": "host pending"}
+
+// foldHostStatus folds the customer's expected hosts into a controller-derived status:
+// worst(controllerDerived, hostStatusOf(each host)). Any host down/stale caps the customer at
+// WARN (a green row over a dead host is the masking bug); the returned cause names the state
+// AND the host ("host down: <id>") so the detail header says WHICH host. "pending" hosts
+// (enrolled, never reported) worsen only after initial onboarding — customerHasReported=false
+// (the customer has never reported) excludes them, a half-installed box is not an incident.
+// Statuses worse than warn (down) and administrative ones (disabled/blocked) keep their own
+// token; the cause chip still surfaces the host signal. Read errors degrade to the unfolded
+// status — the page must render.
+func (s *Server) foldHostStatus(customerID, base string, customerHasReported bool) (status, cause string) {
+	hosts, err := s.store.ListHostsByCustomer(customerID)
+	if err != nil {
+		s.logger.Printf("[ERROR] roll-up: ListHostsByCustomer %s: %v", customerID, err)
+		return base, ""
+	}
+	worst, worstHost := "", ""
+	for i := range hosts {
+		hs := s.hostStatus(hosts[i].LastReportAt)
+		if hs == "pending" && !customerHasReported {
+			continue
+		}
+		if hostFoldRank[hs] > hostFoldRank[worst] {
+			worst, worstHost = hs, hosts[i].HostID
+		}
+	}
+	if worst == "" {
+		return base, ""
+	}
+	cause = hostFoldLabel[worst] + ": " + worstHost
+	// The fold worsens, never improves: ok / pending / no-report ("") cap at warn.
+	if base == "ok" || base == "pending" || base == "" {
+		return "warn", cause
+	}
+	return base, cause
+}

hub/internal/web/rollup_test.go

@@ -0,0 +1,194 @@
+package web
+
+// v0.53.0 dead-host roll-up honesty (drill-1 masking observation; operator ruling 2026-07-13).
+// Scenario C is the LIVE Peti-cluster shape: a host down 23h while the controller keeps
+// reporting through the internet — pre-fix the customer row rendered GREEN (the wrong outcome
+// these tests pin). Scenario D bounds the fold: all-ok is a no-regression pass-through, a stale
+// host warns with its own chip, and pending hosts worsen only after onboarding.
+// RED-PROOF (§10 C): short-circuiting foldHostStatus to `return base, ""` (controller-only
+// derivation) → the C assertions fail with the green row / missing chip visible.
+
+import (
+	"database/sql"
+	"fmt"
+	"io"
+	"log"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// newRollupServer builds a server whose store DB path is known, so tests can backdate host
+// reports over a second connection (the monitor host_staleness_test pattern).
+func newRollupServer(t *testing.T) (*Server, *store.Store, *sql.DB) {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "t.db")
+	st, err := store.New(path, log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatalf("store.New: %v", err)
+	}
+	t.Cleanup(func() { st.Close() })
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		t.Fatalf("sql.Open: %v", err)
+	}
+	t.Cleanup(func() { db.Close() })
+	// staleThreshold 30m → host "stale" past 30m, "down" past 60m (the single definition).
+	s := New(st, "", "", "test", 30*time.Minute, log.New(io.Discard, "", 0))
+	return s, st, db
+}
+
+func backdateHost(t *testing.T, db *sql.DB, hostID string, minutesAgo int) {
+	t.Helper()
+	if _, err := db.Exec(`UPDATE hosts SET last_report_at = datetime('now', ?) WHERE host_id = ?`,
+		fmt.Sprintf("-%d minutes", minutesAgo), hostID); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func renderDashboard(t *testing.T, s *Server) string {
+	t.Helper()
+	rr := httptest.NewRecorder()
+	s.handleDashboard(rr, httptest.NewRequest("GET", "/", nil))
+	if rr.Code != 200 {
+		t.Fatalf("dashboard = %d", rr.Code)
+	}
+	return rr.Body.String()
+}
+
+func renderCustomer(t *testing.T, s *Server, customerID string) string {
+	t.Helper()
+	rr := httptest.NewRecorder()
+	s.handleCustomerUnified(rr, httptest.NewRequest("GET", "/customers/"+customerID, nil), customerID)
+	if rr.Code != 200 {
+		t.Fatalf("customer detail = %d", rr.Code)
+	}
+	return rr.Body.String()
+}
+
+// Scenario C — the Peti shape: proxmox1 down 23h, fresh controller report → the customer row
+// must read WARN with the cause chip naming the host; the detail header must say WHICH host.
+// WRONG outcome (pre-fix): a green row.
+func TestRollup_DeadHostMasking(t *testing.T) {
+	s, st, db := newRollupServer(t)
+	if err := st.SaveCustomerConfig(&store.CustomerConfig{
+		CustomerID: "acme", CustomerName: "Acme", APIKey: "k", RetrievalPassword: "p",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	// Fresh controller report (the guest reports hub-direct, independent of the host agent).
+	if err := st.SaveReport("acme", []byte(`{"customer_id":"acme","customer_name":"Acme"}`)); err != nil {
+		t.Fatal(err)
+	}
+	// The host: reported once, then silent for 23h → "down" by THE definition.
+	if err := st.UpsertHost(&store.Host{HostID: "proxmox1", CustomerID: "acme", APIKey: "hk"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.SaveHostReport("proxmox1", "acme", []byte(`{}`), store.HostReportDenorm{}); err != nil {
+		t.Fatal(err)
+	}
+	backdateHost(t, db, "proxmox1", 23*60)
+
+	body := renderDashboard(t, s)
+	if !strings.Contains(body, "host down: proxmox1") {
+		t.Error("dashboard row lacks the cause chip \"host down: proxmox1\"")
+	}
+	if strings.Contains(body, "status-badge-ok") {
+		t.Error("dashboard row is GREEN over a 23h-dead host — the exact masking bug")
+	}
+	if !strings.Contains(body, `status-badge-warn">host down: proxmox1`) {
+		t.Error("cause chip not rendered as a warn badge")
+	}
+
+	// Detail header: WHICH host.
+	detail := renderCustomer(t, s, "acme")
+	if !strings.Contains(detail, "host down: proxmox1") {
+		t.Error("customer detail header does not name the down host")
+	}
+}
+
+// Scenario D — boundaries: all-ok pass-through (no regression), stale-host warn chip, and the
+// onboarding rule for pending hosts.
+func TestRollup_Boundaries(t *testing.T) {
+	t.Run("all hosts ok leaves controller status untouched", func(t *testing.T) {
+		s, st, _ := newRollupServer(t)
+		if err := st.SaveReport("acme", []byte(`{"customer_id":"acme"}`)); err != nil {
+			t.Fatal(err)
+		}
+		if err := st.UpsertHost(&store.Host{HostID: "h-ok", CustomerID: "acme", APIKey: "hk"}); err != nil {
+			t.Fatal(err)
+		}
+		if err := st.SaveHostReport("h-ok", "acme", []byte(`{}`), store.HostReportDenorm{}); err != nil {
+			t.Fatal(err)
+		}
+		body := renderDashboard(t, s)
+		if !strings.Contains(body, "status-badge-ok") {
+			t.Error("healthy customer + healthy host must stay OK")
+		}
+		if strings.Contains(body, "host down") || strings.Contains(body, "host stale") || strings.Contains(body, "host pending") {
+			t.Error("cause chip rendered with every host ok")
+		}
+	})
+
+	t.Run("single stale host caps at warn with stale chip", func(t *testing.T) {
+		s, st, db := newRollupServer(t)
+		if err := st.SaveReport("acme", []byte(`{"customer_id":"acme"}`)); err != nil {
+			t.Fatal(err)
+		}
+		if err := st.UpsertHost(&store.Host{HostID: "h-stale", CustomerID: "acme", APIKey: "hk"}); err != nil {
+			t.Fatal(err)
+		}
+		if err := st.SaveHostReport("h-stale", "acme", []byte(`{}`), store.HostReportDenorm{}); err != nil {
+			t.Fatal(err)
+		}
+		backdateHost(t, db, "h-stale", 45) // between stale (30m) and down (60m)
+		body := renderDashboard(t, s)
+		if !strings.Contains(body, "host stale: h-stale") {
+			t.Error("stale-host cause chip missing")
+		}
+		if strings.Contains(body, "status-badge-ok") {
+			t.Error("customer stayed green over a stale host")
+		}
+	})
+
+	t.Run("pending host during onboarding does not worsen", func(t *testing.T) {
+		s, st, _ := newRollupServer(t)
+		// Config-only customer (never reported) + freshly-minted host (never reported).
+		if err := st.SaveCustomerConfig(&store.CustomerConfig{
+			CustomerID: "newbie", CustomerName: "Newbie", APIKey: "k", RetrievalPassword: "p",
+		}); err != nil {
+			t.Fatal(err)
+		}
+		if err := st.UpsertHost(&store.Host{HostID: "h-new", CustomerID: "newbie", APIKey: "hk"}); err != nil {
+			t.Fatal(err)
+		}
+		body := renderDashboard(t, s)
+		if !strings.Contains(body, "status-badge-pending") {
+			t.Error("onboarding customer must render PENDING")
+		}
+		if strings.Contains(body, "host pending") {
+			t.Error("a never-reported host worsened a never-reported customer (onboarding exclusion violated)")
+		}
+	})
+
+	t.Run("pending host after onboarding worsens", func(t *testing.T) {
+		s, st, _ := newRollupServer(t)
+		if err := st.SaveReport("acme", []byte(`{"customer_id":"acme"}`)); err != nil {
+			t.Fatal(err)
+		}
+		if err := st.UpsertHost(&store.Host{HostID: "h-silent", CustomerID: "acme", APIKey: "hk"}); err != nil {
+			t.Fatal(err)
+		}
+		body := renderDashboard(t, s)
+		if !strings.Contains(body, "host pending: h-silent") {
+			t.Error("a never-reported host must worsen a LIVE customer (post-onboarding)")
+		}
+		if strings.Contains(body, "status-badge-ok") {
+			t.Error("customer stayed green over a never-reported host")
+		}
+	})
+}

hub/internal/web/server.go

@@ -611,6 +611,7 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
 	type dashboardCustomer struct {
 		store.CustomerSummary
 		OverallStatus  string // "ok", "warn", "down", "pending"
+		HostCause      string // v0.53.0 roll-up: "" or "host down|stale|pending: <id>"
 		BackupAge      string
 		EventCriticals int
 		EventErrors    int
@@ -629,18 +630,9 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
 		seen[c.CustomerID] = true
 		dc := dashboardCustomer{CustomerSummary: c}
 
-		// Determine overall status
-		if c.HealthStatus == "disabled" {
-			dc.OverallStatus = "disabled"
-		} else if c.TimeSinceReport > time.Hour {
-			dc.OverallStatus = "down"
-		} else if c.TimeSinceReport > 30*time.Minute || c.HealthStatus == "warn" {
-			dc.OverallStatus = "warn"
-		} else if c.HealthStatus == "fail" {
-			dc.OverallStatus = "down"
-		} else {
-			dc.OverallStatus = "ok"
-		}
+		// Controller-derived status + the v0.53.0 dead-host roll-up (rollup.go): a customer
+		// may never look better than its worst expected host.
+		dc.OverallStatus, dc.HostCause = s.foldHostStatus(c.CustomerID, controllerStatus(&c), true)
 
 		// Backup age
 		if c.BackupLastSnapshot != nil {
@@ -672,6 +664,9 @@ func (s *Server) handleDashboard(w http.ResponseWriter, r *http.Request) {
 			OverallStatus: "pending",
 			BackupAge:     "–",
 		}
+		// Roll-up for the never-reported customer too: a down/stale host worsens even during
+		// onboarding — only never-reported ("pending") hosts are excluded here.
+		dc.OverallStatus, dc.HostCause = s.foldHostStatus(cfg.CustomerID, dc.OverallStatus, false)
 		data = append(data, dc)
 	}
 

hub/internal/web/templates/configs.html

@@ -63,6 +63,7 @@
                         <span class="status-badge status-badge-{{.OverallStatus}}">
                             {{if eq .OverallStatus "ok"}}OK{{else if eq .OverallStatus "warn"}}WARN{{else if eq .OverallStatus "down"}}DOWN{{else if eq .OverallStatus "disabled"}}PAUSED{{else if eq .OverallStatus "pending"}}PENDING{{else}}{{.OverallStatus}}{{end}}
                         </span>
+                        {{if .HostCause}}<span class="status-badge status-badge-warn">{{.HostCause}}</span>{{end}}
                         {{else}}
                         <span class="text-muted">—</span>
                         {{end}}

hub/internal/web/templates/customer_unified.html

@@ -25,6 +25,7 @@
             <h1>
                 <span class="status-dot status-dot-{{statusColor .OverallStatus}}"></span>
                 {{if .CustomerName}}{{.CustomerName}}{{else}}{{.CustomerID}}{{end}}
+                {{if .HostCause}}<span class="status-badge status-badge-warn">{{.HostCause}}</span>{{end}}
             </h1>
             {{if .HasReports}}
             <p class="subtitle">
@@ -69,6 +70,7 @@
         <div class="summary-strip">
             <span class="strip-name">{{if .CustomerName}}{{.CustomerName}}{{else}}{{.CustomerID}}{{end}}</span>
             <span class="strip-item"><span class="status-dot status-dot-{{statusColor .OverallStatus}}"></span> {{.OverallStatus}}</span>
+            {{if .HostCause}}<span class="strip-item"><span class="status-badge status-badge-warn">{{.HostCause}}</span></span>{{end}}
             {{if .HasReports}}
             <span class="strip-item">Controller <code>{{.Customer.ControllerVersion}}</code></span>
             <span class="strip-item">Last report {{timeAgo .Customer.ReceivedAt}}</span>

hub/internal/web/templates/dashboard.html

@@ -54,6 +54,7 @@
                         <span class="status-badge status-badge-{{.OverallStatus}}">
                             {{if eq .OverallStatus "ok"}}OK{{else if eq .OverallStatus "warn"}}WARN{{else if eq .OverallStatus "disabled"}}PAUSED{{else if eq .OverallStatus "pending"}}PENDING{{else}}DOWN{{end}}
                         </span>
+                        {{if .HostCause}}<span class="status-badge status-badge-warn">{{.HostCause}}</span>{{end}}
                     </td>
                     <td>{{if eq .OverallStatus "pending"}}—{{else}}{{if gt (add (add .EventCriticals .EventErrors) .EventWarnings) 0}}{{if gt .EventCriticals 0}}<span class="severity-badge severity-critical">{{.EventCriticals}}</span>{{end}}{{if gt .EventErrors 0}}<span class="severity-badge severity-error">{{.EventErrors}}</span>{{end}}{{if gt .EventWarnings 0}}<span class="severity-badge severity-warning">{{.EventWarnings}}</span>{{end}}{{else}}<span class="text-muted">—</span>{{end}}{{end}}</td>
                     <td>{{if eq .OverallStatus "pending"}}—{{else}}{{timeAgo .ReceivedAt}}{{end}}</td>