hub v0.131.0: the tunnel status is true (R-841: three states, tunnel_down after two not_running reports, unknown never alarms); OS updates per layer (guest/host separate approved sets, host candidate leaves out kernel/boot/firmware, host_release in the box block), the fleet view and four hourly operator alarms (11 §8 steps 3+4); red-proofs in audits/os-host-lane-2026-10-04
gates / gates (push) Successful in 30s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-04 13:05:00 +02:00
parent 0ed2e8de4c
commit 88b0a2e761
24 changed files with 1254 additions and 149 deletions
+40 -1
View File
@@ -689,7 +689,8 @@ type hostReportPayload struct {
RestoreTests []hostRestoreTest `json:"restore_tests"` // slice 6
PBSSnapshots []hostPBSSnapshot `json:"pbs_snapshots"` // slice 6 Phase B
Cloudflared struct {
Status string `json:"status"`
Status string `json:"status"` // agent >= 0.141.0: running | not_running | unknown (older: active | inactive | …)
Detail string `json:"detail,omitempty"`
} `json:"cloudflared"`
// DR recipe — the agent's storage/guest/PBS half (secret-free). RawMessage = stored verbatim,
// ignore-unknown (forward-compat). Persisted to dr_recipe, assembled with the controller half.
@@ -861,6 +862,8 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
return
}
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
for _, g := range rep.Guests {
status := g.Status
if status == "" {
@@ -2835,3 +2838,39 @@ func (h *Handler) handleAssetFile(w http.ResponseWriter, r *http.Request, filena
h.assetsMgr.ServeFile(w, r, filename)
}
// Tunnel alarm (R-841, hub v0.131.0). The agent (>= 0.141.0) reports the box's tunnel as running | not_running |
// unknown, read from the cloudflared container's own readiness check. `not_running` in TWO reports in a row — more
// than one report cycle — is an operator alarm (`tunnel_down`, error); the first `running` after that is
// `tunnel_recovered`. `unknown` never alarms and breaks a run of not_running (R-96 rule 3: a probe that could not
// ask is not evidence of down). Pinned by TestTunnelAlarm_*.
const (
eventTunnelDown = "tunnel_down"
eventTunnelRecovered = "tunnel_recovered"
)
func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
st, err := h.store.RecentCloudflaredStatuses(hostID, 3)
if err != nil || len(st) < 2 {
return
}
down := func(i int) bool { return i < len(st) && st[i] == "not_running" }
var typ, sev, msg string
switch {
case down(0) && down(1) && !down(2):
typ, sev = eventTunnelDown, "error"
msg = fmt.Sprintf("The tunnel of %s is NOT running in two reports in a row (%s). The box is not reachable from outside its home.", hostID, detail)
case st[0] == "running" && down(1) && down(2):
typ, sev = eventTunnelRecovered, "info"
msg = fmt.Sprintf("The tunnel of %s is running again.", hostID)
default:
return
}
h.logger.Printf("[WARN] host %s tunnel: %s (%s)", hostID, typ, detail)
details, _ := json.Marshal(map[string]any{"host_id": hostID, "status": status, "detail": detail})
if _, eerr := h.store.SaveEvent(custID, typ, sev, msg, string(details), "hub"); eerr != nil {
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, eerr)
} else if h.dispatcher != nil {
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub")
}
}