hub v0.131.0: the tunnel status is true (R-841: three states, tunnel_down after two not_running reports, unknown never alarms); OS updates per layer (guest/host separate approved sets, host candidate leaves out kernel/boot/firmware, host_release in the box block), the fleet view and four hourly operator alarms (11 §8 steps 3+4); red-proofs in audits/os-host-lane-2026-10-04
gates / gates (push) Successful in 30s
gates / gates (push) Successful in 30s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -689,7 +689,8 @@ type hostReportPayload struct {
|
||||
RestoreTests []hostRestoreTest `json:"restore_tests"` // slice 6
|
||||
PBSSnapshots []hostPBSSnapshot `json:"pbs_snapshots"` // slice 6 Phase B
|
||||
Cloudflared struct {
|
||||
Status string `json:"status"`
|
||||
Status string `json:"status"` // agent >= 0.141.0: running | not_running | unknown (older: active | inactive | …)
|
||||
Detail string `json:"detail,omitempty"`
|
||||
} `json:"cloudflared"`
|
||||
// DR recipe — the agent's storage/guest/PBS half (secret-free). RawMessage = stored verbatim,
|
||||
// ignore-unknown (forward-compat). Persisted to dr_recipe, assembled with the controller half.
|
||||
@@ -861,6 +862,8 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
|
||||
|
||||
for _, g := range rep.Guests {
|
||||
status := g.Status
|
||||
if status == "" {
|
||||
@@ -2835,3 +2838,39 @@ func (h *Handler) handleAssetFile(w http.ResponseWriter, r *http.Request, filena
|
||||
|
||||
h.assetsMgr.ServeFile(w, r, filename)
|
||||
}
|
||||
|
||||
// Tunnel alarm (R-841, hub v0.131.0). The agent (>= 0.141.0) reports the box's tunnel as running | not_running |
|
||||
// unknown, read from the cloudflared container's own readiness check. `not_running` in TWO reports in a row — more
|
||||
// than one report cycle — is an operator alarm (`tunnel_down`, error); the first `running` after that is
|
||||
// `tunnel_recovered`. `unknown` never alarms and breaks a run of not_running (R-96 rule 3: a probe that could not
|
||||
// ask is not evidence of down). Pinned by TestTunnelAlarm_*.
|
||||
const (
|
||||
eventTunnelDown = "tunnel_down"
|
||||
eventTunnelRecovered = "tunnel_recovered"
|
||||
)
|
||||
|
||||
func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
|
||||
st, err := h.store.RecentCloudflaredStatuses(hostID, 3)
|
||||
if err != nil || len(st) < 2 {
|
||||
return
|
||||
}
|
||||
down := func(i int) bool { return i < len(st) && st[i] == "not_running" }
|
||||
var typ, sev, msg string
|
||||
switch {
|
||||
case down(0) && down(1) && !down(2):
|
||||
typ, sev = eventTunnelDown, "error"
|
||||
msg = fmt.Sprintf("The tunnel of %s is NOT running in two reports in a row (%s). The box is not reachable from outside its home.", hostID, detail)
|
||||
case st[0] == "running" && down(1) && down(2):
|
||||
typ, sev = eventTunnelRecovered, "info"
|
||||
msg = fmt.Sprintf("The tunnel of %s is running again.", hostID)
|
||||
default:
|
||||
return
|
||||
}
|
||||
h.logger.Printf("[WARN] host %s tunnel: %s (%s)", hostID, typ, detail)
|
||||
details, _ := json.Marshal(map[string]any{"host_id": hostID, "status": status, "detail": detail})
|
||||
if _, eerr := h.store.SaveEvent(custID, typ, sev, msg, string(details), "hub"); eerr != nil {
|
||||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, eerr)
|
||||
} else if h.dispatcher != nil {
|
||||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user