hub 0.166.0 source: the clock-ahead tunnel alarm wg_tunnel_stale/ok (R-1204, operator-only, edge-triggered); D21 + D4 proof evidence

hub/internal/api/handler.go

@@ -772,7 +772,9 @@ type hostReportPayload struct {
 		HostSwapMB  *int64 `json:"host_swap_mb"`
 		GuestSwapMB *int64 `json:"guest_swap_mb"`
 	} `json:"host"`
-	Guests []struct {
+	// Wireguard (R-1204): the box's own view of its tunnel to ep0. nil = an agent with no tunnel stanza.
+	Wireguard *hostWireguard `json:"wireguard"`
+	Guests    []struct {
 		VMID              int    `json:"vmid"`
 		Name              string `json:"name"`
 		Status            string `json:"status"`
@@ -994,6 +996,7 @@ func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
 	}
 	h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
 	h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB)
+	h.checkWGTunnel(hostID, custID, rep.Wireguard)
 	h.checkCrash(hostID, custID, body)
 	h.checkDeliberateOff(hostID, custID, &rep)
 

hub/internal/api/host_tunnel.go

@@ -0,0 +1,97 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// R-1204 (rehearsal 2026-10-20 B6 #8). A box that runs once with its clock AHEAD stamps a WireGuard handshake in the
+// future; ep0 then refuses every newer handshake (WireGuard's replay protection: the TAI64N timestamp must be newer
+// than the last one accepted from that peer), so the box's tunnel to ep0 — its PBS whole-box copies and the
+// operator's door — stays dead until the real date passes the wrong one, while the box itself reports normally.
+// Nothing said why. The hub now tells the OPERATOR once per host, edge-triggered from the host's newest tunnel event
+// (the host_swap shape), and once when it is healthy again. Operator-only (notify.operatorOnlyEvents): the household
+// cannot fix it. The repair (re-key the peer, or ep0 forgets the stored timestamp) is a separate decision.
+const (
+	eventWGTunnelStale = "wg_tunnel_stale"
+	eventWGTunnelOK    = "wg_tunnel_ok"
+	// wgTunnelStaleAfterS: a healthy tunnel shakes hands at least every ~2 minutes (measured 2026-10-22 over 24 h of
+	// reports on three boxes: 113–124 s at most); six hours is far past any reboot or network blip.
+	wgTunnelStaleAfterS = 6 * 3600
+)
+
+// hostWireguard is the agent's `wireguard` stanza (felhom-agent hub.WireguardStatus).
+type hostWireguard struct {
+	Registered        bool   `json:"registered"`
+	Active            bool   `json:"active"`
+	LastHandshakeAgeS *int64 `json:"last_handshake_age_s"`
+	// HandshakeFutureS (agent ≥ the R-1204 commit): > 0 when the latest handshake lies in the future by the box's clock.
+	HandshakeFutureS *int64 `json:"handshake_future_s"`
+}
+
+// wgTunnelStale judges one report: (stale, the cause in plain words). A box without a registered, active tunnel, or
+// one that sent no age, is not judged.
+func wgTunnelStale(w *hostWireguard) (bool, string) {
+	if w == nil || !w.Registered || !w.Active {
+		return false, ""
+	}
+	if w.HandshakeFutureS != nil && *w.HandshakeFutureS > 0 {
+		return true, fmt.Sprintf("its last handshake is stamped %s in the FUTURE by the box's own clock — the clock ran ahead once, and ep0 refuses every newer handshake until that time passes", humanDuration(*w.HandshakeFutureS))
+	}
+	if w.LastHandshakeAgeS != nil && *w.LastHandshakeAgeS > wgTunnelStaleAfterS {
+		return true, fmt.Sprintf("it has not shaken hands for %s while the box reports — the box's clock may have run ahead (then ep0 refuses it), or ep0 dropped the peer", humanDuration(*w.LastHandshakeAgeS))
+	}
+	return false, ""
+}
+
+func humanDuration(s int64) string {
+	switch {
+	case s >= 2*86400:
+		return fmt.Sprintf("%d days", s/86400)
+	case s >= 2*3600:
+		return fmt.Sprintf("%d hours", s/3600)
+	default:
+		return fmt.Sprintf("%d minutes", s/60)
+	}
+}
+
+func (h *Handler) checkWGTunnel(hostID, custID string, w *hostWireguard) {
+	if w == nil || custID == "" || !w.Registered || !w.Active || (w.LastHandshakeAgeS == nil && w.HandshakeFutureS == nil) {
+		return // not judged: no tunnel, or an agent that sent nothing to judge by
+	}
+	stale, cause := wgTunnelStale(w)
+	prev, err := h.store.LatestHostTunnelEvent(custID, hostID)
+	if err != nil {
+		h.logger.Printf("[WARN] host %s tunnel check: reading the last tunnel event failed: %v", hostID, err)
+		return
+	}
+	wasStale := prev != nil && prev.EventType == eventWGTunnelStale
+	var typ, sev, msg string
+	switch {
+	case stale && !wasStale:
+		typ, sev = eventWGTunnelStale, "warning"
+		msg = fmt.Sprintf("Host %s: its WireGuard tunnel to the off-site server (ep0) is down — %s. Its whole-box copies to ep0 "+
+			"and the operator's door do not work until it is repaired (R-1204). The household cannot fix this.", hostID, cause)
+	case !stale && wasStale:
+		typ, sev = eventWGTunnelOK, "info"
+		msg = fmt.Sprintf("Host %s: its WireGuard tunnel to the off-site server (ep0) shakes hands again.", hostID)
+	default:
+		return
+	}
+	h.logger.Printf("[WARN] host %s: %s", hostID, typ)
+	det := map[string]any{"host_id": hostID}
+	if w.LastHandshakeAgeS != nil {
+		det["last_handshake_age_s"] = *w.LastHandshakeAgeS
+	}
+	if w.HandshakeFutureS != nil {
+		det["handshake_future_s"] = *w.HandshakeFutureS
+	}
+	details, _ := json.Marshal(det)
+	if _, eerr := h.store.SaveEvent(custID, typ, sev, msg, string(details), "hub"); eerr != nil {
+		h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, eerr)
+		return // the audit row first: a mail without its event row lies
+	}
+	if h.dispatcher != nil {
+		go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub")
+	}
+}

hub/internal/api/host_tunnel_test.go

@@ -0,0 +1,78 @@
+package api
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+// R-1204: the box's tunnel to ep0 — a future-stamped handshake (the clock ran ahead) or no handshake for 6 h raises
+// wg_tunnel_stale ONCE (edge-triggered from the host's newest tunnel event), and wg_tunnel_ok once when healthy; a
+// healthy tunnel, an inactive one and an older agent are not judged.
+// RED-PROOF: with checkWGTunnel not called from the host-report handler the first count fails (0 events); with
+// wgTunnelStale ignoring handshake_future_s the clock case fails — seen failing 2026-10-22.
+func tunnelReportBody(hostID, wg string) string {
+	b := validReportBody(hostID)
+	i := strings.LastIndex(b, "}")
+	return b[:i] + `,"wireguard":` + wg + b[i:]
+}
+
+func TestR1204_TheClockAheadTunnelAlarmsOnceAndClears(t *testing.T) {
+	h, st, _ := newTestHandler(t)
+	st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ckey", RetrievalPassword: "p"})
+	st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "HKEY"})
+	post := func(wg string) {
+		t.Helper()
+		if rr := do(h, http.MethodPost, "/host-report", "HKEY", tunnelReportBody("h1", wg)); rr.Code != http.StatusOK {
+			t.Fatalf("host-report = %d %s", rr.Code, rr.Body.String())
+		}
+	}
+	count := func(typ string) int {
+		evs, _ := st.GetEventsByType("c1", typ, time.Time{})
+		return len(evs)
+	}
+	post(`{"registered":true,"active":true,"last_handshake_age_s":60}`) // healthy
+	post(`{"registered":true,"active":false}`)                          // service down: not judged here
+	if count(eventWGTunnelStale) != 0 {
+		t.Fatal("a healthy or inactive tunnel must not alarm")
+	}
+	post(`{"registered":true,"active":true,"last_handshake_age_s":0,"handshake_future_s":63072000}`) // ran 2 years ahead
+	post(`{"registered":true,"active":true,"last_handshake_age_s":0,"handshake_future_s":63071100}`)
+	if n := count(eventWGTunnelStale); n != 1 {
+		t.Fatalf("wg_tunnel_stale = %d, want exactly 1 (edge-triggered)", n)
+	}
+	evs, _ := st.GetEventsByType("c1", eventWGTunnelStale, time.Time{})
+	if !strings.Contains(evs[0].Message, "Host h1:") || !strings.Contains(evs[0].Message, "FUTURE") {
+		t.Fatalf("the alarm must name the host and the clock cause: %q", evs[0].Message)
+	}
+	post(`{"registered":true,"active":true,"last_handshake_age_s":45}`)
+	post(`{"registered":true,"active":true,"last_handshake_age_s":50}`)
+	if count(eventWGTunnelOK) != 1 {
+		t.Fatalf("wg_tunnel_ok = %d, want 1", count(eventWGTunnelOK))
+	}
+	post(`{"registered":true,"active":true,"last_handshake_age_s":30000}`) // 8 h 20 min with no handshake
+	if count(eventWGTunnelStale) != 2 {
+		t.Fatal("six hours without a handshake while reporting must alarm again")
+	}
+}
+
+func TestR1204_WhatIsJudged(t *testing.T) {
+	i := func(v int64) *int64 { return &v }
+	for _, c := range []struct {
+		w    *hostWireguard
+		want bool
+	}{
+		{nil, false},
+		{&hostWireguard{Registered: true, Active: true, LastHandshakeAgeS: i(120)}, false},
+		{&hostWireguard{Registered: true, Active: true, LastHandshakeAgeS: i(6*3600 + 1)}, true},
+		{&hostWireguard{Registered: true, Active: true, LastHandshakeAgeS: i(0), HandshakeFutureS: i(3600)}, true},
+		{&hostWireguard{Registered: false, Active: true, LastHandshakeAgeS: i(99999)}, false},
+	} {
+		if got, _ := wgTunnelStale(c.w); got != c.want {
+			t.Errorf("%+v: stale=%t, want %t", c.w, got, c.want)
+		}
+	}
+}