D30 (R-1204): a handshake stamped in the future for 15 min gives a new WireGuard key, registered on the existing peer path (the hub re-keys in place; its full-list push swaps the key on ep0 — nothing new on ep0); old key kept as private.key.prev; red-proved; unreleased

CHANGELOG.md

@@ -1,3 +1,18 @@
+## Unreleased (2026-10-22/23 night) — the tunnel heals itself after a clock that ran ahead (D30, R-1204) — COMMITTED, NOT RELEASED
+
+- **D30 (operator 2026-10-22: the agent makes a new key and registers it on the existing peer path).** When the latest
+  handshake stays stamped in the future for 15 minutes (the R-1204 sign: ep0 keeps the future timestamp and refuses
+  every newer handshake from that key), the agent makes a new WireGuard key pair, keeps the old private key as
+  `wg/private.key.prev` (0600 — the escrow may hold it), and registers the new public key through `POST /hosts/<id>/wg`.
+  The hub re-keys the peer IN PLACE (same /32, generation bump) and its full-list push to ep0 drops the old key and adds
+  the new one — nothing new runs on ep0. A new key is a new peer for ep0, with no stored timestamp, so the next
+  handshake is accepted. At most once an hour; not while the tunnel unit is down; never while a re-register is already
+  in flight. A failed registration is retried by the existing re-key-in-place path. Logs carry public keys only.
+  Tests `TestD30_*` (4), red-proved. A host-loss DR that installs the escrowed (older) key registers it again, and the
+  hub re-keys in place back to it — the same path.
+- **Known gap, unchanged (filed):** after a reboot the kernel forgets the future stamp (latest handshake 0), so neither
+  this re-key nor the hub's alarm sees a box whose clock was corrected and which then restarted.
+
 ## Unreleased (2026-10-22 night) — the tunnel's future-stamped handshake is reported (R-1204) — COMMITTED, NOT RELEASED (no agent release tonight); root SSH by key only (R-982, D4) is a READY PATCH, not on main
 
 - **R-982 / D4 — a ready patch, NOT on main.** Built and proven the same night (felhom.eu

internal/wgtunnel/manager.go

@@ -99,6 +99,7 @@ type Manager struct {
 	resolver  Resolver
 
 	staleAfter time.Duration // handshake-age threshold for the re-resolve watchdog (doc 06 §4.2)
+	rekeyAfter time.Duration // D30: how long a future-stamped handshake stays before the re-key (rekey.go)
 
 	mu              sync.Mutex
 	nextRegAt       time.Time
@@ -111,6 +112,10 @@ type Manager struct {
 	resolveFailLogged bool       // DNS-failure log throttle (reset on the next success)
 	staleSameIPLogged bool       // "stale but IP unchanged" log throttle (endpoint-down, not re-IP)
 
+	// D30 (rekey.go): when the latest handshake was first seen stamped in the future, and the last re-key.
+	futureSince time.Time
+	lastRekey   time.Time
+
 	// conv is the CACHED tunnel-convergence snapshot (v0.90.0, R-28 fast-tick source), refreshed at
 	// the end of every Apply (the tunnel's own cadence) so the fast-tick reads it WITHOUT execing
 	// `wg`/`systemctl` per tick. wgBlockDesired = a wireguard block for THIS key is desired;
@@ -156,6 +161,7 @@ func NewManager(runner proxmox.Runner, registrar Registrar, stateDir string, log
 		now:        time.Now,
 		resolver:   netResolver{},
 		staleAfter: 180 * time.Second,
+		rekeyAfter: rekeyAfterDefault,
 	}
 }
 
@@ -600,6 +606,10 @@ func (m *Manager) Watchdog(ctx context.Context, fetched bool, block *hub.WireWir
 	if !fetched || block == nil || !m.lastResolvedIP.IsValid() {
 		return // nothing applied to watch, or no desired block
 	}
+	// D30: a handshake stamped in the future that stays → a new key on the existing peer path (rekey.go).
+	if m.clockAheadRekeyLocked(ctx, block) {
+		return
+	}
 	if !m.handshakeStaleLocked(ctx) {
 		return // healthy → no DNS, no action (the load-bearing negative)
 	}

internal/wgtunnel/rekey.go

@@ -0,0 +1,122 @@
+package wgtunnel
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
+)
+
+// D30 (operator 2026-10-22, R-1204): the tunnel heals itself after the box ran with its clock ahead.
+//
+// WireGuard refuses a handshake whose timestamp is not newer than the last one it accepted from that key (replay
+// protection). A box whose clock ran ahead handshook with a future timestamp; ep0 keeps it, and once the clock is
+// right again ep0 refuses every new handshake from that key — until the box's clock passes the stored stamp. The
+// sign on the box: its own latest handshake stays stamped in the future (R-1204's handshake_future_s).
+//
+// The repair: when that sign stays for rekeyAfter, the agent makes a NEW key pair and registers the new public key on
+// the existing path (POST /hosts/<id>/wg → the hub re-keys the peer IN PLACE: same /32, new key, generation bump,
+// and the wgsync push of the FULL peer list to ep0 drops the old key and adds the new one). A new key is a new peer
+// for ep0, with no stored timestamp — the next handshake is accepted. Nothing new runs on ep0.
+//
+// The previous private key is kept as private.key.prev (0600): the escrow ceremony may hold it (a host-loss DR that
+// installs it registers it again, and the hub re-keys in place back to it — the same path).
+// Pinned by TestD30_* (rekey_test.go).
+
+const (
+	// rekeyAfter: how long the latest handshake must stay in the future before the re-key. A working tunnel
+	// handshakes every two minutes, which ends the sign within minutes of the clock being right.
+	rekeyAfterDefault = 15 * time.Minute
+	// rekeyMinGap bounds the re-key to once an hour, whatever the clock does.
+	rekeyMinGap = time.Hour
+	prevKeyName = "private.key.prev"
+)
+
+// clockAheadRekeyLocked watches for a handshake stamped in the future and re-keys when it stays. It reports whether
+// it re-keyed (the caller then skips the rest of its tick). Caller holds m.mu.
+func (m *Manager) clockAheadRekeyLocked(ctx context.Context, block *hub.WireWireguard) bool {
+	if m.keyBroken || !m.isActive(ctx) {
+		m.futureSince = time.Time{}
+		return false
+	}
+	out, _, err := m.runner.Run(ctx, "wg", "show", Iface, "latest-handshakes")
+	if err != nil {
+		return false
+	}
+	now := m.now()
+	fut, ok := handshakeFuture(string(out), now)
+	if !ok {
+		if !m.futureSince.IsZero() {
+			m.logger.Info("wgtunnel: the handshake is no longer stamped in the future — no re-key needed")
+		}
+		m.futureSince = time.Time{}
+		return false
+	}
+	if m.futureSince.IsZero() {
+		m.futureSince = now
+		m.logger.Warn("wgtunnel: the latest handshake is stamped in the future — this box's clock ran ahead, and the endpoint refuses its newer handshakes; re-keying if it stays",
+			"future_s", fut, "rekey_after", m.rekeyAfter.String())
+		return false
+	}
+	if now.Sub(m.futureSince) < m.rekeyAfter {
+		return false
+	}
+	if !m.lastRekey.IsZero() && now.Sub(m.lastRekey) < rekeyMinGap {
+		return false
+	}
+	pub, _, err := EnsureKey(m.stateDir)
+	if err != nil || block.Pubkey != pub {
+		return false // a re-register is already in flight, or the key is not readable — never re-key over that
+	}
+	newPub, err := rotateKey(m.stateDir)
+	if err != nil {
+		m.logger.Error("wgtunnel: re-key after a future handshake FAILED — the old key stays", "err", err)
+		m.lastRekey = now // not again before rekeyMinGap
+		return false
+	}
+	m.lastRekey, m.futureSince = now, time.Time{}
+	m.logger.Warn("wgtunnel: RE-KEYED after a handshake stamped in the future (D30) — registering the new key; the hub re-keys the peer in place and the endpoint takes the new key",
+		"old_pubkey", pub, "new_pubkey", newPub, "future_s", fut)
+	m.nextRegAt = time.Time{} // register now, not after an older backoff
+	m.registerLocked(ctx, newPub)
+	return true
+}
+
+// rotateKey makes a new key pair: the new private key is written next to the old one, the old one is kept as
+// private.key.prev, then the new one takes its place (renames in one directory). Returns the new PUBLIC key only.
+func rotateKey(stateDir string) (string, error) {
+	dir := filepath.Join(stateDir, "wg")
+	cur := filepath.Join(dir, keyFileName)
+	if _, err := os.Stat(cur); err != nil {
+		return "", fmt.Errorf("no current key: %w", err)
+	}
+	priv := make([]byte, 32)
+	if _, err := rand.Read(priv); err != nil {
+		return "", fmt.Errorf("entropy: %w", err)
+	}
+	clampPrivateKey(priv)
+	pub, err := derivePublic(priv)
+	if err != nil {
+		return "", err
+	}
+	tmp := filepath.Join(dir, keyFileName+".new")
+	if err := os.WriteFile(tmp, []byte(base64.StdEncoding.EncodeToString(priv)+"\n"), 0o600); err != nil {
+		return "", fmt.Errorf("writing the new key: %w", err)
+	}
+	if err := os.Rename(cur, filepath.Join(dir, prevKeyName)); err != nil {
+		os.Remove(tmp)
+		return "", fmt.Errorf("keeping the old key: %w", err)
+	}
+	if err := os.Rename(tmp, cur); err != nil {
+		// put the old key back — never leave the box without a key
+		os.Rename(filepath.Join(dir, prevKeyName), cur)
+		os.Remove(tmp)
+		return "", fmt.Errorf("installing the new key: %w", err)
+	}
+	return pub, nil
+}

internal/wgtunnel/rekey_test.go

@@ -0,0 +1,146 @@
+package wgtunnel
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// D30 (R-1204): a handshake stamped in the future that STAYS for rekeyAfter → a new key pair, registered on the
+// existing peer path (the hub re-keys in place; its full-list push to ep0 swaps the key). The fake `wg` reports a
+// handshake at 1783107115; a clock three hours behind that sees it three hours in the future.
+// RED-PROOF (2026-10-22 night): with the clockAheadRekeyLocked call removed from Watchdog,
+// TestD30_AFutureHandshakeThatStaysReKeys and …AFailedRegistration… fail (no new key, no registration); with the
+// rekeyAfter wait removed, …ThatStaysReKeys fails (a re-key at 10 min); with the isActive check removed,
+// TestD30_NoReKeyWhileTheTunnelIsDown fails.
+
+const hs = 1783107115
+
+func at(m *Manager, unix int64) { m.now = func() time.Time { return time.Unix(unix, 0) } }
+
+func readKey(t *testing.T, m *Manager, name string) string {
+	t.Helper()
+	b, err := os.ReadFile(filepath.Join(m.stateDir, "wg", name))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return strings.TrimSpace(string(b))
+}
+
+func TestD30_AFutureHandshakeThatStaysReKeys(t *testing.T) {
+	fh := &fakeHub{}
+	m, _, active, logs := testManager(t, fh)
+	oldPub := establishWG(t, m, active)
+	oldPriv := readKey(t, m, keyFileName)
+	ctx := context.Background()
+	calls := fh.count()
+
+	t0 := int64(hs - 3*3600) // the clock is right again; the last handshake (made while it ran ahead) is 3 h ahead
+	at(m, t0)
+	m.Watchdog(ctx, true, testBlock(oldPub))
+	at(m, t0+10*60)
+	m.Watchdog(ctx, true, testBlock(oldPub))
+	if fh.count() != calls || localPub(t, m) != oldPub {
+		t.Fatal("no re-key before the future stamp has stayed rekeyAfter")
+	}
+	at(m, t0+16*60)
+	m.Watchdog(ctx, true, testBlock(oldPub))
+	newPub := localPub(t, m)
+	if newPub == oldPub {
+		t.Fatal("a future stamp that stayed 16 min must give a new key")
+	}
+	if fh.count() != calls+1 {
+		t.Fatalf("the new key must be registered on the existing peer path (RegisterWG calls %d → %d)", calls, fh.count())
+	}
+	if mk := m.loadMarker(); mk == nil || mk.Pubkey != newPub {
+		t.Fatalf("the registration marker must carry the new key: %+v", mk)
+	}
+	if readKey(t, m, prevKeyName) != oldPriv {
+		t.Fatal("the old private key must be kept as private.key.prev (the escrow may hold it)")
+	}
+	if strings.Contains(logs.String(), oldPriv) || strings.Contains(logs.String(), readKey(t, m, keyFileName)) {
+		t.Fatal("a private key reached the log")
+	}
+	// The served block still names the OLD key until the next fetch: no second re-key over that.
+	at(m, t0+17*60)
+	m.Watchdog(ctx, true, testBlock(oldPub))
+	if localPub(t, m) != newPub {
+		t.Fatal("a second re-key ran before the hub served the new key")
+	}
+	// The hub serves the new key → the tunnel is applied with the NEW private key.
+	m.Apply(ctx, true, testBlock(newPub))
+	conf, err := os.ReadFile(m.stagedConfPath())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(conf), readKey(t, m, keyFileName)) || strings.Contains(string(conf), oldPriv) {
+		t.Fatal("the applied tunnel must use the new private key, not the old one")
+	}
+}
+
+func TestD30_AShortFutureSpellDoesNotReKey(t *testing.T) {
+	fh := &fakeHub{}
+	m, _, active, _ := testManager(t, fh)
+	pub := establishWG(t, m, active)
+	ctx := context.Background()
+	t0 := int64(hs - 3*3600)
+	at(m, t0)
+	m.Watchdog(ctx, true, testBlock(pub))
+	at(m, hs+100) // a handshake now lies in the past: the sign ended (a working tunnel)
+	m.Watchdog(ctx, true, testBlock(pub))
+	at(m, t0+20*60) // in the future again: a NEW window starts, it does not continue the old one
+	m.Watchdog(ctx, true, testBlock(pub))
+	if localPub(t, m) != pub {
+		t.Fatal("a future stamp that did not STAY for rekeyAfter must not re-key")
+	}
+}
+
+func TestD30_NoReKeyWhileTheTunnelIsDown(t *testing.T) {
+	fh := &fakeHub{}
+	m, _, active, _ := testManager(t, fh)
+	pub := establishWG(t, m, active)
+	*active = false
+	ctx := context.Background()
+	t0 := int64(hs - 3*3600)
+	for _, d := range []int64{0, 20 * 60, 40 * 60} {
+		at(m, t0+d)
+		m.Watchdog(ctx, true, testBlock(pub))
+	}
+	if localPub(t, m) != pub {
+		t.Fatal("no re-key while the tunnel unit is not active")
+	}
+}
+
+func TestD30_AFailedRegistrationIsRetriedByTheExistingPath(t *testing.T) {
+	fh := &fakeHub{}
+	m, _, active, _ := testManager(t, fh)
+	oldPub := establishWG(t, m, active)
+	ctx := context.Background()
+	t0 := int64(hs - 3*3600)
+	at(m, t0)
+	m.Watchdog(ctx, true, testBlock(oldPub))
+	fh.mu.Lock()
+	fh.err = errors.New("hub down")
+	fh.mu.Unlock()
+	at(m, t0+16*60)
+	m.Watchdog(ctx, true, testBlock(oldPub))
+	newPub := localPub(t, m)
+	if newPub == oldPub {
+		t.Fatal("the key rotates before the registration")
+	}
+	if mk := m.loadMarker(); mk == nil || mk.Pubkey != oldPub {
+		t.Fatal("a failed registration leaves the marker on the old key")
+	}
+	fh.mu.Lock()
+	fh.err = nil
+	fh.mu.Unlock()
+	at(m, t0+30*60) // past the backoff: Apply sees the served (old) key ≠ the local key → re-key-in-place registers
+	m.Apply(ctx, true, testBlock(oldPub))
+	if mk := m.loadMarker(); mk == nil || mk.Pubkey != newPub {
+		t.Fatalf("the existing re-key-in-place path must register the new key: %+v", mk)
+	}
+}