From ae69ad4512d4959cf520b04c214d2237de1041a3 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 22 Oct 2026 20:52:52 +0200 Subject: [PATCH] =?UTF-8?q?D30=20(R-1204):=20a=20handshake=20stamped=20in?= =?UTF-8?q?=20the=20future=20for=2015=20min=20gives=20a=20new=20WireGuard?= =?UTF-8?q?=20key,=20registered=20on=20the=20existing=20peer=20path=20(the?= =?UTF-8?q?=20hub=20re-keys=20in=20place;=20its=20full-list=20push=20swaps?= =?UTF-8?q?=20the=20key=20on=20ep0=20=E2=80=94=20nothing=20new=20on=20ep0)?= =?UTF-8?q?;=20old=20key=20kept=20as=20private.key.prev;=20red-proved;=20u?= =?UTF-8?q?nreleased?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 15 ++++ internal/wgtunnel/manager.go | 10 +++ internal/wgtunnel/rekey.go | 122 ++++++++++++++++++++++++++ internal/wgtunnel/rekey_test.go | 146 ++++++++++++++++++++++++++++++++ 4 files changed, 293 insertions(+) create mode 100644 internal/wgtunnel/rekey.go create mode 100644 internal/wgtunnel/rekey_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index dca92db..3d3db7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +## Unreleased (2026-10-22/23 night) — the tunnel heals itself after a clock that ran ahead (D30, R-1204) — COMMITTED, NOT RELEASED + +- **D30 (operator 2026-10-22: the agent makes a new key and registers it on the existing peer path).** When the latest + handshake stays stamped in the future for 15 minutes (the R-1204 sign: ep0 keeps the future timestamp and refuses + every newer handshake from that key), the agent makes a new WireGuard key pair, keeps the old private key as + `wg/private.key.prev` (0600 — the escrow may hold it), and registers the new public key through `POST /hosts//wg`. + The hub re-keys the peer IN PLACE (same /32, generation bump) and its full-list push to ep0 drops the old key and adds + the new one — nothing new runs on ep0. A new key is a new peer for ep0, with no stored timestamp, so the next + handshake is accepted. At most once an hour; not while the tunnel unit is down; never while a re-register is already + in flight. A failed registration is retried by the existing re-key-in-place path. Logs carry public keys only. + Tests `TestD30_*` (4), red-proved. A host-loss DR that installs the escrowed (older) key registers it again, and the + hub re-keys in place back to it — the same path. +- **Known gap, unchanged (filed):** after a reboot the kernel forgets the future stamp (latest handshake 0), so neither + this re-key nor the hub's alarm sees a box whose clock was corrected and which then restarted. + ## Unreleased (2026-10-22 night) — the tunnel's future-stamped handshake is reported (R-1204) — COMMITTED, NOT RELEASED (no agent release tonight); root SSH by key only (R-982, D4) is a READY PATCH, not on main - **R-982 / D4 — a ready patch, NOT on main.** Built and proven the same night (felhom.eu diff --git a/internal/wgtunnel/manager.go b/internal/wgtunnel/manager.go index 03b8afe..74d9287 100644 --- a/internal/wgtunnel/manager.go +++ b/internal/wgtunnel/manager.go @@ -99,6 +99,7 @@ type Manager struct { resolver Resolver staleAfter time.Duration // handshake-age threshold for the re-resolve watchdog (doc 06 §4.2) + rekeyAfter time.Duration // D30: how long a future-stamped handshake stays before the re-key (rekey.go) mu sync.Mutex nextRegAt time.Time @@ -111,6 +112,10 @@ type Manager struct { resolveFailLogged bool // DNS-failure log throttle (reset on the next success) staleSameIPLogged bool // "stale but IP unchanged" log throttle (endpoint-down, not re-IP) + // D30 (rekey.go): when the latest handshake was first seen stamped in the future, and the last re-key. + futureSince time.Time + lastRekey time.Time + // conv is the CACHED tunnel-convergence snapshot (v0.90.0, R-28 fast-tick source), refreshed at // the end of every Apply (the tunnel's own cadence) so the fast-tick reads it WITHOUT execing // `wg`/`systemctl` per tick. wgBlockDesired = a wireguard block for THIS key is desired; @@ -156,6 +161,7 @@ func NewManager(runner proxmox.Runner, registrar Registrar, stateDir string, log now: time.Now, resolver: netResolver{}, staleAfter: 180 * time.Second, + rekeyAfter: rekeyAfterDefault, } } @@ -600,6 +606,10 @@ func (m *Manager) Watchdog(ctx context.Context, fetched bool, block *hub.WireWir if !fetched || block == nil || !m.lastResolvedIP.IsValid() { return // nothing applied to watch, or no desired block } + // D30: a handshake stamped in the future that stays → a new key on the existing peer path (rekey.go). + if m.clockAheadRekeyLocked(ctx, block) { + return + } if !m.handshakeStaleLocked(ctx) { return // healthy → no DNS, no action (the load-bearing negative) } diff --git a/internal/wgtunnel/rekey.go b/internal/wgtunnel/rekey.go new file mode 100644 index 0000000..4251fcb --- /dev/null +++ b/internal/wgtunnel/rekey.go @@ -0,0 +1,122 @@ +package wgtunnel + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + "os" + "path/filepath" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +// D30 (operator 2026-10-22, R-1204): the tunnel heals itself after the box ran with its clock ahead. +// +// WireGuard refuses a handshake whose timestamp is not newer than the last one it accepted from that key (replay +// protection). A box whose clock ran ahead handshook with a future timestamp; ep0 keeps it, and once the clock is +// right again ep0 refuses every new handshake from that key — until the box's clock passes the stored stamp. The +// sign on the box: its own latest handshake stays stamped in the future (R-1204's handshake_future_s). +// +// The repair: when that sign stays for rekeyAfter, the agent makes a NEW key pair and registers the new public key on +// the existing path (POST /hosts//wg → the hub re-keys the peer IN PLACE: same /32, new key, generation bump, +// and the wgsync push of the FULL peer list to ep0 drops the old key and adds the new one). A new key is a new peer +// for ep0, with no stored timestamp — the next handshake is accepted. Nothing new runs on ep0. +// +// The previous private key is kept as private.key.prev (0600): the escrow ceremony may hold it (a host-loss DR that +// installs it registers it again, and the hub re-keys in place back to it — the same path). +// Pinned by TestD30_* (rekey_test.go). + +const ( + // rekeyAfter: how long the latest handshake must stay in the future before the re-key. A working tunnel + // handshakes every two minutes, which ends the sign within minutes of the clock being right. + rekeyAfterDefault = 15 * time.Minute + // rekeyMinGap bounds the re-key to once an hour, whatever the clock does. + rekeyMinGap = time.Hour + prevKeyName = "private.key.prev" +) + +// clockAheadRekeyLocked watches for a handshake stamped in the future and re-keys when it stays. It reports whether +// it re-keyed (the caller then skips the rest of its tick). Caller holds m.mu. +func (m *Manager) clockAheadRekeyLocked(ctx context.Context, block *hub.WireWireguard) bool { + if m.keyBroken || !m.isActive(ctx) { + m.futureSince = time.Time{} + return false + } + out, _, err := m.runner.Run(ctx, "wg", "show", Iface, "latest-handshakes") + if err != nil { + return false + } + now := m.now() + fut, ok := handshakeFuture(string(out), now) + if !ok { + if !m.futureSince.IsZero() { + m.logger.Info("wgtunnel: the handshake is no longer stamped in the future — no re-key needed") + } + m.futureSince = time.Time{} + return false + } + if m.futureSince.IsZero() { + m.futureSince = now + m.logger.Warn("wgtunnel: the latest handshake is stamped in the future — this box's clock ran ahead, and the endpoint refuses its newer handshakes; re-keying if it stays", + "future_s", fut, "rekey_after", m.rekeyAfter.String()) + return false + } + if now.Sub(m.futureSince) < m.rekeyAfter { + return false + } + if !m.lastRekey.IsZero() && now.Sub(m.lastRekey) < rekeyMinGap { + return false + } + pub, _, err := EnsureKey(m.stateDir) + if err != nil || block.Pubkey != pub { + return false // a re-register is already in flight, or the key is not readable — never re-key over that + } + newPub, err := rotateKey(m.stateDir) + if err != nil { + m.logger.Error("wgtunnel: re-key after a future handshake FAILED — the old key stays", "err", err) + m.lastRekey = now // not again before rekeyMinGap + return false + } + m.lastRekey, m.futureSince = now, time.Time{} + m.logger.Warn("wgtunnel: RE-KEYED after a handshake stamped in the future (D30) — registering the new key; the hub re-keys the peer in place and the endpoint takes the new key", + "old_pubkey", pub, "new_pubkey", newPub, "future_s", fut) + m.nextRegAt = time.Time{} // register now, not after an older backoff + m.registerLocked(ctx, newPub) + return true +} + +// rotateKey makes a new key pair: the new private key is written next to the old one, the old one is kept as +// private.key.prev, then the new one takes its place (renames in one directory). Returns the new PUBLIC key only. +func rotateKey(stateDir string) (string, error) { + dir := filepath.Join(stateDir, "wg") + cur := filepath.Join(dir, keyFileName) + if _, err := os.Stat(cur); err != nil { + return "", fmt.Errorf("no current key: %w", err) + } + priv := make([]byte, 32) + if _, err := rand.Read(priv); err != nil { + return "", fmt.Errorf("entropy: %w", err) + } + clampPrivateKey(priv) + pub, err := derivePublic(priv) + if err != nil { + return "", err + } + tmp := filepath.Join(dir, keyFileName+".new") + if err := os.WriteFile(tmp, []byte(base64.StdEncoding.EncodeToString(priv)+"\n"), 0o600); err != nil { + return "", fmt.Errorf("writing the new key: %w", err) + } + if err := os.Rename(cur, filepath.Join(dir, prevKeyName)); err != nil { + os.Remove(tmp) + return "", fmt.Errorf("keeping the old key: %w", err) + } + if err := os.Rename(tmp, cur); err != nil { + // put the old key back — never leave the box without a key + os.Rename(filepath.Join(dir, prevKeyName), cur) + os.Remove(tmp) + return "", fmt.Errorf("installing the new key: %w", err) + } + return pub, nil +} diff --git a/internal/wgtunnel/rekey_test.go b/internal/wgtunnel/rekey_test.go new file mode 100644 index 0000000..e7bd30a --- /dev/null +++ b/internal/wgtunnel/rekey_test.go @@ -0,0 +1,146 @@ +package wgtunnel + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// D30 (R-1204): a handshake stamped in the future that STAYS for rekeyAfter → a new key pair, registered on the +// existing peer path (the hub re-keys in place; its full-list push to ep0 swaps the key). The fake `wg` reports a +// handshake at 1783107115; a clock three hours behind that sees it three hours in the future. +// RED-PROOF (2026-10-22 night): with the clockAheadRekeyLocked call removed from Watchdog, +// TestD30_AFutureHandshakeThatStaysReKeys and …AFailedRegistration… fail (no new key, no registration); with the +// rekeyAfter wait removed, …ThatStaysReKeys fails (a re-key at 10 min); with the isActive check removed, +// TestD30_NoReKeyWhileTheTunnelIsDown fails. + +const hs = 1783107115 + +func at(m *Manager, unix int64) { m.now = func() time.Time { return time.Unix(unix, 0) } } + +func readKey(t *testing.T, m *Manager, name string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(m.stateDir, "wg", name)) + if err != nil { + t.Fatal(err) + } + return strings.TrimSpace(string(b)) +} + +func TestD30_AFutureHandshakeThatStaysReKeys(t *testing.T) { + fh := &fakeHub{} + m, _, active, logs := testManager(t, fh) + oldPub := establishWG(t, m, active) + oldPriv := readKey(t, m, keyFileName) + ctx := context.Background() + calls := fh.count() + + t0 := int64(hs - 3*3600) // the clock is right again; the last handshake (made while it ran ahead) is 3 h ahead + at(m, t0) + m.Watchdog(ctx, true, testBlock(oldPub)) + at(m, t0+10*60) + m.Watchdog(ctx, true, testBlock(oldPub)) + if fh.count() != calls || localPub(t, m) != oldPub { + t.Fatal("no re-key before the future stamp has stayed rekeyAfter") + } + at(m, t0+16*60) + m.Watchdog(ctx, true, testBlock(oldPub)) + newPub := localPub(t, m) + if newPub == oldPub { + t.Fatal("a future stamp that stayed 16 min must give a new key") + } + if fh.count() != calls+1 { + t.Fatalf("the new key must be registered on the existing peer path (RegisterWG calls %d → %d)", calls, fh.count()) + } + if mk := m.loadMarker(); mk == nil || mk.Pubkey != newPub { + t.Fatalf("the registration marker must carry the new key: %+v", mk) + } + if readKey(t, m, prevKeyName) != oldPriv { + t.Fatal("the old private key must be kept as private.key.prev (the escrow may hold it)") + } + if strings.Contains(logs.String(), oldPriv) || strings.Contains(logs.String(), readKey(t, m, keyFileName)) { + t.Fatal("a private key reached the log") + } + // The served block still names the OLD key until the next fetch: no second re-key over that. + at(m, t0+17*60) + m.Watchdog(ctx, true, testBlock(oldPub)) + if localPub(t, m) != newPub { + t.Fatal("a second re-key ran before the hub served the new key") + } + // The hub serves the new key → the tunnel is applied with the NEW private key. + m.Apply(ctx, true, testBlock(newPub)) + conf, err := os.ReadFile(m.stagedConfPath()) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(conf), readKey(t, m, keyFileName)) || strings.Contains(string(conf), oldPriv) { + t.Fatal("the applied tunnel must use the new private key, not the old one") + } +} + +func TestD30_AShortFutureSpellDoesNotReKey(t *testing.T) { + fh := &fakeHub{} + m, _, active, _ := testManager(t, fh) + pub := establishWG(t, m, active) + ctx := context.Background() + t0 := int64(hs - 3*3600) + at(m, t0) + m.Watchdog(ctx, true, testBlock(pub)) + at(m, hs+100) // a handshake now lies in the past: the sign ended (a working tunnel) + m.Watchdog(ctx, true, testBlock(pub)) + at(m, t0+20*60) // in the future again: a NEW window starts, it does not continue the old one + m.Watchdog(ctx, true, testBlock(pub)) + if localPub(t, m) != pub { + t.Fatal("a future stamp that did not STAY for rekeyAfter must not re-key") + } +} + +func TestD30_NoReKeyWhileTheTunnelIsDown(t *testing.T) { + fh := &fakeHub{} + m, _, active, _ := testManager(t, fh) + pub := establishWG(t, m, active) + *active = false + ctx := context.Background() + t0 := int64(hs - 3*3600) + for _, d := range []int64{0, 20 * 60, 40 * 60} { + at(m, t0+d) + m.Watchdog(ctx, true, testBlock(pub)) + } + if localPub(t, m) != pub { + t.Fatal("no re-key while the tunnel unit is not active") + } +} + +func TestD30_AFailedRegistrationIsRetriedByTheExistingPath(t *testing.T) { + fh := &fakeHub{} + m, _, active, _ := testManager(t, fh) + oldPub := establishWG(t, m, active) + ctx := context.Background() + t0 := int64(hs - 3*3600) + at(m, t0) + m.Watchdog(ctx, true, testBlock(oldPub)) + fh.mu.Lock() + fh.err = errors.New("hub down") + fh.mu.Unlock() + at(m, t0+16*60) + m.Watchdog(ctx, true, testBlock(oldPub)) + newPub := localPub(t, m) + if newPub == oldPub { + t.Fatal("the key rotates before the registration") + } + if mk := m.loadMarker(); mk == nil || mk.Pubkey != oldPub { + t.Fatal("a failed registration leaves the marker on the old key") + } + fh.mu.Lock() + fh.err = nil + fh.mu.Unlock() + at(m, t0+30*60) // past the backoff: Apply sees the served (old) key ≠ the local key → re-key-in-place registers + m.Apply(ctx, true, testBlock(oldPub)) + if mk := m.loadMarker(); mk == nil || mk.Pubkey != newPub { + t.Fatalf("the existing re-key-in-place path must register the new key: %+v", mk) + } +}