From d5ce5c364a5aca789a1d17f3a26f372f09c05199 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sat, 14 Nov 2026 17:43:33 +0100 Subject: [PATCH] agent (unreleased): a boot token that a clock step cannot move (R-1395) /disks answers guest_boot_key (the kernel's boot_id and the guest init's start) beside the unchanged guest_boot_id. Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- CHANGELOG.md | 10 ++- REUSE.md | 1 + internal/localapi/disks.go | 9 +- internal/localapi/disks_test.go | 1 + internal/localapi/guest_boot_key_test.go | 109 +++++++++++++++++++++++ internal/localapi/intermediary.go | 59 ++++++++++-- 6 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 internal/localapi/guest_boot_key_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dda495..48d4262 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,15 @@ -## Unreleased (2026-11-14 night) — release 3 („the hub tells you”), the agent's parts, built and NOT released +## Unreleased (2026-11-14 night) — release 3 („the hub tells you”) and the clock-step fix, the agent's parts, built and NOT released Commit only. Each part can be released alone; none needs release 2b. No `configs/` change in this section. +- **A boot token that a clock step cannot move (R-1395).** `/disks` answers `guest_boot_key` = + `-` beside the unchanged `guest_boot_id` (`-`). The + kernel computes the boot time from the wall clock, so a step of the host's clock moved the old token and the + controller recreated every drive-backed app (the N100, 2026-11-14, +32.7 s). `GuestBinder.GuestBootKey`; `""` when + the guest does not run or the boot_id is unreadable. Only a controller that knows the key (built there, not + released) reads it. Tests `TestGuestBootKey_AClockStepDoesNotMoveIt`, `TestDisks_CarriesBothBootTokens`; 6 mutations + seen red (felhom.eu `documentation/audits/night-build-2026-11-14/C/redproof-agent-boot-key.txt`; two mutants did not + compile at first and were done again). - **A refused signed job and a watchdog give-up reach the hub (R-1383, D78, felhom.eu `09` §3 decision 299; `05` §16.6).** (1) `signedjobs.Runner` clears a job with `hub.Client.CompleteJobWithOutcome`: `outcome` (done · rejected · failed · malformed), a `reason` from the closed `signedjobs.ReasonCodes` (the two clock reasons `expired` and diff --git a/REUSE.md b/REUSE.md index d9309e6..eaf7a99 100644 --- a/REUSE.md +++ b/REUSE.md @@ -62,6 +62,7 @@ | `GuestBinder.EnsureSharedParent` | internal/localapi/intermediary.go | `EnsureSharedParent(ctx) error` | shared-parent bind + boot unit | make-private+make-shared ONLY on first bind — re-running orphans the guest's slave; F2-a: compares script AND unit for staleness | | `StablePathForRaw` / `DriveNameFromRaw` | internal/localapi/intermediary.go | `StablePathForRaw("/mnt/") string` | raw host mount → stable guest path | Single source of truth both repos derive from | | `GuestBinder.GuestSeesMount` / `GuestBootID` | internal/localapi/intermediary.go | `GuestSeesMount(ctx, vmid, path) bool` | guest-visible (usable) signal; reboot detection | Host bind present ≠ guest sees it (non-recursive parent bind) | +| `GuestBinder.GuestBootKey` (R-1395) | internal/localapi/intermediary.go | `GuestBootKey(ctx, vmid) string` | "is this a new boot of the guest?" — `-` | **`GuestBootID` carries `/proc/stat` `btime`, which the kernel computes from the wall clock: a clock STEP moves it.** Use the key for any „same boot?” question; `""` = unreadable. Seams `hostBtimeFn` / `hostBootIDFn` / `procStarttimeFn` | | `SudoHostOps.EnsureNetworkMount` / `RemoveNetworkMount` / `ListNetworkMounts` | internal/storage/netmount.go | `EnsureNetworkMount(ctx, spec) error` | NAS automount pair | rm glob confined to `mnt-felhom*` units; NAS ≠ drive (no durable-id/SMART/wipe); RemoveNetworkMount doubles as the verify-fail rollback (idempotent) | | `NetworkMountedAt` / `NetworkEndpointReachable` | internal/storage/netmount.go | `NetworkMountedAt(where) bool` | verify mount-truth + the 2 s add pre-probe | /proc/mounts is the ONLY mount-success judge (autofs trigger ≠ mounted; readability ≠ mounted — SPIKE-nas-verify §8) | | `SudoHostOps.ReassertNetworkAutomounts` + `Server.ReassertNetworkMounts` + `guesthook.PostStartNetworkReassert` | internal/storage/netreassert.go, internal/localapi/netreassert.go, internal/guesthook/netreassert.go | `ReassertNetworkAutomounts(ctx) []NetReassertResult` | NAS guest-reboot heal (RCA fix 1): re-arm idle automount triggers (stop + enable --now) so the fresh mount event propagates into running guests | NEVER call from periodic health paths (an idle trigger is HEALTHY); active real mounts are never touched; hook leg runs as root (direct systemctl), daemon leg via sudo | diff --git a/internal/localapi/disks.go b/internal/localapi/disks.go index 9150bb1..9d15cf6 100644 --- a/internal/localapi/disks.go +++ b/internal/localapi/disks.go @@ -87,6 +87,9 @@ type GuestAttacher interface { // GuestBootID returns a token that changes on every guest boot (host or guest) but is stable across a // controller-only restart — the deterministic guest-reboot signal the controller recreates apps on. GuestBootID(ctx context.Context, vmid int) string + // GuestBootKey is the same signal built from the kernel's boot_id, which a step of the host's clock cannot + // move (R-1395). "" = not readable; the controller then reads GuestBootID. + GuestBootKey(ctx context.Context, vmid int) string // AttachBind is the LEGACY per-drive `pct set -mpN` bind (pre-intermediary). Retained for the // transition; new attaches use AttachDrive. AttachBind(ctx context.Context, vmid int, mountKey, where string) error @@ -431,11 +434,13 @@ func (s *Server) handleDisks(w http.ResponseWriter, r *http.Request, vmid int) { // Guest boot-id (intermediary model): changes on every guest boot, stable across controller restarts. // The controller persists it and deterministically recreates drive-backed apps when it changes. - bootID := "" + // R-1395: guest_boot_key is the clock-proof form (kernel boot_id); guest_boot_id stays for older controllers. + bootID, bootKey := "", "" if s.guestAttach != nil { bootID = s.guestAttach.GuestBootID(r.Context(), vmid) + bootKey = s.guestAttach.GuestBootKey(r.Context(), vmid) } - writeOK(w, map[string]any{"vmid": vmid, "disks": out, "guest_boot_id": bootID}) + writeOK(w, map[string]any{"vmid": vmid, "disks": out, "guest_boot_id": bootID, "guest_boot_key": bootKey}) } // handleDiskCandidates (Impl-2a) lists the host disks FREE for Felhom to enroll — the raw-device diff --git a/internal/localapi/disks_test.go b/internal/localapi/disks_test.go index 32d2cfe..7140ef1 100644 --- a/internal/localapi/disks_test.go +++ b/internal/localapi/disks_test.go @@ -466,6 +466,7 @@ func (f *fakeGuestAttacher) AttachDrive(_ context.Context, _ int, where string) } func (f *fakeGuestAttacher) GuestSeesMount(_ context.Context, _ int, _ string) bool { return true } func (f *fakeGuestAttacher) GuestBootID(_ context.Context, _ int) string { return "boot-1" } +func (f *fakeGuestAttacher) GuestBootKey(_ context.Context, _ int) string { return "key-1" } func (f *fakeGuestAttacher) attachDriveCount() int { f.mu.Lock() defer f.mu.Unlock() diff --git a/internal/localapi/guest_boot_key_test.go b/internal/localapi/guest_boot_key_test.go new file mode 100644 index 0000000..d7127ec --- /dev/null +++ b/internal/localapi/guest_boot_key_test.go @@ -0,0 +1,109 @@ +package localapi + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-agent/internal/hub" +) + +type initPIDRunner struct{ pid string } + +func (r initPIDRunner) Run(_ context.Context, name string, _ ...string) ([]byte, []byte, error) { + if name == "lxc-info" { + return []byte(r.pid + "\n"), nil, nil + } + return nil, nil, nil +} + +func (r initPIDRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) { + return r.Run(ctx, name, args...) +} + +// R-1395: on 2026-11-14 the N100's time service came back and stepped the host's clock by +32.7 s. The kernel +// computes the boot TIME from the wall clock, so guest_boot_id moved and the controller recreated every drive-backed +// app. The consequence asserted: the boot KEY does not move when the boot time moves; it does move on a host reboot +// and on a guest reboot; and it is "" (so the controller falls back to the old token) when the boot_id is unreadable. +// COMPANION RED-PROOFS (observed, felhom.eu audits/night-build-2026-11-14/C/redproof-agent-boot-key.txt). +func TestGuestBootKey_AClockStepDoesNotMoveIt(t *testing.T) { + oldB, oldK, oldS := hostBtimeFn, hostBootIDFn, procStarttimeFn + t.Cleanup(func() { hostBtimeFn, hostBootIDFn, procStarttimeFn = oldB, oldK, oldS }) + btime, bootID, start := "1794112518", "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11", "4242" + hostBtimeFn = func() string { return btime } + hostBootIDFn = func() string { return bootID } + procStarttimeFn = func(pid string) string { + if pid != "31337" { + t.Errorf("start time read for pid %q, want the guest's init", pid) + } + return start + } + b := NewGuestBinder(initPIDRunner{"31337"}, slog.New(slog.NewTextHandler(io.Discard, nil))) + ctx := context.Background() + + id0, key0 := b.GuestBootID(ctx, 9201), b.GuestBootKey(ctx, 9201) + if id0 != "1794112518-4242" || key0 != "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11-4242" { + t.Fatalf("tokens = %q / %q", id0, key0) + } + btime = "1794112550" // THE STEP: the wall clock jumped +32 s, so the kernel's btime did too + id1, key1 := b.GuestBootID(ctx, 9201), b.GuestBootKey(ctx, 9201) + if id1 == id0 { + t.Fatalf("control: the old token did not move with the boot time (%q) — the test does not stage the defect", id1) + } + if key1 != key0 { + t.Fatalf("the boot key moved with the clock: %q → %q — a clock step reads as a new boot", key0, key1) + } + if strings.Contains(key1, "1794") { + t.Errorf("the boot key carries the boot time: %q", key1) + } + bootID = "c0ffee00-1111-2222-3333-444455556666" // a host reboot + if k := b.GuestBootKey(ctx, 9201); k == key0 { + t.Errorf("a host reboot (new boot_id) did not change the key: %q", k) + } + bootID, start = "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11", "991234" // a guest reboot + if k := b.GuestBootKey(ctx, 9201); k == key0 || !strings.HasSuffix(k, "-991234") { + t.Errorf("a guest reboot (new init start) did not change the key: %q", k) + } + for _, bad := range []string{"", "not-a-boot-id", "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11\nx"} { + bootID = bad + if k := b.GuestBootKey(ctx, 9201); k != "" { + t.Errorf("boot_id %q gave the key %q, want \"\" (the controller then reads the old token)", bad, k) + } + if id := b.GuestBootID(ctx, 9201); id == "" { + t.Errorf("the old token must still be sent when the boot_id is unreadable") + } + } + bootID, start = "5b1f6c1e-8f0a-4c56-9d1c-0a8c3b7d2e11", "" // the guest does not run + if k := b.GuestBootKey(ctx, 9201); k != "" { + t.Errorf("a guest that does not run gave the key %q", k) + } +} + +// The key is no use if /disks does not carry it: both tokens are on the wire, the old one unchanged for controllers +// that do not know the key. +func TestDisks_CarriesBothBootTokens(t *testing.T) { + srv := targetRowServer(t, "", []hub.StorageTarget{}) + srv.guestAttach = &fakeGuestAttacher{} + body := do(t, srv.Handler(), "GET", "/disks", "A", "").Body.Bytes() + var w struct { + Data struct { + ID string `json:"guest_boot_id"` + Key string `json:"guest_boot_key"` + } `json:"data"` + ID string `json:"guest_boot_id"` + Key string `json:"guest_boot_key"` + } + if err := json.Unmarshal(body, &w); err != nil { + t.Fatalf("decode: %v (%s)", err, body) + } + if w.Data.ID+w.ID != "boot-1" || w.Data.Key+w.Key != "key-1" { + t.Fatalf("/disks boot tokens: id=%q key=%q, want boot-1 and key-1 (%s)", w.Data.ID+w.ID, w.Data.Key+w.Key, body) + } +} + +// The real binder is what main.go hands the server: it must have the method (the interface makes this a build +// error if not — stated here so the reason is on record). +var _ GuestAttacher = (*GuestBinder)(nil) diff --git a/internal/localapi/intermediary.go b/internal/localapi/intermediary.go index fe1ec39..4d3d38f 100644 --- a/internal/localapi/intermediary.go +++ b/internal/localapi/intermediary.go @@ -1,6 +1,7 @@ package localapi import ( + "regexp" "bufio" "context" "fmt" @@ -288,21 +289,67 @@ func (b *GuestBinder) guestInitPID(ctx context.Context, vmid int) string { // /proc//stat — ticks since host boot, unique per process launch) changes on a guest reboot. "" // on any read error (the controller then keeps its last-seen → no spurious recreate). func (b *GuestBinder) GuestBootID(ctx context.Context, vmid int) string { - pid := b.guestInitPID(ctx, vmid) - if pid == "" { - return "" - } - start := procStarttime(pid) + start := b.guestInitStart(ctx, vmid) if start == "" { return "" } - bt := hostBtime() + bt := hostBtimeFn() if bt == "" { bt = "0" } return bt + "-" + start } +// guestInitStart is the guest init's start time in ticks since the host's boot ("" = the guest does not run, or a +// read error). Both boot tokens end with it. +func (b *GuestBinder) guestInitStart(ctx context.Context, vmid int) string { + pid := b.guestInitPID(ctx, vmid) + if pid == "" { + return "" + } + return procStarttimeFn(pid) +} + +// GuestBootKey (R-1395) is GuestBootID with the host's boot TIME replaced by the kernel's boot_id: +// "-". The kernel computes the boot time (`btime`) from the wall clock, so a STEP of +// the host's clock moves GuestBootID — on the N100 on 2026-11-14 the time service came back, stepped +32.7 s, and the +// controller recreated every drive-backed app as if the guest had booted. The kernel's boot_id is a random value made +// once per boot; no clock can move it. Same meaning as before: it changes on a host reboot (new boot_id) and on a +// guest reboot (new init start), and is stable across an agent or controller restart. +// +// "" when the guest does not run or the boot_id cannot be read — the controller then decides by GuestBootID, which +// is still sent (controllers that do not know the key read only that). Pinned by TestGuestBootKey_*. +func (b *GuestBinder) GuestBootKey(ctx context.Context, vmid int) string { + start := b.guestInitStart(ctx, vmid) + return guestBootKey(hostBootIDFn(), start) +} + +// guestBootKey joins the two halves; "" when either is missing or the boot_id is not the kernel's shape. +func guestBootKey(bootID, start string) string { + if start == "" || !kernelBootIDRe.MatchString(bootID) { + return "" + } + return bootID + "-" + start +} + +var kernelBootIDRe = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +// The three reads of /proc behind the boot tokens, as seams (the tests move the boot time and keep the boot_id). +var ( + hostBtimeFn = hostBtime + hostBootIDFn = hostBootID + procStarttimeFn = procStarttime +) + +// hostBootID returns the kernel's boot_id (/proc/sys/kernel/random/boot_id): random, made once per boot. "" on error. +func hostBootID() string { + data, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + // procStarttime returns field 22 (starttime) of /proc//stat. The comm field (2) can contain spaces // and parentheses, so we split AFTER the last ')': field 22 is index 19 of the post-comm fields // (field 3 = state is index 0). "" on any error.