Compare commits
169 Commits
915642aaaa
...
v0.128.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 28ba8593b8 | |||
| 6981450110 | |||
| 703db166e7 | |||
| aa74294a7d | |||
| 5b2666e3a2 | |||
| 062a7027ab | |||
| a2e914f683 | |||
| 0404f60e6a | |||
| 3f5f61b716 | |||
| 6d7904786c | |||
| 856a127cd6 | |||
| 257c4d85c0 | |||
| 72161f6cf0 | |||
| 03b58cec0a | |||
| fe14bc62c0 | |||
| 0b28eae7bb | |||
| 7581f8140a | |||
| 3d0a1d615d | |||
| 77e2cc4583 | |||
| cd1b087db7 | |||
| 53d0c6bfc4 | |||
| 4d82591052 | |||
| 4618169036 | |||
| 1b14cfd0b4 | |||
| 0db77666c6 | |||
| dd2d1feb6e | |||
| 9dfd89cb94 | |||
| 4bb84fc3ca | |||
| cd6e26785a | |||
| 587dbb43fe | |||
| eb99144509 | |||
| 2c4efed5de | |||
| 75245a467c | |||
| 054e85a2bf | |||
| 4663df7ff3 | |||
| 14642e3c7b | |||
| 6b5dade4dc | |||
| 1c8a67eece | |||
| 1913e12031 | |||
| 966d8f41ff | |||
| 6be168d1a0 | |||
| d4eb259da2 | |||
| 21b0164fad | |||
| 2f4ccab166 | |||
| a58239f6de | |||
| b58d7bcf39 | |||
| 58b598b697 | |||
| 958e54f6a6 | |||
| 38176ada9d | |||
| d5c769173b | |||
| 50751b8901 | |||
| ff7f68e089 | |||
| 88b3cf03dd | |||
| f27f7a2659 | |||
| 8db92947cd | |||
| 367a503a0f | |||
| a18b18e5de | |||
| af1c21abc4 | |||
| c9a5cc664a | |||
| 023655370b | |||
| 9842c52853 | |||
| 1c2664b0c1 | |||
| 5bca7bfc9a | |||
| 043c7622bc | |||
| 765d8b3168 | |||
| edde8a01ca | |||
| a7ef497cc4 | |||
| 5acf1033a2 | |||
| e4f22f4c4f | |||
| 13ca2d96b2 | |||
| 005083b558 | |||
| 0fabc15896 | |||
| a7421b09c7 | |||
| 3d955e4edd | |||
| a667c269c7 | |||
| 68bcebe493 | |||
| 739b3c3b58 | |||
| dfd5d731ee | |||
| 36ed6594d4 | |||
| 271aa3d9ed | |||
| ed97232598 | |||
| 643899c191 | |||
| 21fee69154 | |||
| c230258542 | |||
| eba040d0be | |||
| a452dc3314 | |||
| 935904fa4e | |||
| 332235bfd1 | |||
| 3f58121b98 | |||
| 71e1777369 | |||
| 98adb72204 | |||
| 0e8fd814ba | |||
| c0966d753d | |||
| 08b55a1015 | |||
| 042a401e71 | |||
| edac02ec9f | |||
| d4a7a5bad3 | |||
| dcf3a5530b | |||
| ab8f682f0b | |||
| b2ca63ee9f | |||
| 8c55ac7fda | |||
| d81fddc7b1 | |||
| 8fc44d8d9e | |||
| 25c30ba625 | |||
| f22f70ce58 | |||
| 9596d5a8d0 | |||
| f222a7b366 | |||
| 8208a80cc7 | |||
| ac112c956e | |||
| 9127f547f9 | |||
| 9effc85a47 | |||
| a659e5dc09 | |||
| c040c180e9 | |||
| 1c3a3ef9ad | |||
| adf7882f7d | |||
| 4e6171442f | |||
| 3c174bc6f2 | |||
| c20814e6c2 | |||
| bcb8dad2aa | |||
| dbd6d4c57a | |||
| b6300250aa | |||
| 47eb0bf967 | |||
| 0b55a11667 | |||
| a527b29ac3 | |||
| bc4eda926b | |||
| bec4bac076 | |||
| a8b773f892 | |||
| 474b858c0b | |||
| 0df72ea643 | |||
| cb692f8788 | |||
| 461eaf42c1 | |||
| fa9c7fe198 | |||
| 1e60e88eb2 | |||
| added9d226 | |||
| 300f06722b | |||
| 3d1323cc09 | |||
| e5e8f3920a | |||
| a6e8bcb475 | |||
| 301c84d9b5 | |||
| bd9e777f41 | |||
| 98ea1ed49c | |||
| 4449118783 | |||
| 0c22b9bbf3 | |||
| 7f07393623 | |||
| b1697874ec | |||
| 5a72a4b59c | |||
| b3446213df | |||
| 3bf0110697 | |||
| c12b512316 | |||
| 4c40846769 | |||
| ced60ddf76 | |||
| ca0b169a4e | |||
| e04b75e1f8 | |||
| 7545af8a2c | |||
| 26664d6d51 | |||
| 223c235f01 | |||
| 72d6132907 | |||
| 2bf9354c0e | |||
| b99d02b7a9 | |||
| d880289b06 | |||
| a34aac64d0 | |||
| 9d9e4a5ea7 | |||
| c983a25609 | |||
| effff53f99 | |||
| 4b7c5bf128 | |||
| 8675bff3da | |||
| fd4e177216 | |||
| 1c75a45a42 | |||
| 084fc691f5 |
@@ -0,0 +1,46 @@
|
||||
---
|
||||
paths: ["internal/backup/**", "internal/pbs/**", "internal/pbsdr/**", "internal/dr/**"]
|
||||
---
|
||||
|
||||
# Backup, PBS and DR
|
||||
|
||||
`internal/backup/` is the vzdump runner, restore-test scheduler and report store. `internal/pbs/` is
|
||||
the fingerprint-pinned PBS-API client plus the verify maintenance loop. `internal/pbsdr/` and
|
||||
`internal/dr/` carry the DR tier and recipe halves.
|
||||
|
||||
## The three PBS laws
|
||||
|
||||
1. **Set-only.** `pvesm remove` **DELETES the encryption key**. Re-apply configuration; never remove
|
||||
and re-add a PBS storage to change it.
|
||||
2. **Secret on stdin.** A token secret is passed on stdin, never as an argv the process table shows.
|
||||
3. **Verify the pin BEFORE consuming the secret.** A fingerprint check after the secret has been sent
|
||||
protects nothing.
|
||||
|
||||
## Verify is server-side, and its default skips the work
|
||||
|
||||
The agent drives verification **remotely** via the PBS API; `proxmox-backup-client` has **no** verify
|
||||
subcommand. `POST .../verify` defaults to **`ignore-verified=true`, which SKIPS already-verified
|
||||
snapshots** — send `ignore-verified=false` to actually re-read and detect corruption. A verify that
|
||||
skipped everything reports success.
|
||||
|
||||
## Presence is not success
|
||||
|
||||
A timestamp recording an **attempt** is not evidence of a **result**. Where a status field travels
|
||||
beside a timestamp, the verdict must consult **both** — or the timestamp must record only successes.
|
||||
Ask of any timestamp: *what exactly must have happened for this to be set?* If the answer is "we
|
||||
tried", it cannot answer "did it work".
|
||||
|
||||
**Corollary:** when a verdict changes which field it counts from, the alarm text changes with it.
|
||||
Leaving a message reading `last run 8h ago` while alarming on a six-day-old **success** turns a true
|
||||
alarm into one the operator dismisses.
|
||||
|
||||
## Prune is server-side now
|
||||
|
||||
`DatastoreBackup` carries **no** `Datastore.Prune`. Boxes set `keep_last: 0` and the off-site endpoint
|
||||
runs the prune jobs. **Box tokens stay write-only — never widen that grant** (R-89).
|
||||
|
||||
<!--
|
||||
The ignore-verified default is the sharpest instance of the "absent log line" class in this repo: a
|
||||
verify that silently skipped every snapshot completes fast, exits clean, and reports the same shape
|
||||
as one that read every byte.
|
||||
-->
|
||||
@@ -0,0 +1,26 @@
|
||||
---
|
||||
paths: ["internal/capability/**", "internal/storage/**", "internal/localapi/**", "internal/hub/**", "internal/guesthook/**"]
|
||||
---
|
||||
|
||||
# A health check issues no block I/O
|
||||
|
||||
No `statfs`, no `getdents`, no read, write or `fsync` — **not even behind a timeout**.
|
||||
|
||||
A probe that touches a wedged device enters uninterruptible sleep, survives `SIGKILL`, and cannot be
|
||||
recovered until the device returns or the host reboots — so `systemctl restart` hangs too. A timeout
|
||||
protects the caller's control flow and nothing else: the blocked thread remains.
|
||||
|
||||
**Liveness is decided from `/proc` and the kernel's own state**, never by reading or writing the
|
||||
filesystem.
|
||||
|
||||
<!--
|
||||
Measured, R-117 spike §6.3 (felhom.eu/documentation/audits/SPIKE-r117-bind-liveness-2026-07-30.md):
|
||||
a probe stayed in D state 3m50s after kill -9; a buffered write with no fsync blocked too (O_CREAT
|
||||
needs journal access); and statfs/getdents returned HEALTHY on a namespace that EIOs every byte —
|
||||
fast, and wrong.
|
||||
|
||||
This rule used to be duplicated verbatim in felhom-agent/CLAUDE.md with a note explaining that
|
||||
felhom.eu/CLAUDE.md "does not load in an agent-only session". That reasoning was correct before
|
||||
path-scoped rules existed. The single source is now felhom.eu/CLAUDE.md "Code quality rules"; this
|
||||
file is the scoped copy that loads exactly where health checks are written. (2026-08-06)
|
||||
-->
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
paths: ["internal/localapi/**", "internal/authz/**", "internal/guesthook/**"]
|
||||
---
|
||||
|
||||
# Local API, authz and guest hooks — the per-guest blast radius
|
||||
|
||||
`internal/localapi/` is the narrow per-guest local API: token store, disks/format, guest binds,
|
||||
controller swap, stale-lock recovery, pinned self-signed leaf. `internal/authz/` is the operator
|
||||
signed-op verifier (SSHSIG) plus the durable nonce store. `internal/guesthook/` installs the
|
||||
pre-start self-heal hookscript.
|
||||
|
||||
> **Overlap note:** `health-checks.md` also matches `internal/localapi/**` and
|
||||
> `internal/guesthook/**`. That is deliberate — both rules apply there and both load. Neither
|
||||
> supersedes the other.
|
||||
|
||||
## Scoping is the whole security property
|
||||
|
||||
This API is reachable **from inside a customer guest**. Every route must be scoped to the guest that
|
||||
called it — a route that can name another guest's id has escaped its blast radius. Fail **safe to
|
||||
protected**: an unrecognised or unresolvable caller gets less access, never more.
|
||||
|
||||
## Replay protection must survive a restart
|
||||
|
||||
**`authz.MemoryNonceStore` on a real host is a defect** — replay protection dies on restart. Use
|
||||
`authz.FileNonceStore`. The memory store exists for tests.
|
||||
|
||||
## The token is a hash on disk, plaintext only at mint
|
||||
|
||||
The store keeps **hashes**. The plaintext token exists in exactly one place, `bootstrap.json` on the
|
||||
PVE host — so a "read the token" step means reading that file, and a lost token is re-minted, never
|
||||
recovered.
|
||||
|
||||
## Binds can brick guest boot
|
||||
|
||||
| Do not | Because | Use |
|
||||
|---|---|---|
|
||||
| `GuestBinder.AttachBind`/`DetachBind` (per-drive `pct set -mpN`) | legacy model; a missing bind source can **brick guest boot** (C1) | `AttachDrive`/`DetachDrive` (intermediary model) |
|
||||
| `isHostMountpoint` to reconcile bind state | a boolean cannot converge stacked double-binds (the `/mnt` doubling bug) | `countHostMounts` normalization inside `AttachDrive` |
|
||||
|
||||
<!--
|
||||
Why fail-safe-to-protected rather than fail-closed: this API also carries the recovery paths. A hard
|
||||
refusal on an unresolvable caller would make a half-broken guest unrecoverable through the very
|
||||
interface built to recover it. Less access, never none.
|
||||
-->
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
paths: ["internal/proxmox/**", "internal/reconcile/**", "internal/signedjobs/**"]
|
||||
---
|
||||
|
||||
# Proxmox — the API contract, and how destructive work is gated
|
||||
|
||||
`internal/proxmox/` is the API-first `Client` plus the fenced root-CLI `Privileged`.
|
||||
`internal/reconcile/` is the reconcile engine, reversibility gate, op journal and crash recovery.
|
||||
`internal/signedjobs/` holds the operator-signed destructive executors (wipe, decommission).
|
||||
|
||||
## A 200 on the POST is not success
|
||||
|
||||
**Every mutating op is async**: it returns a **UPID**, and `WaitTask` must assert
|
||||
`exitstatus == "OK"`. Authorization can fail at *task execution* long after the HTTP call returned
|
||||
200. Treating the POST's status as the result is how a failed destroy reads as a successful one.
|
||||
|
||||
## The privsep token gotcha
|
||||
|
||||
A `--privsep 1` token's rights are the **intersection** of the backing user's permissions **and** the
|
||||
token's own ACLs. The role must be granted on **both** or every call 403s. The same intersection rule
|
||||
bites on PBS (`token ∩ user`).
|
||||
|
||||
## TLS
|
||||
|
||||
**SHA-256 leaf-cert pinning** against the self-signed host cert. **No insecure default**, ever. The
|
||||
pin is the raw leaf-DER sha — the SAN is never checked, so a cert rotation changes the pin and the
|
||||
agent must be re-pinned.
|
||||
|
||||
## The destructive path — never the direct call
|
||||
|
||||
| Do not | Because | Use |
|
||||
|---|---|---|
|
||||
| `Client.DestroyLXC` / `Vzdump` / `SetConfig` ad-hoc | skips classification, signature, per-guest serialization, crash recovery | `reconcile.Engine` paths / `RunSignedJob`; queue via `Queue.Submit` |
|
||||
| add a method to `proxmox.Privileged` | breaks the 3-exception root-CLI fence (`routing_test.go`) | `proxmox.Runner` + a new sudoers `Cmnd_Alias` + `validate.go`-style checks |
|
||||
| treat `ListLXC` output as "guests we own" | audit A1 — pre-v0.62.0 the stale-lock reaper did exactly this, contained only by the pool-scoped token | intersect with `Client.Pool` membership (`staleLockController.Guests()`); **fail safe on read failure** |
|
||||
|
||||
Full trap table: `REUSE.md` §3. Every guest joins the `felhom` pool — `VM.Audit` comes from the
|
||||
`/pool` grant, not from a per-guest ACL.
|
||||
|
||||
<!--
|
||||
The fence is not stylistic. It is what makes this component auditable: two types, one of which can
|
||||
only speak HTTP and one of which can only shell out, with a test asserting neither crosses. A single
|
||||
convenience method on Privileged that also makes an HTTP call would end that property silently.
|
||||
-->
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
paths: ["internal/storage/**", "internal/escrow/**"]
|
||||
---
|
||||
|
||||
# Storage and escrow — format safety and zero-knowledge recovery
|
||||
|
||||
`internal/storage/` is the storage observer, durable IDs, role/claim classifiers, `SudoHostOps` and
|
||||
the watchdog. `internal/escrow/` is the PBS-key escrow with its zero-knowledge recovery code.
|
||||
|
||||
> **Overlap note:** `health-checks.md` also matches `internal/storage/**`. Deliberate — both rules
|
||||
> apply there and both load.
|
||||
|
||||
## Never format the device you inspected
|
||||
|
||||
**AGENT-001 is a TOCTOU:** acting on the caller's `req.Device` (or any remembered `/dev` path) after
|
||||
inspection lets `/dev` re-enumeration retarget the node to a **different physical disk**. Format the
|
||||
**re-resolved** device — `Server.reresolveWipe` / `reresolveBlank`.
|
||||
|
||||
**Never exec raw `mkfs.*`** (including `Binaries.MkfsExt4`/`MkfsXfs`): sudoers no longer allowlists
|
||||
raw mkfs, and going direct bypasses the claim filter and the wrapper's re-checks. Use
|
||||
`SudoHostOps.Format`, which routes through `felhom-mkfs-guarded`.
|
||||
|
||||
## The two durable-ID schemes refuse each other
|
||||
|
||||
They are not interchangeable, and each returns a `binding_mismatch` for the other's scheme:
|
||||
|
||||
| Purpose | Scheme | Resolver |
|
||||
|---|---|---|
|
||||
| wipe confirmation | `byid:` / `byuuid:` | `ResolveDurableDevice`, `DiskInfo.WipeDurableID` |
|
||||
| enrolled-storage remount | `uuid:` | `ResolveStorageDevice` |
|
||||
|
||||
Using `DiskInfo.DurableID` (a `uuid:`) as a wipe-confirmation id is F20-BUG2.
|
||||
|
||||
## Drive data is never taken by force
|
||||
|
||||
Plain `umount` only — **never `-l`, never `-f`**, and never any format operation under
|
||||
`/mnt/felhom-drives`.
|
||||
|
||||
## Escrow is zero-knowledge, and a fetch failure is not a wrong code
|
||||
|
||||
The server holds no client key; a no-key restore fails with `missing key`. **A fetch failure must
|
||||
never be reported as a wrong recovery code** — that told a customer their correct code was bad, in
|
||||
hundredths of a second, when checking a code actually takes about one. Distinguish "we could not
|
||||
reach the store" from "the code did not match", always.
|
||||
|
||||
<!--
|
||||
The escrow recovery-code "flake" was a REAL defect, not a flake. "Known flake, re-run" needs evidence
|
||||
before it is said out loud — that phrase cost this project a real finding once.
|
||||
-->
|
||||
@@ -0,0 +1,116 @@
|
||||
# gates — re-run this repo's gate entry point on every push, on a machine that does not care who
|
||||
# pushed or what they typed.
|
||||
#
|
||||
# *** THIS REPORTS. IT CANNOT REFUSE. ***
|
||||
#
|
||||
# felhom repos push straight to `main` with no pull request, so there is no merge for a status
|
||||
# check to stand at. The refusing half is `.githooks/pre-push`, which is local to a clone and which
|
||||
# `git push --no-verify` skips; this half is what notices when that happened. Neither half is the
|
||||
# whole thing, and both are named in felhom.eu documentation/backlog/OPEN-ITEMS.md R-168.
|
||||
#
|
||||
# NO `uses:` STEP ANYWHERE, deliberately: JavaScript actions need a node runtime in the runner, and
|
||||
# the runner is a host-mode container with python3 and git and nothing else (see
|
||||
# homelab-manifests/gitea-system/act-runner.yaml for why it is not privileged). Probe P3 measured
|
||||
# that a plain `git fetch` of the pushed SHA from the in-cluster Gitea service is enough.
|
||||
#
|
||||
# A failing run must reach a person — a detector nobody hears is the defect R-29 filed, rebuilt one
|
||||
# layer up. That is the last step, and it runs ONLY on failure.
|
||||
name: gates
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
gates:
|
||||
runs-on: felhom-gates
|
||||
steps:
|
||||
- name: Fetch the pushed commit and the sibling clone it needs
|
||||
# This repo's entry point invokes a SHARED checker that lives in the felhom.eu clone next
|
||||
# door and is deliberately never copied here — so CI has to reproduce the workspace's
|
||||
# sibling layout or the gate fails closed with "gate is MISSING". The sibling is also
|
||||
# needed for CONTENT: this repo's REUSE.md cites a path that lives in the hub.
|
||||
run: |
|
||||
# Shallow, and pinned to the exact SHA that was pushed — not to the branch tip,
|
||||
# which can move under us if two pushes race.
|
||||
mkdir -p ws/felhom-agent
|
||||
cd ws/felhom-agent
|
||||
git init -q .
|
||||
git remote add origin http://gitea.gitea-system.svc.cluster.local:3000/admin/felhom-agent.git
|
||||
git fetch -q --depth 1 origin "$GITHUB_SHA"
|
||||
git checkout -q FETCH_HEAD
|
||||
echo "checked out $(git rev-parse HEAD)"
|
||||
cd .. && git clone -q --depth 1 http://gitea.gitea-system.svc.cluster.local:3000/admin/felhom.eu.git felhom.eu
|
||||
echo "sibling felhom.eu present at $(cd felhom.eu && git rev-parse --short HEAD)"
|
||||
|
||||
- name: Run the gate entry point
|
||||
# The ONLY thing CI runs. No go build, no go test, no linting, no deploy. The
|
||||
# exit code IS the result: no `|| true`, no pipe that could swallow it.
|
||||
#
|
||||
# THE FULL SET, NOT `--fast` (R-115, 2026-08-03). `--fast` means "no network and no
|
||||
# container runtime" and exists for `.githooks/pre-push`, where a push must not fail
|
||||
# because Gitea blinked or because someone is on a train. CI is the opposite machine: it
|
||||
# has the network, it is not in anyone's way, and it is the half that emails. The
|
||||
# published-versions gate — the R-115 mechanism, which asks Gitea whether a released
|
||||
# version can actually be downloaded — is network-bound and therefore runs ONLY here.
|
||||
# Leaving `--fast` in place would have registered that gate and never run it, which is the
|
||||
# built-but-never-wired failure this project has shipped four times.
|
||||
env:
|
||||
# In-cluster, so the check does not depend on public DNS or the ingress TLS chain.
|
||||
GITEA_BASE: http://gitea.gitea-system.svc.cluster.local:3000
|
||||
run: cd ws/felhom-agent && python3 scripts/agent_gates.py
|
||||
|
||||
- name: Alarm on failure
|
||||
# THE POINT OF THE WHOLE THING. Probe P5 measured that a failed run produces NO mail, NO
|
||||
# notification row and NO log line from Gitea itself — a red tick in a web UI nobody watches
|
||||
# is exactly the shape R-29 filed against. So the run sends its own alarm, on the project's
|
||||
# existing transactional path (Resend, the same one the hub uses), and prints the provider's
|
||||
# accepted id so "a message left the machine" is an observable, not an assumption.
|
||||
#
|
||||
# Pure python3 and urllib, NOT curl: the runner image carries python3 and git and nothing
|
||||
# else on purpose, and the first version of this step died on `curl: command not found`.
|
||||
# Reaching for a bigger image to send one HTTP request would have been the wrong trade.
|
||||
if: failure()
|
||||
env:
|
||||
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import json, os, sys, urllib.request, urllib.error
|
||||
|
||||
key = os.environ.get("RESEND_API_KEY", "")
|
||||
if not key:
|
||||
sys.exit("ALARM FAILED: RESEND_API_KEY is empty — the alarm cannot be sent, and a "
|
||||
"silent alarm is worse than none. Set the user-level Actions secret.")
|
||||
|
||||
repo = os.environ.get("GITHUB_REPOSITORY", "?")
|
||||
sha = os.environ.get("GITHUB_SHA", "?")
|
||||
run = os.environ.get("GITHUB_RUN_NUMBER", "?")
|
||||
srv = os.environ.get("GITHUB_SERVER_URL", "https://gitea.dooplex.hu")
|
||||
|
||||
body = json.dumps({
|
||||
"from": "Felhom CI <monitoring@felhom.eu>",
|
||||
"to": ["admin@felhom.eu"],
|
||||
"subject": "[felhom CI] gates FAILED in %s" % repo,
|
||||
"text": (
|
||||
"The gate entry point exited non-zero.\n\n"
|
||||
"Repository : %s\n"
|
||||
"Commit : %s\n"
|
||||
"Run : %s/%s/actions/runs/%s\n\n"
|
||||
"The failing gate names itself in the run log.\n\n"
|
||||
"If the local pre-push hook was GREEN for this commit, then CI and the hook\n"
|
||||
"disagree - that is a finding about the gates themselves, not about CI, and it\n"
|
||||
"outranks whatever the push was for.\n"
|
||||
) % (repo, sha, srv, repo, run),
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
"https://api.resend.com/emails", data=body, method="POST",
|
||||
headers={"Authorization": "Bearer %s" % key,
|
||||
"Content-Type": "application/json",
|
||||
# Cloudflare fronts api.resend.com and BLOCKS the default
|
||||
# "Python-urllib/3.x" agent with its own 403 (error 1010) — which looks
|
||||
# exactly like an auth failure and is not one. Measured 2026-08-02.
|
||||
"User-Agent": "felhom-ci/1.0"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
print("RESEND-ACCEPTED id=%s" % json.load(r)["id"])
|
||||
except urllib.error.HTTPError as e:
|
||||
sys.exit("ALARM FAILED: Resend returned HTTP %s: %s" % (e.code, e.read().decode()[:300]))
|
||||
PY
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/bin/sh
|
||||
# pre-push — refuse a push that carries a broken gate. (2026-08-02, R-29 leg (b) first half.)
|
||||
#
|
||||
# Runs this repo's ONE gate entry point in --fast mode: only checks that touch no network and no
|
||||
# container runtime, so a push stays a push and never pulls images or starts containers. The slow
|
||||
# gates stay deliberate periodic runs; a hook that takes minutes gets bypassed within a week and
|
||||
# the bypass becomes the habit.
|
||||
#
|
||||
# BOTH LINES BELOW ARE DELIBERATE. An absent log line is not evidence a hook ran — a silent pass is
|
||||
# equally consistent with "gates green" and "hook never fired", so a passing push says so out loud.
|
||||
#
|
||||
# HONEST LIMITS, stated so this is not mistaken for enforcement it cannot provide:
|
||||
# * per-clone — core.hooksPath is local config and a clone does not carry it. Arm a clone once:
|
||||
# git config core.hooksPath .githooks
|
||||
# Any manual entry-point run WARNS when the clone is unarmed.
|
||||
# * skippable — `git push --no-verify` bypasses this entirely. That is on purpose: an escape
|
||||
# hatch that cannot be reached is one that gets removed the first time it is
|
||||
# inconvenient. USING IT MUST BE STATED IN THE SESSION REPORT.
|
||||
# The half that is neither per-clone nor skippable is CI — felhom.eu OPEN-ITEMS.md R-168.
|
||||
#
|
||||
# Measured 2026-08-02 (git 2.47.3): a relative core.hooksPath resolves correctly and the hook's cwd
|
||||
# is the repo root whether `git push` is issued from the root or from any subdirectory. The
|
||||
# explicit rev-parse below does not depend on that.
|
||||
set -u
|
||||
|
||||
root=$(git rev-parse --show-toplevel 2>/dev/null) || {
|
||||
echo "pre-push: FAIL - cannot resolve the repo root (git rev-parse --show-toplevel)." >&2
|
||||
exit 1
|
||||
}
|
||||
cd "$root" || exit 1
|
||||
|
||||
# ── WORKSPACE-ROOT ASSERTION (2026-08-05, R-204 rider) ───────────────────────────────────────────
|
||||
# Refuse a push from a clone outside the felhom workspace.
|
||||
#
|
||||
# WHY THIS IS A HOOK AND NOT A LINE IN A DOCUMENT: the workspace root is ALREADY written down, in
|
||||
# documentation/runbooks/workspace-CLAUDE.md and in the workspace-root CLAUDE.md ("stay inside it"),
|
||||
# and work drifted into a home directory anyway. A rule that has failed once as a reminder is not
|
||||
# fixed by writing it down again — it has to be asserted where it can bite.
|
||||
#
|
||||
# A PUSH IS THE RIGHT TRIGGER, deliberately: throwaway clones under /tmp for probes and red-proofs
|
||||
# never push, so nothing legitimate breaks. Reads and builds elsewhere stay unaffected.
|
||||
#
|
||||
# Symlinks are resolved on BOTH sides before comparison, so a symlinked path neither falsely passes
|
||||
# nor falsely fails. If the workspace root does not exist on this machine the check is SKIPPED, not
|
||||
# failed — this hook must not brick a legitimate clone on a different host.
|
||||
#
|
||||
# The only bypass is the documented `git push --no-verify`, whose use is already reportable.
|
||||
FELHOM_WORKSPACE_ROOT=/mnt/5_hdd/felhom.eu
|
||||
if [ -d "$FELHOM_WORKSPACE_ROOT" ]; then
|
||||
ws_real=$(cd "$FELHOM_WORKSPACE_ROOT" 2>/dev/null && pwd -P) || ws_real=""
|
||||
root_real=$(pwd -P) || root_real=""
|
||||
if [ -n "$ws_real" ] && [ -n "$root_real" ]; then
|
||||
case "$root_real/" in
|
||||
"$ws_real"/*) : ;; # inside the workspace — proceed
|
||||
*)
|
||||
echo "pre-push: PUSH REFUSED - this clone is OUTSIDE the felhom workspace." >&2
|
||||
echo " clone: $root_real" >&2
|
||||
echo " expected: under $ws_real (repos live in $ws_real/git/<repo>)" >&2
|
||||
echo " Work in the workspace clone, or bypass with 'git push --no-verify'" >&2
|
||||
echo " and state that you did in the session report." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
echo "pre-push: FAIL - python3 not found, so the gates CANNOT run. This is a failure, never a" >&2
|
||||
echo " pass by default. Install python3, or push with --no-verify and say so." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "pre-push [felhom-agent]: running scripts/agent_gates.py --fast ..."
|
||||
python3 "scripts/agent_gates.py" --fast
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "pre-push [felhom-agent]: PUSH REFUSED - gates exited $rc. Fix the finding above, or bypass with" >&2
|
||||
echo " 'git push --no-verify' and state that you did in the session report." >&2
|
||||
else
|
||||
echo "pre-push [felhom-agent]: gates OK - push proceeding."
|
||||
fi
|
||||
exit $rc
|
||||
+2781
-1
File diff suppressed because it is too large
Load Diff
@@ -1,123 +1,103 @@
|
||||
# CLAUDE.md — `felhom-agent`
|
||||
|
||||
> Loads when Claude Code touches this repo. Stable orientation only — **current state lives in
|
||||
> `CONTEXT.md` and the top of `CHANGELOG.md`**, never here. Cross-repo orientation: workspace-root
|
||||
> `e:\git\CLAUDE.md`.
|
||||
> Stable orientation only — **current state lives in `CONTEXT.md` and the top of `CHANGELOG.md`**,
|
||||
> never here. Cross-repo conventions (artifact taxonomy, access, clean-tree gate, secrets,
|
||||
> CHANGELOG/REPORT): workspace-root `/mnt/5_hdd/felhom.eu/git/CLAUDE.md`. Path-scoped detail:
|
||||
> `.claude/rules/`.
|
||||
|
||||
## What this repo is
|
||||
|
||||
`felhom-agent` is the operator-tier **host agent** that runs on each Proxmox host and owns **all**
|
||||
Proxmox interaction: provision/restore guests, host storage, backup/restore orchestration, the hub
|
||||
control loop, and a narrow per-guest local API. It is the **most privilege-sensitive** component.
|
||||
The operator-tier **host agent**, one per Proxmox host, owning **all** Proxmox interaction:
|
||||
provision/restore guests, host storage, backup/restore orchestration, the hub control loop, and a
|
||||
narrow per-guest local API. It is the **most privilege-sensitive component in the system**.
|
||||
|
||||
- Renamed former `proxmox-controller` repo.
|
||||
- **Distinct from `felhom-controller`** — that is the *in-guest* controller (Docker-only, no Proxmox
|
||||
creds). Do not confuse them.
|
||||
- Control plane, not data plane: if the agent dies, apps keep serving; only management degrades.
|
||||
- Renamed from `proxmox-controller`.
|
||||
- **Distinct from `felhom-controller`** — that is the *in-guest* controller, Docker-only, holding no
|
||||
Proxmox credentials. Do not confuse them.
|
||||
- **Control plane, not data plane:** if the agent dies, apps keep serving; only management degrades.
|
||||
- Pure Go stdlib + `golang.org/x/crypto`. No web frameworks.
|
||||
|
||||
## Read before writing code
|
||||
## Doing X → read Y
|
||||
|
||||
- **`REUSE.md`** — canonical helpers, format-safety guards, traps, seams. Check it first; update it
|
||||
in the same commit that changes a shared helper or pattern.
|
||||
- `CONTEXT.md` (current state + open threads) and the top `CHANGELOG.md` entry (authoritative history).
|
||||
- Design doc: `felhom.eu/documentation/architecture/03-host-agent.md` (locked). Platform facts:
|
||||
`felhom.eu/documentation/proxmox-platform.md` + `tests/phase{0,1-2,3,4}-findings.md`.
|
||||
| Doing | Read |
|
||||
|---|---|
|
||||
| writing any new code | `REUSE.md` — helpers, format-safety guards, traps, seams |
|
||||
| needing current state / open threads | `CONTEXT.md` + the top `CHANGELOG.md` entry |
|
||||
| Proxmox, reconcile or signed jobs | loads itself: `.claude/rules/proxmox.md` |
|
||||
| local API, authz or guest hooks | loads itself: `.claude/rules/localapi.md` |
|
||||
| backup, PBS or DR | loads itself: `.claude/rules/backup.md` |
|
||||
| storage or escrow | loads itself: `.claude/rules/storage.md` |
|
||||
| writing a health check | loads itself: `.claude/rules/health-checks.md` |
|
||||
| **release, build, publish, deploy, verify a version** | the **`felhom-build-deploy`** skill — **never hand-roll it** |
|
||||
| writing or reviewing a test, fixing a bug | the **`felhom-testing`** skill |
|
||||
| host addresses, break-glass, node facts | `felhom.eu/documentation/operations/nodes.md` — never restate them |
|
||||
| which box may I break | `felhom.eu/documentation/runbooks/target-selection.md` |
|
||||
| what version is live anywhere | ask the hub (`/hosts`, `/configs`) or the box — **never a doc** |
|
||||
| the authoritative design | `felhom.eu/documentation/architecture/03-host-agent.md` (locked) |
|
||||
|
||||
## Layout (verified against the tree)
|
||||
## The root-CLI fence — API-first, exactly three exceptions
|
||||
|
||||
```
|
||||
cmd/felhom-agent/ main + flags + --selftest modes + the daemon entry
|
||||
cmd/felhom-opsign/ offline operator signing CLI (SSHSIG)
|
||||
internal/authz/ operator signed-op verifier (SSHSIG) + durable FileNonceStore
|
||||
internal/backup/ vzdump backup runner + restore-test scheduler + report store
|
||||
internal/capability/ live sudo-policy capability probe (degradation visibility)
|
||||
internal/config/ JSON config + FELHOM_AGENT_* env overlay; secrets redacted (Redacted())
|
||||
internal/desired/ hub desired-state syncer (envelope observer)
|
||||
internal/escrow/ PBS-key escrow (zero-knowledge recovery code)
|
||||
internal/guesthook/ pre-start self-heal hookscript install
|
||||
internal/hub/ daemon: HostReport collector + Bearer client + resilient Loop
|
||||
internal/lanresolver/ split-horizon DNS on guest IP change (dnsmasq RESTART, not reload)
|
||||
internal/localapi/ per-guest local API: token store, disks/format, guest binds, controller swap,
|
||||
stale-lock recovery, pinned self-signed leaf
|
||||
internal/log/ slog setup
|
||||
internal/pbs/ PBS-API client (fingerprint-pinned) + verify maintenance loop
|
||||
internal/provision/ guest bootstrap back-half (token mint → bootstrap.json → pct bind)
|
||||
internal/proxmox/ API-first Client + fenced root-CLI Privileged + UPID WaitTask
|
||||
internal/reconcile/ reconcile engine + reversibility gate + op journal + crash recovery
|
||||
internal/signedjobs/ operator-signed destructive executors (wipe, decommission)
|
||||
internal/storage/ storage observer + durable ids + role/claim classifiers + SudoHostOps + watchdog
|
||||
```
|
||||
|
||||
## Build / run
|
||||
|
||||
- Module `gitea.dooplex.hu/admin/felhom-agent`; binary `felhom-agent` (`cmd/felhom-agent/`).
|
||||
- **Pure Go stdlib + `golang.org/x/crypto` only** — no web frameworks. `go.mod` directive go 1.25.0;
|
||||
the build server (192.168.0.180) runs a newer upstream Go — build/run live tests there (same LAN
|
||||
as the demo host).
|
||||
- Version via `-ldflags "-X main.version=<v>"`; `--version` flag. Bump on meaningful changes + CHANGELOG entry.
|
||||
- **Full build/deploy/publish runbook: use the `felhom-build-deploy` skill.** Summary:
|
||||
|
||||
| Step | Where | One-liner |
|
||||
|---|---|---|
|
||||
| Build | 180 | `cd ~/git/felhom-agent && git pull && go build -ldflags '-X main.version=<v>' -o /tmp/... ./cmd/felhom-agent` |
|
||||
| Deploy | felhom-pve | backup `.bak-<old>` → `install -m0755` → `systemctl restart felhom-agent` (non-root `felhom-agent` user, config `/etc/felhom-agent/agent.json`) |
|
||||
| Ship configs | felhom-pve | sudoers (`/etc/sudoers.d/felhom-agent`) + guarded-mkfs wrapper WITH the binary when `configs/` changed |
|
||||
| Publish | local | `scripts/publish-agent.sh <ver> <bin>` (REGISTRY_* creds); hub Day-0 manifest vouch = operator follow-up |
|
||||
| Verify | felhom-pve | `felhom-agent --version` + journal (clean ReassertGuestBinds, no capability degradation) |
|
||||
|
||||
## Proxmox model (the load-bearing rules)
|
||||
This is in the core because breaching it is how this component stops being auditable.
|
||||
|
||||
- **API-first** via a scoped `FelhomAgent` token. Raw root-CLI is **fenced to exactly 3 exceptions**:
|
||||
keyctl `pct create` (golden image), USB mount/fstab, SMART/sensors. `Client` never shells out;
|
||||
`Privileged` never makes HTTP calls (asserted by `routing_test.go`). Keep that fence.
|
||||
- **Every mutating op is async** → returns a UPID → `WaitTask` asserts `exitstatus == "OK"`. A 200 on
|
||||
the POST is **not** success; authorization can fail at task execution.
|
||||
- **TLS:** SHA-256 leaf-cert pinning (self-signed host cert). No insecure default.
|
||||
- **Privsep token gotcha:** a `--privsep 1` token's rights = intersection of the backing user's perms
|
||||
AND the token's ACLs — the role must be granted on **both**, or every call 403s.
|
||||
- Destructive ops go through the reconcile gate / signed-jobs path — never call `Client.DestroyLXC`/
|
||||
`Vzdump`/`SetConfig` ad-hoc (REUSE.md §3).
|
||||
keyctl `pct create` (golden image), USB mount/fstab, SMART/sensors.
|
||||
- **`Client` never shells out; `Privileged` never makes HTTP calls** — asserted by `routing_test.go`.
|
||||
Adding a method to `proxmox.Privileged` breaks the fence; use `proxmox.Runner` plus a new sudoers
|
||||
`Cmnd_Alias` and `validate.go`-style checks (`REUSE.md` §3).
|
||||
- **Destructive ops go through the reconcile gate / signed-jobs path.** Never call
|
||||
`Client.DestroyLXC` / `Vzdump` / `SetConfig` ad-hoc — that skips classification, signature,
|
||||
per-guest serialization and crash recovery.
|
||||
- **Ownership must be PROVEN, never assumed.** A raw `ListLXC` list is not "guests the agent owns";
|
||||
intersect with `Client.Pool` membership and fail safe on a read failure (audit A1).
|
||||
|
||||
## Demo host (for live tests)
|
||||
## Gates — ONE entry point
|
||||
|
||||
Node **`demo-felhom`**, API `https://192.168.0.162:8006`. SSH alias `felhom-pve` (root@pam) —
|
||||
available to CC; use `MSYS_NO_PATHCONV=1` for pct commands. The agent pins the served leaf cert —
|
||||
verify the fingerprint still matches before a live run. Selftest modes (run from 180, pointed at the
|
||||
demo API): `--selftest[=read|task|hub|storage|backup|restore-test|pbs-verify]`; no flag = the daemon.
|
||||
**Run `python3 scripts/agent_gates.py` from the repo root after ANY change here.** It runs this
|
||||
repo's gates — `reuse_refs_check` and `instructions_gate`, both the **shared** copies in
|
||||
`felhom.eu/scripts/`, never copied into this repo (a copy recreates the drift they detect; an absent
|
||||
sibling clone FAILS). `--fast` selects the gates touching no network and no container runtime; today
|
||||
that is all of them. **A missing gate is a FAILURE, never a skip.**
|
||||
|
||||
**The pre-push hook** (`.githooks/pre-push`) runs it with `--fast` and refuses a failing push. It is
|
||||
**per-clone** — switch it on once with `git config core.hooksPath .githooks`, and a manual run WARNS
|
||||
when this clone is unarmed. `git push --no-verify` bypasses it deliberately; **say so in the session
|
||||
report when you use it** — CI re-runs the same entry point on every push and **emails the operator on
|
||||
failure**, so a bypass is noticed even though it is not blocked (R-168, CLOSED 2026-08-02).
|
||||
|
||||
<!--
|
||||
WHY ONE ENTRY POINT (2026-08-02, R-29): a census of all gates across the four repos found every check
|
||||
a CLAUDE.md names was passing, and two of the four nobody is told to run were failing. This repo was
|
||||
the extreme case — nothing ran against it at all, and 90 cited paths were checked by no one.
|
||||
-->
|
||||
|
||||
## Live validation — the fence
|
||||
|
||||
Exercise the **SERVER-SIDE PIPELINE** a real user triggers, end-to-end. **The forbidden shortcut is
|
||||
BYPASSING it** — the F9 episode was a raw guest-attach with hand-set state, and it proved nothing.
|
||||
|
||||
`claude-in-chrome` is NOT available on DooPlex. Invoking the exact endpoint the UI invokes is an
|
||||
acceptable proxy — **say which method was used**. Low-level mechanism tests where the direct call IS
|
||||
the mechanism are exempt.
|
||||
|
||||
## Conventions
|
||||
|
||||
### Trunk-based — no branches
|
||||
- **Trunk-based — no branches.** All shippable work commits directly to `main`; `main` equals what is
|
||||
deployed. Report-only artifacts (audits, findings, fixspecs) go to `felhom.eu/documentation/`.
|
||||
- **Unattended escape hatch:** if a fix cannot be cleanly verified and shipped, **revert and report**
|
||||
— never park it on a branch.
|
||||
- **Logging**: the slog logger fans out to journald (configured level) plus the always-DEBUG
|
||||
`applog.Ring` (remote pulls). English, keys-never-values, durations on outcomes. Full rules:
|
||||
`felhom.eu/documentation/runbooks/logging-conventions.md`.
|
||||
- Update `REUSE.md` in the same commit that adds, changes or deprecates a shared helper or pattern.
|
||||
|
||||
All shippable work commits **directly to `main`**; `main` equals what is deployed.
|
||||
- Report-only artifacts (audits, findings, fixspecs) → `felhom.eu/documentation/` (`audits/`, `backlog/`).
|
||||
- Risky/supervised fixes are spec'd, then implemented **during the supervised session, on `main`**.
|
||||
- Unattended escape hatch: if a fix can't be cleanly verified/shipped, revert + report — never park on a branch.
|
||||
## End-of-session checklist
|
||||
|
||||
> **In every repository where you make a change, update both files in that repo:**
|
||||
> - **`CHANGELOG.md`** — cumulative log, newest on top.
|
||||
> - **`REPORT.md`** — **overwrite** with the most recent implementation/validation summary only.
|
||||
>
|
||||
> **Never write secrets** into any committed file — reference them as "stored out-of-band".
|
||||
|
||||
- Code quality: verify generated code for bugs/edge cases; add debug logging; **ask rather than
|
||||
guess** when you'd otherwise invent input/output.
|
||||
- Update `REUSE.md` if you added/changed/deprecated a shared helper or pattern (same commit).
|
||||
- Testing doctrine (non-hollow tests, red-proofs, seams): use the `felhom-testing` skill.
|
||||
|
||||
### Live validation
|
||||
|
||||
Exercise the SERVER-SIDE PIPELINE a real user triggers, end-to-end. The forbidden shortcut is
|
||||
BYPASSING it (the F9 episode: raw guest-attach + hand-set state). Invoking the exact endpoint the UI
|
||||
invokes is an acceptable proxy when a browser isn't available — say which method was used. Low-level
|
||||
mechanism tests where the direct call IS the mechanism are exempt.
|
||||
|
||||
## Workflow & artifacts
|
||||
|
||||
- Implement **`TASK.md` / `TASK-*.md`** specs (when placed as `TASK.md` or told to), then push +
|
||||
CHANGELOG + REPORT.md.
|
||||
- **`RUNBOOK-*.md`** — an operational procedure. CC executes the steps it has access and capability
|
||||
for, including live validation on the demo Proxmox host (CC has root@felhom-pve SSH + the
|
||||
felhom-agent token). Mark a step HUMAN only when it genuinely needs physical presence, a real-world
|
||||
decision, or credentials CC truly lacks. Judgment still applies: confirm before irreversible ops on
|
||||
real customer data — demo scratch guests are fair game.
|
||||
- **`CHANGELOG.md`** (cumulative, newest on top) and **`REPORT.md`** (overwritten with this run only)
|
||||
— in every repo touched.
|
||||
- **`CONTEXT.md`** — decisions, state, what is next.
|
||||
- **`REUSE.md`** — if a shared helper or pattern moved.
|
||||
- **A finding goes in `felhom.eu/documentation/backlog/OPEN-ITEMS.md` first**, never only in a report
|
||||
or an audit.
|
||||
- **Confirm your own last push's CI run went green, by run ID** — CI mails on failure, which is a PUSH
|
||||
signal; this is the PULL check that catches a lost or unread mail. An unchecked green is an
|
||||
assumption, not an observation.
|
||||
|
||||
+371
-4
@@ -3,8 +3,370 @@
|
||||
> Snapshot of the current state + open threads. Authoritative history lives in `CHANGELOG.md` (top
|
||||
> entry = current); the end-of-task detail lives in `REPORT.md`.
|
||||
|
||||
## R-199 (v0.125.0) — links 6–8 of the recovery chain, assembled and walked
|
||||
|
||||
`POST /escrow/recover-offsite-password` (pinned local API, `withGuest`): the controller supplies the
|
||||
customer's recovery code, the agent fetches THIS host's own sealed blob from the hub
|
||||
(`hub.Client.FetchIdentityEscrow` → `GET /hosts/{id}/escrow`, hub >= v0.94.0, self-scoped by the
|
||||
per-host key), unseals it via `escrow.OffsiteKeyRecoverer`, and returns **only** the offsite restic
|
||||
repository password plus its sha256.
|
||||
|
||||
**Rules that must not erode:**
|
||||
- **Only that field.** Not the tunnel token, not the PBS token, not the WG key — the controller is a
|
||||
trust tier down and needs none of them. Narrowing cost nothing and is not recoverable later.
|
||||
- **The unseal stays in the agent.** `age` is an agent runtime dependency (`/usr/bin/age` — hardcoded,
|
||||
no config override; 1.2.1 on demo-felhom) and is deliberately absent from the controller image.
|
||||
- **R:** in memory for one call, cleared on the success path AND every failure path, never on disk,
|
||||
never in argv, never logged at any level including inside an error, never echoed. Verified live: 0
|
||||
log lines, 0 files, 0 leftover `felhom-idesc-*` dirs, with a positive control proving the search worked.
|
||||
- **Three distinct outcomes**, not one generic failure: no blob (404), a bundle that opens but predates
|
||||
the field (409 — pre-fork-4, cannot be retro-fitted), a code that does not open it (400 — fail-closed
|
||||
at age's KDF, nothing written).
|
||||
- **The wiring is pinned by an AST walk** (`cmd/felhom-agent/escrow_recover_wiring_test.go`):
|
||||
`main` → `runDaemon` → `buildLocalAPIServer`, an `escrow.OffsiteKeyRecoverer` constructed there, the
|
||||
`Options.EscrowRecovery` field present, and the fetcher calling the DAEMON's own `hubClient` (the
|
||||
self-scoping that makes cross-host retrieval impossible is a property of WHICH key is used).
|
||||
Links 6 and 7 were two of this project's six built-but-never-wired instances.
|
||||
|
||||
**Proven live on demo-felhom 2026-08-04:** recovered sha256 == on-disk sha256 == the hub's stored hash.
|
||||
A wrong code five minutes earlier failed closed. **The chain stops at link 8** — nothing installs a
|
||||
recovered password, reopens a repository, or restores a file.
|
||||
|
||||
**§8.6, fixed while here:** `runSelftestIdentityConsume`'s success line used to recite
|
||||
"tunnel_token + pbs_token", which became a misstatement when v0.77.0 sealed the repository password
|
||||
into the same bundle — anyone reading it would conclude the password was not there. It now names what
|
||||
THIS bundle carried and what it did not.
|
||||
|
||||
## Current
|
||||
|
||||
- **2026-08-03 — v0.123.0 (R-185): a tier the box cannot READ now says so.** The agent's token had
|
||||
`FelhomAgentStore` on `local`, `local-lvm`, `felhom-pbs` and **not** on `felhom-backup` — the
|
||||
storage both demo boxes configure as `local_backup_target`. That storage answered `{"data":[]}`
|
||||
through the token while root listed three archives, and `pickForThisRun` skipped it as *"no settled
|
||||
archive yet"* — **which is what a brand-new tier reports**, so the host tier was never
|
||||
restore-testable and nothing said so.
|
||||
- **The permission question is asked directly**, because unlike the listing it has a definite
|
||||
answer: `Client.Permissions` reads `/access/permissions?path=/storage/<target>` **as the agent's
|
||||
own token**, and `storeGrantStatuses` emits one `capability.Status` per configured tier. It
|
||||
composes AROUND the sudo prober, the way `poolReadStatus` already does — an API read does not
|
||||
belong inside a sudo-policy probe. `Status`'s wire shape is untouched, so the hub's critical
|
||||
degraded alert applies with **no hub change**.
|
||||
- **MEASURED FIRST, and the obvious reading is wrong:** an ungranted path answers neither empty nor
|
||||
403 — it carries the privileges INHERITED from the box-wide `/` grant
|
||||
(`Sys.Audit, SDN.Use, Datastore.Audit`). Checking path-presence, or `Datastore.Audit`, reports a
|
||||
blinded storage HEALTHY. The probe tests **`Datastore.AllocateSpace`**; re-measure before ever
|
||||
changing that constant (`storeGrantRequiredPriv`, red-proved).
|
||||
- **The probed set comes from `BackupTiers()`, never a fixed list** — a hardcoded probe list is the
|
||||
defect reproduced inside the fix. Critical, EXCEPT the `local` fallback target (reported, but it
|
||||
does not page). It never consults content, so it cannot alarm on a newborn tier; it never reports
|
||||
ok when it could not ask.
|
||||
- **LIVE:** degraded observed on the still-blind box (hub emailed `agent_capability_degraded`) →
|
||||
grant applied on **both** demo boxes → token lists 3 and 4 archives → `ok=70 total=70 degraded=0`
|
||||
and `degraded → ok` at the hub → **the host tier became a due-check candidate for the first time**,
|
||||
correctly picking the 08-02 archive (08-03 had not settled 24 h).
|
||||
- **The installer's real defect was NOT `PVE_STORAGES`** — see `felhom.eu` CONTEXT S-22: Case A
|
||||
grants, the Scenario-F reuse arm did not. Fixed in installer **1.24.0** with a gate.
|
||||
|
||||
- **2026-08-03 — v0.122.0 (R-189 · R-188 · R-186): three signals that lied about their own work.**
|
||||
None touches data; all three cost attention, which every other signal depends on.
|
||||
- **R-189 — a passing restore-test no longer vanishes on a restart.** `restore_tests[]` came only
|
||||
from the in-memory `backup.Store` (*"lost on restart; the cadence re-populates"* — true under a
|
||||
timer, FALSE since R-86, because the agent will not re-test a proven archive). **Observed live:**
|
||||
a 14.5 GB offsite PASS at 15:25:14, agent restarted 2 m 43 s later, hub logged `0 restore-tests`
|
||||
twice. `RestoreTestState` now stores `tier` + `verified` beside the archive (v3 shape; v1/v2
|
||||
still read, and a record missing archive-or-tier is NOT reported), exposes
|
||||
`ProvenRestoreTests`, and `Collector.SetProvenRestoreTests` merges it — **one entry per tier,
|
||||
newest by `TestedAt` wins**, so a fresh failure beats a stored success and a tier never appears
|
||||
twice. Wiring pinned by an AST test: the method this replaces (`Snapshot`) claimed a
|
||||
"host-report gauge" in its doc comment and had **no caller** for weeks.
|
||||
- **ONLY SUCCESSES ARE PERSISTED, and the reason is now in the code:** a success *suppresses*
|
||||
future work (a proven archive is never re-tested, so a lost proof leaves the box quietly less
|
||||
tested than it believes); a failure *causes* future work and heals itself at the next evaluation.
|
||||
- **R-188 — the release stopped emailing false failures.** Only the tag PUSH moved (build → tag
|
||||
locally → publish → push tag): the push is what wakes CI, and a tag visible before its package
|
||||
made the gate correctly fail a correct release ~half the time. The old order's invariant is now
|
||||
asserted directly — `check-published-versions.py` refuses a **published version with no tag**, as
|
||||
a bounded, printed probe (the package listing api is still 401 without a token, re-measured).
|
||||
- **R-186 — a released binary is verifiable.** `-trimpath -buildvcs=false`: same source → same
|
||||
bytes whether or not the tag exists. Measured. `publish-agent.sh`'s fallback also forced
|
||||
`CGO_ENABLED=0` and built a **74 KB different** binary for the same version — both paths now
|
||||
identical. The verification command is in `CLAUDE.md`.
|
||||
|
||||
- **2026-08-03 — v0.121.0 (R-86): the restore-test follows the BACKUP, not the clock.** The ticker is
|
||||
now only the **evaluation interval**; a tier is **DUE** when its newest archive that has settled for
|
||||
`settle` (default 24 h) **has not been proven**. Daily tier → proved daily on yesterday's archive;
|
||||
weekly tier → weekly on its own; newborn → UNKNOWN. **The trap, so it is not reintroduced:** the
|
||||
literal reading of R-86 — *"due when the newest archive is ≥ 24 h old"* — is NEVER true on a daily
|
||||
tier (a new archive resets the age before it reaches the lag), so it switches restore-testing off
|
||||
where it matters most. Red-proved at 0 runs over 5 simulated days.
|
||||
- **The state now records WHICH archive was proven**, not just when a tier passed. A pre-R-86 file
|
||||
keeps its time (ordering survives) and yields no proven archive → each tier is due once after the
|
||||
upgrade, deliberately.
|
||||
- **The old cadence key:** `restore_test_cadence_seconds` is DEPRECATED. Negative still DISABLES
|
||||
(verbatim); a positive value now seeds the **settle lag** and the daemon WARNs once at start-up
|
||||
naming `restore_test_eval_interval_seconds` (default 6 h) and `restore_test_settle_seconds`
|
||||
(default 24 h). It is NOT carried into the evaluation interval.
|
||||
- **6 h is bounded from both ends:** measured evaluation cost (local 18 ms, PBS-over-WAN 392 ms,
|
||||
both 430 ms) says cost is irrelevant; the ceiling is that a FAILING tier stays due, so the
|
||||
evaluation interval is also its retry interval for a multi-GB restore.
|
||||
- The due-check now runs **before** the heavy-operation gate is taken (a frequent poll must not be
|
||||
able to make a starting backup record a failure — F-A1), and the candidate picker skips archives
|
||||
failing `archivePlausiblyComplete` (a phantom would be due forever and fail forever).
|
||||
- New read-only `--selftest=restore-test-due` prints the per-tier verdict + its cost.
|
||||
- **v0.121.1 — a quiet evaluation is AUDIBLE.** "Nothing is due" is now the NORMAL outcome, and at
|
||||
DEBUG it was silent: an empty journal would have been equally consistent with a healthy loop and
|
||||
a dead goroutine (standing rule 3 — the shape the R-88 watcher was retired for). A not-due
|
||||
evaluation logs ONE INFO line naming every tier's verdict; an unlistable tier reads `UNKNOWN`
|
||||
with its error in that same line.
|
||||
- **PROVEN LIVE 2026-08-03 on demo-felhom:** due-triggered offsite restore-test of a 14.5 GB
|
||||
encrypted PBS archive — restored, booted, verified, scratch destroyed, **635 s**; the state then
|
||||
named that archive, a second evaluation ran nothing, and an agent restart ran nothing.
|
||||
- **R-185 (filed, NOT fixed here):** on demo-felhom the agent token has no ACL on
|
||||
`/storage/felhom-backup`, so its content listing comes back EMPTY (root sees 3 archives) — the
|
||||
host tier has never been restore-testable there, and the due-check cannot distinguish that from
|
||||
a newborn tier.
|
||||
|
||||
- **2026-07-28 — v0.107.0: F-REBOOT fixed — a guest rebooted mid-backup now comes back by itself.**
|
||||
New `internal/localapi/guestpower.go`: a 60 s watchdog that starts a guest which is `onboot:1`,
|
||||
stopped, unlocked, and has no vzdump in flight. It closes the two narrow gaps that let
|
||||
`RecoverStaleLockedGuests` miss campaign fault 11 — that recovery acts only on a **stale vzdump
|
||||
lock** (fault 11's guest was unlocked) and runs **once at agent startup** (fault 11's guest went
|
||||
down while the agent was already up). `onboot` is the deliberate-stop discriminator and is *not*
|
||||
invented here: it is already what `stalelock.go` uses for this decision, it is 0 on scratch/golden
|
||||
guests, and it is what `pve-guests` consults at host boot — so the agent agrees with the platform
|
||||
instead of keeping a second private definition of "should be running". Retry bounded at 3
|
||||
(1m/2m/4m) then escalates **once**; an unbounded silent retry loop is the over-correction here.
|
||||
Live on demo-hp: **120 s unattended** recovery vs the incident's **587 s** with a human; Scenario B
|
||||
proven (an `onboot:0` guest left stopped throughout). Detail: `REPORT.md`.
|
||||
|
||||
- **2026-07-28 — F-LEAK took THREE attempts; v0.108.0 and v0.110.0 are the corrections.** The cause is
|
||||
structural: `FelhomAgentGuest` is granted at `/pool/felhom` and a guest joins that pool only when its
|
||||
restore **completes**, so a *failed* restore-test leaves a pool-less guest out of reach (403).
|
||||
**(1) v0.107.0 pool adoption — REFUTED LIVE:** `PUT /pools/{pool}` also requires `VM.Allocate` on the
|
||||
VM being added, so membership cannot bootstrap its own authority; removed in **v0.108.0**.
|
||||
**(2) host-install v1.21.0 per-path `/vms/990000..990009` ACLs — works, but exactly ONCE per slot:**
|
||||
PVE's destroy calls `AccessControl::remove_vm_access` (`API2/LXC.pm:906`) which deletes every ACL at
|
||||
`/vms/<vmid>` (`AccessControl.pm:1898`) — **the grant is consumed by the op it authorises**. Caught by
|
||||
counting ACL rows after the fix, not by reasoning. **(3) v0.110.0 SHIPPED —
|
||||
`Privileged.DestroyScratchLXC`, the FOURTH root-fenced exception** (was exactly three: keyctl
|
||||
`pct create`, USB mount/fstab, SMART/sensors). Band enforced in **sudoers literally**
|
||||
(`pct destroy 99000[0-9] --purge`) + re-checked in code + journal provenance at the caller; none is
|
||||
consumed by use. API destroy still tried FIRST; band ACLs stay provisioned so the common case needs no
|
||||
privileged call. **Ships with a sudoers change — deploy `configs/felhom-agent.sudoers` WITH the
|
||||
binary.** Live: token 403 on a stranded scratch → fenced path removed the guest and all 3 LVs; sudo
|
||||
PERMITS the band and REFUSES `9201`/`9100`/`9999`/`990010`/`1`, and refuses `pct start 990000` too.
|
||||
|
||||
- **2026-07-28 — v0.109.0: the guest-power watchdog got the observable it shipped without.** A
|
||||
self-correction: v0.107.0's watchdog logged only at startup and when it *acted*, so on a healthy box
|
||||
its health could be read only from **absence** — F-OBS's exact shape, shipped in the same session
|
||||
F-OBS was fixed in the controller. Now an INFO summary every 10th sweep carrying
|
||||
`sweeps_since_boot`/`guests_evaluated`/`currently_stopped`. An **aborted** sweep (unproven
|
||||
ownership) does not count, or the heartbeat would claim liveness for a watchdog examining nothing.
|
||||
|
||||
- **2026-07-28 — v0.106.0: F-CRIT-2 fixed — a failed backup no longer looks like a fresh one.**
|
||||
`NewestArchiveTime` counted an aborted PBS upload (1 byte, manifest-less, NEWEST) as a successful
|
||||
backup, so the tier reported fresh, went **not due**, and was never retried — 7 days of silence on
|
||||
the real 168h cadence, invisible to both the R-88 breaker (defers only DUE tiers) and the hub
|
||||
deadline monitor (reads the same freshness). Now only *plausibly complete* entries count, via a
|
||||
measured floor `minPlausibleArchiveBytes` = 1 MiB; undecidable ⇒ not counted.
|
||||
**Size is the only tier-agnostic discriminator** — `verification` and `encrypted` are absent on
|
||||
every local (dir) archive and on a good PBS snapshot until verify-new catches up, so gating on
|
||||
either would reject 100% of local backups and cause fleet-wide backup THRASH. Floor measured:
|
||||
smallest real backup on the fleet is 612,397,450 B, so 1 MiB leaves 584x headroom (asserted by a
|
||||
test). Rejections logged at WARN once per volid. Re-tested live by replaying campaign fault 2 on
|
||||
demo-hp — both directions, incl. a no-thrash window with 91 scheduler ticks as the positive
|
||||
observable. Deployed on both boxes. Detail: `REPORT.md`.
|
||||
**Also established:** server-side prune does NOT count phantoms toward `keep-last` (dry-run kept
|
||||
2 real + the phantom) ⇒ **no retention/data-loss bug** — but it never removes them either, so they
|
||||
accumulate. Filed as R-99 (LOW).
|
||||
|
||||
- **2026-07-25 — v0.95.0 (additive): SMART coverage fixes (spike B+A) + device model.** Union-path
|
||||
drives (USB/registry) now get SMART via `storage.SmartReader.SMARTForBacking` wired into the localapi
|
||||
`/disks` union (localapi `Smart` seam); `smartDeviceFor` resolves dm/LVM to the whole disk via
|
||||
`/sys/block/<dm>/slaves` (recursive, skips >1-disk); the builtin `local` dir on the LVM root gets a
|
||||
**SMART-only** device from its containing filesystem (never touches backing/durable_id — the
|
||||
removable-safety guard in build() stays intact); `SmartSummary.ModelName` captured from smartctl. The
|
||||
watchdog `Known` path stays enrich-free. Consumed by controller v0.171.0. Source of WHERE:
|
||||
`felhom.eu/documentation/audits/SPIKE-smart-coverage-2026-07-25.md`.
|
||||
- **2026-07-24 — v0.94.0 (additive): SMART serialized into /disks.** `localapi.DiskInfo` gains
|
||||
`Smart *hub.SmartSummary` (omitempty), copied from the target's already-computed Observe-time
|
||||
enrichment when `Health != ""` — no new smartctl load, no endpoint, no sudoers/MinAgent change. The
|
||||
controller v0.169.0 renders a "Lemezek állapota" card + 6h degradation alert from it; old controllers
|
||||
ignore it. **NOTE: at the remote-site vacation window the agent is DOWN (localapi binds .162 → fails),
|
||||
so live /disks-from-real-agent validation is deferred — the field is unit-proven; publish only.**
|
||||
- **2026-07-22 — v0.93.0 is the FLEET AGENT.** Built, published (sha `a68b2ff73200622e…`),
|
||||
Day-0-manifest-vouched (MinAgent also 0.93.0, operator-ruled) and deployed to BOTH boxes
|
||||
(`demo-felhom-8363b5` + `demo-hp-bb76ea`, the latter over G1 break-glass — still no key baked);
|
||||
clean-restart 5/5 on both, `.bak-0.92.1` retained. Discharges the onboarding runbook §A5
|
||||
ceremony gate. Record: `felhom.eu/documentation/pilot/RUNBOOK-publish-agent-0.93-2026-07-22.md`.
|
||||
**The bullet below ("agent is DOWN … deployed 0.90.0") is SUPERSEDED history** — vmbr0 was made
|
||||
static .162 on 2026-07-20 (F1 mitigation) and the agent has been up since; kept for the record.
|
||||
|
||||
- **2026-07-20 — REMOTE SITE until ~2026-08-02; the agent is DOWN there and cannot self-recover.**
|
||||
felhom-pve moved off the home LAN; `ssh felhom-pve` = tailnet `100.70.170.35` (direct, ~37 ms). The
|
||||
host is on DHCP and holds `192.168.0.147`, so `localapi`'s literal `192.168.0.162` bind fails with
|
||||
`bind: cannot assign requested address` — the daemon exits ~1.1 s after start, systemd gave up after
|
||||
4 retries, and a manual restart reproduces it exactly. Deployed binary is **0.90.0**. Fix needs
|
||||
`listen_addr` in `/etc/felhom-agent/agent.json` **and** the guest bootstrap endpoint (plus the
|
||||
pinned leaf's SAN) → **Viktor GO**; re-pinning to another literal just re-breaks on the next lease.
|
||||
Also re-observed each start: `pbs: cannot read token secret … /etc/pve/priv/storage/felhom-pbs.pw:
|
||||
permission denied` (R-39-adjacent). Evidence + ranked findings:
|
||||
`felhom.eu/documentation/audits/AUDIT-vacation-remote-ops-2026-07-20.md`
|
||||
|
||||
- **v0.90.0** (2026-07-17) — **agent train: guest RAM resize (R-24) + fast-tick (R-28); LIVE on BOTH
|
||||
demo hosts (felhom-pve + nested demo-vm-felhom-4846bc).** MinAgent coupling: felhom-controller
|
||||
v0.143.0 gates its resize UI on this agent. (1) **R-24 guest RAM resize (controller-direct)** —
|
||||
self-scoped `GET`/`POST /guest/memory` (`internal/localapi/guestmemory.go`); the AGENT enforces
|
||||
every bound fresh per request (min 2048 / max host_total−2048 / shrink floor max(2048, usage+512)) and
|
||||
applies via PVE `SetConfig` — **live cgroup apply, no reboot** (Phase-0 PROVEN on the nested box; the
|
||||
break-glass access path + the proof are in `~/.claude/.../nested-vm-access-breakglass.md`). Verify-
|
||||
after-apply re-reads maxmem before claiming success. New narrow `MemoryOps` seam (GuestAPI untouched);
|
||||
memory only. (2) **R-28 fast-tick** (`internal/fasttick/`) — while any desired-state item is unapplied
|
||||
(esp. the pre-tunnel WG-registration window a hub poke can't reach) pulse the shared out-of-band
|
||||
trigger every 30 s, self-disarm on convergence; four cached sources (desired-gen==0, reconcile
|
||||
Planned−Pending>0, pbsdr waiting_secret ONLY, wgtunnel desired-not-operational). Seams:
|
||||
`reconcile.Engine.LastResult()` + `wgtunnel.Manager.TunnelConvergence()` (cached — no per-tick exec).
|
||||
(3) **Guests-0/0** REFUTED live: the 0/0 was the pre-provision window (guest not yet created), not a
|
||||
pool-membership bug; the fast-tick shortens that window. **OPEN (operator GO):** publish 0.90.0 +
|
||||
hub Day-0 manifest vouch + MinAgent-floor raise to 0.90.0 (password-gated UI; the safety gate — both
|
||||
agents on 0.90.0 — is satisfied and the coupling is proven live via the version header). See REPORT.md.
|
||||
|
||||
- **v0.89.0** (2026-07-16) — **agent train: three bundled agent-plane items; built + published to
|
||||
Gitea (sha256 `3969fd91…`); paired with hub 0.59.0 (LIVE).** (1) **pbsdr self-grant (R-22)** —
|
||||
closes the F4 self-deadlock: a 403 on the token-auth `StorageEntry` pre-check now self-grants via
|
||||
the root wrapper + re-reads instead of aborting before the grant (the demo's `felhom-offsite`
|
||||
case). (2) **escrow config live-reload** — `/escrow/preflight`'s `pbs_storage_id` row now reads
|
||||
the live agent.json (late-bound `CurrentPBSStorageID`) so a pbsdr-seeded id flips green with no
|
||||
restart. (3) **agent-plane poke listener (Direction-2a)** — `internal/poke`: contentless UDP poke
|
||||
bound to the box WG /32 (port **51822**), leading-edge debounced, fires the hub-loop out-of-band
|
||||
trigger for an immediate desired-state cycle; enabled with `wg_tunnel.enabled`; first slice of
|
||||
R-13. Red-proofs for all three (run-fail-revert). **ALL THREE LIVE LEGS PROVEN on the demo
|
||||
(2026-07-17), demo now LIVE on 0.89.0:** Scenario 4 floor-driven A/B train 0.88→0.89 (operator
|
||||
signed+enqueued the `agent_update` op — the vouch+floor alone does NOT trigger it; committed, no
|
||||
rollback); Scenario 1 R-22 self-heal (marker aside + ACLs revoked → `pre-check 403 … self-granting
|
||||
(R-22)` → `converged state=adopted` in ~3 s, ACLs restored, offsite active); Scenario 3 poke→tick
|
||||
~31 ms ep0→box + immediate report cycle (save→tick ≈ ~0.45 s). Details: REPORT.md.
|
||||
|
||||
- **v0.88.0** (2026-07-13 eve) — **controller-driven escrow ceremony (agent half), LIVE on demo
|
||||
host + drill VM (63/63 capabilities both).** `--output=json` machine mode (text mode
|
||||
byte-identical; extraction into `escrowCeremony()`); the ONE fixed argv
|
||||
(`escrow.CeremonyArgs()` — shared by the localapi exec + the `escrow-ceremony` capability
|
||||
(Critical, pbs_dr-gated EXPLICIT) + the new `FELHOM_ESCROW` sudoers alias, three-way
|
||||
pin-tested); localapi job endpoints (`POST /escrow/ceremony` single-flight 60 s,
|
||||
status, ONE-SHOT claim → 410, 10-min TTL → `unclaimed_void`, `GET /escrow/preflight`).
|
||||
R in-memory ONLY (never the job struct — snapshot-hygiene-tested; restart loses it safely).
|
||||
Live-proven on drill endpoint-exact: stage → preflight all-green (live FELHOM_ESCROW
|
||||
list-probe) → job ~4 s → hub blob `restic_pw_sha256` covering (repaired the spike's hash-less
|
||||
blob) → claim 200 once → 410. Coupled: controller v0.127.0 (MinAgent 0.88.0 for the wizard).
|
||||
**OPEN: publish 0.88.0 + Day-0 manifest vouch (operator) at the next train; deployed hosts got
|
||||
direct deploys.** Details: REPORT.md + felhom.eu RUNBOOK-escrow-ceremony.md (F1 threat model).
|
||||
|
||||
- **v0.87.0** (2026-07-13) — **SystemDisks device-mapper walk (IA finding 2, MEDIUM):
|
||||
legacy-boot hosts get a working drive wizard.** Operator ruling (approved 2026-07-13,
|
||||
verbatim): *resolve device-mapper/raid parents — for the root filesystem's backing block
|
||||
device, walk `/sys/block/<dev>/slaves` recursively down to physical disks; those, plus any
|
||||
ESP holder when present, are system. Disks outside that set become wizard candidates (still
|
||||
subject to the existing data-bearing guards). The all-system fail-safe remains ONLY for walk
|
||||
failure — it returns to being the error case, not the legacy-boot common case.* Implemented as
|
||||
`physicalDisksOf`/`walkSlaves` + `HostReader.BlockSlaves` (one seam method); per-branch
|
||||
conservatism (any unresolvable slave → ok=false → unchanged all-system path); signature test
|
||||
`TestSystemDisks_WalkTopologies` (root-backing disk ALWAYS system — never weaken). §3 spike
|
||||
transcripts: drill (legacy) dm-1→sda3→sda; felhom-pve (EFI+LVM) ESP+walk agree on sda →
|
||||
byte-identical regression. §13.2 wizard leg COMPLETE (offered → enrolled → formatted →
|
||||
torn down, boxes as found) + Day-0 manifest vouched to 0.87.0 (operator). The leg also
|
||||
surfaced two CONTROLLER bugs (fixed same-day: v0.126.3 claimed-box wizard CSRF, v0.126.4
|
||||
502-through-CF + native-alert ban).
|
||||
|
||||
- **v0.83.0** (2026-07-11, LIVE on felhom-pve; NOT published — Peti stays 0.81.0) — **observability
|
||||
pass** (pairs with controller v0.116.1 + hub v0.46.0). `applog.New` → `(logger, *Ring)`: slog
|
||||
fan-out, journald at the configured level, ~1000-entry ring FIXED at DEBUG. `GET /debug/logs`
|
||||
(local API, token-authed; the controller Debug page's Ügynök tab) + request-level DEBUG
|
||||
middleware. Heartbeat log-pull: envelope `log_tail_requested` → next heartbeat ships `log_tail`
|
||||
(128 KB, consume-once; failed push re-armed by the next envelope; `operator log pull served` INFO
|
||||
on fulfillment). Gap-fill sweep: netverify phase/verdict lines (job start, trigger outcome,
|
||||
/proc/mounts verdict, journal bytes, classification code, rollback outcome, durations), netmount
|
||||
unit steps, signedjobs op-received (class/host/expiry — never signatures) + fetch duration,
|
||||
selfupdate invariants + download sha/duration, disks outcome INFOs, controller-swap pre-pull +
|
||||
health verdicts, desired/loop per-exchange DEBUG. Logging conventions:
|
||||
`felhom.eu/documentation/runbooks/logging-conventions.md`. OPEN: the hub-side live pull awaits
|
||||
the operator's button click (hub UI password-gated); pre-existing lanresolver
|
||||
permission-denied WARN on /var/lib/felhom-agent/guests noted in REPORT.
|
||||
|
||||
- **v0.77.0** (2026-07-09) — **fork-4: escrow the offsite restic repo password under R.** `IdentityBundle`
|
||||
gains `ResticRepoPassword` (rides the existing age-under-R `WrapIdentityBundle` path — validated by the
|
||||
custody spike `febdc56`). New `POST /escrow/stage-secret` (`withGuest`) transiently stages the
|
||||
controller-pushed password (0600, never logged), which the `--selftest=escrow-create` ceremony
|
||||
auto-injects into the bundle and then wipes. `AttachResticPassword`/`StagedResticPasswordPath`/
|
||||
`WipeStagedResticPassword` added. Pairs with controller v0.105.0 (push + atomicity gate + DR inject +
|
||||
`DRResticCoord`). **NOT yet live-validated** — the supervised escrow ceremony is operator-run.
|
||||
|
||||
- **v0.76.0** (2026-07-08, LIVE on felhom-pve + **PUBLISHED sha `9828c5f7…f50b`** — THE Day-0
|
||||
manifest bump target; **0.75.0 superseded unpublished**) — **GL-5b / G12: restore-test
|
||||
full-fidelity**. Params derive from the ARCHIVE's embedded config (`drRestoreOverrides`, same as
|
||||
DR — the old live-source-config path verified the wrong object AND dropped storage mpN per PVE's
|
||||
all-or-nothing rule; deleted with `bindMountOverrides`/`archiveVMID`). NEW mount-parity assert
|
||||
(restored mpN vs archive; miss/mispath/undersize/extra = FAIL naming the delta) + `MountParity`/
|
||||
`MountInventory` on the wire record (additive). Live-proven: scratch 990000 ← 6.5GB 9201 archive,
|
||||
parity ok, inventory mp0 200G+mp1 50G+2 throwaways, **3m4s local tier** (cheaper than feared);
|
||||
rotated-out archive volid → clean up-front refusal (nice failure mode). bringup.go untouched.
|
||||
|
||||
- **v0.75.0** (2026-07-08, LIVE on felhom-pve) — **GL-5 / go-live G8: guest-loss DR bring-up
|
||||
actually restores** (closes the v0.74.0 OPEN item + SPIKE-dr-bindmount-source §8). DR passes the
|
||||
COMPLETE explicit restore param set derived from the archive's embedded config (NEW
|
||||
`Client.ExtractArchiveConfig`, 200 under the scoped token) — **two live-discovered PVE rules: mpN
|
||||
params need an explicit rootfs, AND unlisted mountpoints are silently DROPPED** (first run booted
|
||||
without mp0/mp1!) — storage mpN passed through, structural mp8/mp9 → throwaways, then step 4d
|
||||
swaps the REAL binds in via the host runner (root pct; new `EngineOptions.HostRunner`+`StateDir`
|
||||
seam) and deletes the unusedN residue. Scratch-DR live-proven end-to-end (9310 from a real 9201
|
||||
archive: mp0 200G + mp1 50G + real binds + no residue + clean teardown). Provision = nil
|
||||
overrides (regression-tested). NOTE: published/vouch-pending agent is 0.74.0 — publish 0.75.0
|
||||
before/with the manifest bump. OBSERVATION: the DR selftest hardcodes KeepMAC=true — a scratch DR
|
||||
while the SOURCE guest is live briefly duplicates its MAC on the bridge (pre-existing; fine for
|
||||
supervised runs, worth a -keep-mac flag someday). Full customer-data DR drill = GL-6/S5 family.
|
||||
|
||||
- **2026-07-07 — v0.74.0 Gitea-PUBLISHED (RUNBOOK GL-1)** — the LIVE felhom-pve binary's exact
|
||||
bytes, sha256 `1ec3f58842edce1e…76af05`, anon-fetch-verified. This supersedes/closes every
|
||||
standing "publish 0.6x + Day-0 vouch" OPEN item below (0.64→0.73 were never published; 0.74.0
|
||||
is the vouch target). Golden 0.103.0 published in the same run (felhom.eu execution record
|
||||
`documentation/pilot/RUNBOOK-GL1-publish-2026-07-07.md`). **Day-0 manifest vouch = operator
|
||||
step** (agent 0.74.0 / golden 0.103.0).
|
||||
|
||||
- **v0.74.0** (2026-07-07) — **campaign-2 R2 CLOSED; the mislabelled "R1" was a symptom** (LIVE on
|
||||
felhom-pve). Pool membership is what lets the pool-scoped token reach a guest; `pct restore --pool`
|
||||
sets it only at CREATE, so a restore-over-existing dropped 9201 from the `felhom` pool → no
|
||||
`VM.Audit` → restore-test's *existing* `bindMountOverrides` never ran → "mp8 … only possible for
|
||||
root". Fix: `Client.PoolAddVMID` + bring-up re-asserts membership post-restore (warn-not-fail).
|
||||
Role/ACL + `bindMountOverrides` untouched (both correct). **Live restore-test PASSED for the first
|
||||
time** once the pool was healed (Part A one-liner): read config → neutralize 2 binds → restore →
|
||||
boot+running → clean teardown, 4m35s. B3 (scratch-teardown 403) confirmed a cascade — no code.
|
||||
OPEN: DR `bring-up -mode dr` bind-override gap (spike `SPIKE-dr-bindmount-source-2026-07-07.md`:
|
||||
small known-constant override reusing `bindMountOverrides`; mp8/mp9 are structural constants).
|
||||
|
||||
- **v0.73.0** (2026-07-06) — **F2 mount-role fallback CLOSED** (LIVE on felhom-pve). `roleForMountPath`
|
||||
gained a mount-table fallback (Impl-2b style): a bind-mounted RAW enrolled user-data drive is not a PVE
|
||||
storage, so it fail-safe'd to `system` and the eject/decommission gates 403'd EVERY user-data drive
|
||||
(campaign F2, `where=/mnt/teszt_enroll role=system`). Device-keyed classification + whole-disk containment
|
||||
(`storage.SameWholeDisk`); Observe-error keeps the fail-safe BEFORE the fallback. Only `roleForMountPath`
|
||||
touched. Live-proven full lifecycle on teszt_enroll (eject/decommission 200, no-rebind across restart,
|
||||
end==pre). OPEN follow-up: the `deviceRole`/`roleForMountPath` unification refactor (deferred).
|
||||
|
||||
- **v0.72.0** (2026-07-05) — **OOB operator access (merged E1+H1)** — TASK H1, provenance both
|
||||
`SPIKE-{felhom-sshd,oob-wg-operator-peer}-2026-07-05`. Operator `/32` RENDERED into wg-felhom
|
||||
AllowedIPs (survives self-heal, [OF-1]); dedicated `internal/felhomsshd` (port claim + config
|
||||
render→sshd -t→reload + operator authorized_keys + heal + oob heartbeat stanza); static
|
||||
`inet felhom_oob` belt (agent mutates SET ELEMENTS ONLY); `configs/felhom-sshd.service` (NO
|
||||
RuntimeDirectory [SF-1]) + `felhom-oob.nft` + `felhom-op.sudoers`; `FELHOM_SSHD`+`FELHOM_OOB`
|
||||
grants; `oob.enabled` DEFAULT FALSE. Live on felhom-pve (8822, belt filled, operator SSH as
|
||||
felhom-op with scoped sudo); hub v0.35.0. Rollback `.bak-0.71.0`. 5 live-found bugs fixed (port
|
||||
path, self-listen flip-flop, nil-block lockout, reachable-via-dial, operator-configured source).
|
||||
- **v0.71.0** (2026-07-05) — **management-plane break-glass: privsep-dir watchdog + mgmt_plane
|
||||
health** — TASK G1 (prereq for felhom-sshd/H1), provenance `SPIKE-felhom-sshd-2026-07-05` §8.
|
||||
Host artifacts (`configs/felhom-privsep.tmpfiles` + `felhom-mgmt-watchdog.{sh,service,timer}`) make
|
||||
`/run/sshd` boot-persistent AND auto-heal it every ~60s **agent-independently** (heals with the
|
||||
agent stopped — proven live: `/run/sshd` removed → restored in 30.0s, `:22` back, no login).
|
||||
`internal/mgmtplane` reports the additive `mgmt_plane` heartbeat stanza; hub v0.34.1 raises
|
||||
`mgmt_plane_healed`. **NO unit declares `RuntimeDirectory=`** (the incident cause). H1 may now
|
||||
assume `/run/sshd` is guaranteed present. Live on felhom-pve; rollback `.bak-0.70.0`.
|
||||
- **v0.70.0** (2026-07-05) — **agent self-update (operator-signed A/B slots + crash-loop
|
||||
auto-rollback)** — TASK D1, provenance `SPIKE-agent-selfupdate-2026-07-05`. An operator-signed
|
||||
`agent_update` op (version+sha256, sha is the only integrity root) rides the signed-jobs gate;
|
||||
@@ -13,11 +375,16 @@
|
||||
60s dwell; a crash-looping binary is auto-reverted by `OnFailure=felhom-agent-rollback.service`
|
||||
(first-crash trigger [SF-1]) with the tuned `[Unit]` start-limit (120s/4) as backstop. Host
|
||||
artifacts + sudoers `FELHOM_SELFUPDATE` + `felhom-host-install.sh` day-0 install + report field
|
||||
`selfupdate_pending`. Green tests + companions; **live validation pending (build/publish v0.70.0,
|
||||
manual install the artifacts on felhom-pve, then the happy-path + crash-rollback drills)**. Rollback
|
||||
`selfupdate_pending`. Green tests + companions. **LIVE-VALIDATED on felhom-pve (2026-07-05): all 4
|
||||
drills PASS** — happy path (0.70.0→0.70.1 signed op → download+verify+flip+commit), crash-rollback
|
||||
(0.70.2-crash → OnFailure → **~2s crash-to-recovered**, byte-identical revert, no loop), no-pending
|
||||
guard, gate refusal (non-pinned key). Full agent-side pipeline ran real (envelope injected into the
|
||||
hub `signed_jobs` queue — CC lacks the hub global operator key; hub enqueue-auth is hub-unit-tested).
|
||||
Box restored to canonical **v0.70.0** (host artifacts KEPT installed; scratch operator key REMOVED —
|
||||
self-update dormant until an operator pins a real key, a Day-0-vouch-style follow-up). Rollback
|
||||
`felhom-agent.bak-0.69.0`. OPEN (v1 scope-outs): no hub-floor auto-update, no failed-update
|
||||
auto-retry, no pending-timeout auto-rollback (a runs-but-never-commits binary is caught by
|
||||
`host_staleness` + the pending report flag). Detail: REPORT.md.
|
||||
auto-retry, no pending-timeout auto-rollback; per-crash OnFailure can double-fire (idempotent —
|
||||
future: serialize the rollback oneshot). Detail: REPORT.md.
|
||||
|
||||
- **v0.69.0** (2026-07-04, live on felhom-pve) — **S5: host-loss DR — safe halves shipped**.
|
||||
**Part 1** `wgtunnel.InstallRecoveredKey` — writes an escrow-recovered WG privkey (create-only,
|
||||
|
||||
@@ -1,73 +1,41 @@
|
||||
# REPORT — S5: host-loss DR (recovered WG-key install + directive→PLAN; safe halves shipped)
|
||||
# REPORT — felhom-agent v0.127.0: a mount Felhom made is not foreign (R-220)
|
||||
|
||||
**Date:** 2026-07-04 · **Class:** implementation (safe halves) + diagnosis + a prepared,
|
||||
operator-present destructive drill (NOT executed by CC). **Design of record:** doc-06 §3.5 + S5 row;
|
||||
doc-01 §9.
|
||||
**Scope: the host half of R-220.** The customer-facing refusal message is the controller's half and
|
||||
ships as felhom-controller v0.203.0.
|
||||
|
||||
## 1. Baselines & commits (felhom-agent → `main`)
|
||||
## What changed
|
||||
|
||||
Start: `567cf9f` (v0.68.0). **`bd4bced`** v0.69.0 — the two safe DR halves (Parts 1-2). This commit
|
||||
— REPORT + CONTEXT + doc-06 S5 row.
|
||||
| File | Change |
|
||||
|---|---|
|
||||
| `internal/storage/claim.go` | `claimFacts.felhomOwnedMounts`; `classifyClaim` forgives a non-managed mountpoint **only when corroborated**; `felhomOwnedMounts()` + `procMounts()` |
|
||||
| `internal/storage/hostops.go` | `mountTable` seam (nil ⇒ real `/proc/mounts`) |
|
||||
| `internal/storage/claim_r220_test.go` | new — the own-drive case, the fence, and the corroboration's four edges |
|
||||
|
||||
## 2. Part 1 (SAFE, shipped) — install the recovered WG key
|
||||
## The shape chosen, and why (§7.3)
|
||||
|
||||
`wgtunnel.InstallRecoveredKey(stateDir, privB64)` writes an escrow-recovered WG private key (32-byte
|
||||
base64, re-encoded canonical) to the key file so the tunnel re-establishes with the **same
|
||||
identity/pubkey → same hub /32**, no fresh keygen. **CREATE-ONLY** — refuses if a key file exists (a
|
||||
present key may be a live identity); the value is never logged. Wired into
|
||||
`--selftest=identity-consume -install-wg-key` (opt-in; installs `bundle.WGPrivateKey` after
|
||||
`UnwrapIdentityBundle`; a pre-S3 blob with no WG key → logged fallback to fresh keygen + re-register,
|
||||
which keeps the /32 via hub re-key-in-place).
|
||||
**Test + red-proof:** install→`EnsureKey` LOADS it (same pubkey, no keygen); a second install REFUSES
|
||||
(create-only — red-proofed against dropping the exists-check); invalid key → error, nothing written.
|
||||
**Candidate (b): the claimed check distinguishes a mount Felhom made from a foreign one** — the task
|
||||
called it "nearer the truth" and it is, because the host and its knowledge survive the rebuild while
|
||||
the guest's registry does not. Candidate (a) — having the rebuild path clear the raw mounts — would
|
||||
have made correctness depend on a cleanup step running, and a cleanup that does not run leaves exactly
|
||||
today's defect.
|
||||
|
||||
## 3. Part 2 (SAFE, shipped) — consume `restore_directive` → PLAN
|
||||
**The discriminator is corroboration, not a path prefix**: the same device must ALSO be mounted under
|
||||
`/mnt/felhom-drives`. Only enrolment produces that pairing.
|
||||
|
||||
New `internal/dr`: the host_loss `restore_directive` (was logged-and-ignored in `syncer.go`) is
|
||||
consumed via the `desired.Syncer.AddConsumer` raw seam into an inspectable **RestorePlan** — per
|
||||
guest `{vmid, archive, target storage, sizing}`, per drive `{durable_id → expected mount}`, + the
|
||||
offsite PBS coord. **Derive-and-surface only:** the `Consumer` holds NO restore/destroy dependency, so
|
||||
"execute nothing" is structural. `guest_loss`/absent → no plan; recipe fetched on-demand (rare
|
||||
directive) via a fresh `Collect`.
|
||||
**Test + red-proof:** host_loss builds the expected plan; guest_loss/absent/nil-recipe → none
|
||||
(red-proofed against a relaxed mode gate); the consumer surfaces on host_loss + clears otherwise +
|
||||
consults the recipe only then. Full `go test ./...` green; deployed 0.69.0; `-install-wg-key` flag
|
||||
live; daemon starts clean (dr consumer wired).
|
||||
**`/proc/mounts` rather than `lsblk MOUNTPOINTS`**, because the lsblk invocation is pinned verbatim in
|
||||
the sudoers file; changing it would have coupled this fix to a config rollout. `/proc/mounts` is
|
||||
world-readable and needs neither.
|
||||
|
||||
## 4. Part 3 — hub escrow export: NOT needed
|
||||
## Green gate
|
||||
|
||||
The operator has a clean blob-export path already: `sqlite3 … "SELECT writefile('/tmp/blob', blob),
|
||||
writefile('/tmp/idblob', identity_blob) FROM host_escrow WHERE host_id='demo-felhom-01'"` on a
|
||||
`kubectl cp`'d `hub.db`. No hub change (respects the read-only-unless-no-clean-way gate).
|
||||
`go build` · `go vet` clean · `go test ./...` → **29 packages ok** · `agent_gates.py --fast` → all OK.
|
||||
|
||||
## 5. Part 4-A — re-attach wrong-disk safety: ALREADY PROVEN (no new probe built)
|
||||
| Red-proof | Result |
|
||||
|---|---|
|
||||
| remove the `felhomOwnedMounts` exemption | **FAILS** — "device is mounted at /mnt/adatok (sdb)", the pre-fix refusal |
|
||||
| over-widen the exemption to any `/mnt/*` | **FAILS** — "/mnt/someone-elses-disk was offered for formatting" |
|
||||
|
||||
The re-attach matcher is `storage.ResolveStorageDevice` (the `uuid:<fs-uuid>` scheme DR drives use).
|
||||
Its wrong-disk guard is unit-established: `TestResolveStorageDevice_ToleratesDeviceLetterMove` (a
|
||||
MATCHING UUID resolves to the current /dev path) + `TestResolveStorageDevice_AbsentAndScheme` (an
|
||||
absent/mismatched UUID **ERRORS** — "skips a gone drive instead of fail-mounting"; a bare node /
|
||||
non-`uuid:` scheme is refused). It resolves ONLY the exact UUID's symlink — never a "nearest" disk.
|
||||
Combined with S4.1's restore-to-scratch proof, the composition's safety is established; a new live
|
||||
scratch+marker-drive probe would add confidence but not new safety, so it was not built (recorded).
|
||||
## Not changed
|
||||
|
||||
## 6. Part 4-B — the destructive in-place 9201 drill: PREPARED + OPERATOR-GATED (not executed)
|
||||
|
||||
Pre-flight **GREEN** (non-destructive): offsite `ct/9201` snapshots present (the restore source);
|
||||
9201 running; the escrow blob is on the hub (383 B + 499 B identity, offsite key `b0:fe:2a…`),
|
||||
exportable via `sqlite3 writefile`; **S4.1 already proved 9201 restores from offsite (pass:true)**.
|
||||
The destructive drill (destroy 9201 → restore in place from offsite → re-attach the data drive by
|
||||
durable_id) is **operator-present + STOP-gated**: the operator runs the R-consume steps
|
||||
(`escrow-consume` + `identity-consume -install-wg-key`, R via env — **CC never runs these**, §9-4a)
|
||||
and confirms the destroy at the STOP checkpoint. Prepared procedure + the go/no-go is with the
|
||||
operator (destroying the live demo controller is their decision, even with the backup confirmed
|
||||
recoverable).
|
||||
|
||||
## 7. NOT yet live-validated / observations
|
||||
|
||||
- The live consume→install→tunnel→restore pipeline (the Part 4-B drill) — operator-present, pending
|
||||
the operator's go-ahead + R-run. Parts 1-2 are unit-proven + deployed; their live consume path is
|
||||
exercised only in that drill (R-gated).
|
||||
- Deferred (per slice scope): guest_loss DR; hub-driven full-auto DR (the hub issuing recovery-mode +
|
||||
directive itself); multi-guest host-loss. true-CGNAT-SIM retest still parked.
|
||||
- Observation: the DR plan is surfaced via structured log + `Consumer.LastPlan()`; a report field
|
||||
(so the hub/operator sees the plan without the box's journal) is a small future refinement.
|
||||
No sudoers, no allowlisted command, no PVE surface, no format path. Every other claim signal
|
||||
(system disk, read-only, LVM PV, ZFS member, member FSTYPEs, empty-topology backstop) is untouched.
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
| `Runner` / `ExecRunner.Run`, `RunStdin` | internal/proxmox/privileged.go | `Run(ctx, name, args...) (stdout, stderr []byte, err)` | ALL host command exec (direct or `sudo -n` prefix) | Arg vectors only, never a shell string; `capBuf` caps output at 1 MiB |
|
||||
| `Privileged` (CreateGoldenLXC/MountUSBByUUID/SMART/Sensors) | internal/proxmox/privileged.go | methods on `*Privileged` | the 3 fenced root-CLI exceptions ONLY | Do NOT add methods — fence is structural (`routing_test.go` asserts it) |
|
||||
| `SudoHostOps.run` | internal/storage/hostops.go | `run(ctx, name, args...) error` | allowlisted exec with stderr-wrapped error | Every arg pre-validated via validate.go before this is called |
|
||||
| `Prober.Probe` | internal/capability/probe.go | `Probe(ctx) []Status` | live sudo-policy capability check (`sudo -n -l --`) | Needs a DIRECT runner (never the sudo-prefixing one — double-sudo); never executes probed cmds |
|
||||
| `Prober.Probe` | internal/capability/probe.go | `Probe(ctx) []Status` | live sudo-policy capability check (`sudo -n -l --`) | Needs a DIRECT runner (never the sudo-prefixing one — double-sudo); never executes probed cmds. v0.86.0: config-gated caps (`Capability.GatedBy` + `Prober.GateActive`) report `inactive`/"disabled by configuration" ONLY when healthy — broken plumbing stays degraded; the pbsdr-* gate answers from `pbsdr.Manager.DRConfigured` (marker-backed across restarts) |
|
||||
| `stageTemp` | internal/localapi/intermediary.go | `stageTemp(pattern, content) (path, err)` | random-named temp before a root `install` (audit B1) | Fixed /tmp names are a TOCTOU — sudoers globs expect `/tmp/felhom-*-*.ext` |
|
||||
| `guesthook.InstallSnippet` / `Register` | internal/guesthook/install.go | `InstallSnippet(ctx, runner) error` | pre-start self-heal hook install (C1 net) | Same random-temp+install pattern; snippet delegates to the agent binary (no shell logic). Issues `mkdir -p /var/lib/vz/snippets` FIRST (v0.63.0, B2 — fresh boxes lack the dir; sudoers grants exactly that argv) |
|
||||
|
||||
@@ -48,7 +48,10 @@
|
||||
| `GuestBinder.EnsureSharedParent` | internal/localapi/intermediary.go | `EnsureSharedParent(ctx) error` | shared-parent bind + boot unit | make-private+make-shared ONLY on first bind — re-running orphans the guest's slave; F2-a: compares script AND unit for staleness |
|
||||
| `StablePathForRaw` / `DriveNameFromRaw` | internal/localapi/intermediary.go | `StablePathForRaw("/mnt/<n>") string` | raw host mount → stable guest path | Single source of truth both repos derive from |
|
||||
| `GuestBinder.GuestSeesMount` / `GuestBootID` | internal/localapi/intermediary.go | `GuestSeesMount(ctx, vmid, path) bool` | guest-visible (usable) signal; reboot detection | Host bind present ≠ guest sees it (non-recursive parent bind) |
|
||||
| `SudoHostOps.EnsureNetworkMount` / `RemoveNetworkMount` / `ListNetworkMounts` | internal/storage/netmount.go | `EnsureNetworkMount(ctx, spec) error` | NAS automount pair | rm glob confined to `mnt-felhom*` units; NAS ≠ drive (no durable-id/SMART/wipe) |
|
||||
| `SudoHostOps.EnsureNetworkMount` / `RemoveNetworkMount` / `ListNetworkMounts` | internal/storage/netmount.go | `EnsureNetworkMount(ctx, spec) error` | NAS automount pair | rm glob confined to `mnt-felhom*` units; NAS ≠ drive (no durable-id/SMART/wipe); RemoveNetworkMount doubles as the verify-fail rollback (idempotent) |
|
||||
| `NetworkMountedAt` / `NetworkEndpointReachable` | internal/storage/netmount.go | `NetworkMountedAt(where) bool` | verify mount-truth + the 2 s add pre-probe | /proc/mounts is the ONLY mount-success judge (autofs trigger ≠ mounted; readability ≠ mounted — SPIKE-nas-verify §8) |
|
||||
| `SudoHostOps.ReassertNetworkAutomounts` + `Server.ReassertNetworkMounts` + `guesthook.PostStartNetworkReassert` | internal/storage/netreassert.go, internal/localapi/netreassert.go, internal/guesthook/netreassert.go | `ReassertNetworkAutomounts(ctx) []NetReassertResult` | NAS guest-reboot heal (RCA fix 1): re-arm idle automount triggers (stop + enable --now) so the fresh mount event propagates into running guests | NEVER call from periodic health paths (an idle trigger is HEALTHY); active real mounts are never touched; hook leg runs as root (direct systemctl), daemon leg via sudo |
|
||||
| `ClassifyNetVerifyFailure` | internal/storage/netverify.go | `ClassifyNetVerifyFailure(journalTail, tcpReachable) (code, hint)` | NAS verify failure categories | String-based BY DESIGN (every mount failure is rc=32); substrings verbatim from SPIKE-nas-verify Q4; `nfs_export` merges not-found/not-permitted (NFSv4 identical) |
|
||||
|
||||
### Durable stores (atomic state)
|
||||
|
||||
@@ -71,6 +74,7 @@
|
||||
| `EnsureLeaf` | internal/localapi/cert.go | `EnsureLeaf(certPath, keyPath, host) (cert, fingerprint, generated, err)` | pinned self-signed leaf | `generated=true` invalidates every issued bootstrap pin — log LOUD (B.1) |
|
||||
| `Server.RecoverStaleLockedGuests` | internal/localapi/stalelock.go | `RecoverStaleLockedGuests(ctx)` | startup stale vzdump-lock heal (F2-b) | Clears ONLY `backup`/`snapshot-delete`, only when no vzdump in-flight; A1 RESOLVED (v0.62.0): scan is pool-intersected (`ListLXC` ∩ `Client.Pool`), fail-safe skip on pool-read failure |
|
||||
| `ControllerSwapper.Swap` + `ValidControllerImage` | internal/localapi/controllerswap.go | `Swap(ctx, vmid, target) *ControllerSwapState` | agent-owned controller image swap + rollback | Strict image regex (repo + 3-part semver); state file written BEFORE swap; no-healthcheck images need `verifyDwell` |
|
||||
| `MemoryOps` + `Server.readMemoryBounds` | internal/localapi/guestmemory.go | `readMemoryBounds(ctx, vmid) (memoryBounds, err)` | guest RAM resize (v0.90.0, R-24): GET/POST /guest/memory | NEW narrow seam (never extend `GuestAPI` — it breaks every fake); the AGENT is the boundary — bounds recomputed FRESH per request (min 2048 / max host_total−2048 / shrink floor max(2048, usage+512)); §8 UNITS TRAP (config `memory`=MB, status/node=bytes); verify maxmem==target after `SetConfig` before claiming success; SetConfig NEVER called on a refusal path |
|
||||
|
||||
### Proxmox client / hub / PBS / provisioning
|
||||
|
||||
@@ -79,14 +83,23 @@
|
||||
| `Client.WaitTask` | internal/proxmox/task.go | `WaitTask(ctx, upid, opts) (TaskStatus, error)` | asserting EVERY mutating op | POST 200 ≠ success; authz can fail at task exec; `AllowWarnings` opt-in |
|
||||
| `Client.Pool` | internal/proxmox/query.go | `Pool(ctx, name) (PoolInfo, error)` | felhom-pool membership (the ownership registry, A1) | Needs `Pool.Audit` at `/pool/<name>` (host-install v1.9.0+); `Pool.Allocate` does NOT satisfy the read; members can be storages (type `storage`, vmid 0) — filter them |
|
||||
| `Client` mutate wrappers (`RestoreLXC/Vzdump/DestroyLXC/Snapshot/Rollback/SetConfig/ResizeLXC/Start/Stop`) | internal/proxmox/mutate.go | return `(upid, error)` | all API mutations | Async → always pair with WaitTask; route via gate/queue, not ad-hoc |
|
||||
| `Client.PoolAddVMID` | internal/proxmox/mutate.go | `PoolAddVMID(ctx, pool, vmid) error` | re-assert pool membership after a restore-over-existing (campaign-2 R2) | SYNC (no UPID, don't WaitTask); PVE `PUT /pools` is additive (merge, not replace) — `delete=1` removes; idempotent (already-member swallowed); needs `Pool.Allocate` at `/pool/<pool>`. `pct restore --pool` sets membership only at CREATE — a restore over an existing vmid drops it, so bring-up re-asserts post-restore |
|
||||
| `TLSConfig.build` / `normalizeFingerprint` | internal/proxmox/tls.go | `build() (*tls.Config, error)` | PVE leaf-cert SHA-256 pinning | No insecure default |
|
||||
| `pinnedTLS` | internal/pbs/pin.go | `pinnedTLS(fingerprint) (*tls.Config, error)` | PBS leaf pinning | Same model as PVE; 64-hex fingerprint normalized |
|
||||
| `hub.Client.Report` | internal/hub/client.go | `Report(ctx, *HostReport) (*ControlEnvelope, error)` | the heartbeat | Typed `TransportError`/`HTTPError`, never contain the bearer token |
|
||||
| `hub.Loop` + `MultiObserver` | internal/hub/loop.go | `NewLoop(...)`; `MultiObserver(obs...)` | resilient report loop + envelope fan-out | Errors logged, loop continues; interval clamped 60–3600 s |
|
||||
| `provision.BackHalf.Provision` | internal/provision/backhalf.go | `Provision(ctx, Input) (Result, error)` | guest bootstrap back-half | mint→render→0600 write→chown 100000:100000→`pct set` ro bind→onboot; token NEVER logged/returned |
|
||||
| `provision.BackHalf.Provision` | internal/provision/backhalf.go | `Provision(ctx, Input) (Result, error)` | guest bootstrap back-half | mint→render→0600 write→chown 100000:100000→`pct set` ro bind→onboot; token NEVER logged/returned. Bootstrap `local_api.endpoint` = the caller's `cfg.LocalAPI.ListenAddr` (main.go) — moving the agent bind to the island moves the guest dial for free (R-50, no template) |
|
||||
| `buildBringUpConfig` island NIC | internal/reconcile/bringup.go | (pure) `BringUpSpec{IslandBridge,IslandGuestAddr}` → `params["net1"]` | R-50 island control plane | When BOTH island fields are set (from `cfg.LocalAPI`), attaches a static `net1=name=eth1,bridge=<vmbr9>,ip=<.2/30>` (no hwaddr → fresh MAC), so the controller reaches the agent over a fixed private address immune to LAN/DHCP/site moves. Empty = pre-R-50, no net1. All-or-nothing + CIDR enforced in `LocalAPIConfig.Validate`. The guestnet healer is eth0-only (`parseMode` is dev-scoped) so it never touches the static island NIC |
|
||||
| `reconcile.Queue.Submit` | internal/reconcile/queue.go | `Submit(vmid, fn) <-chan error` | per-guest serialization of ALL mutations | Same vmid strictly FIFO; lanes parallel across guests |
|
||||
| `Engine.RunSignedJob` | internal/reconcile/job.go | `RunSignedJob(ctx, intent, signed, exec) JobResult` | executing a gated destructive job | Idempotency by nonce; journaled |
|
||||
| `escrow.Create` | internal/escrow/escrow.go | `Create(ctx, CreateOptions) (CreateResult, R, error)` | PBS-key escrow (zero-knowledge) | Recovery code returned SEPARATELY from the result (anti-log); self-verifies recoverability |
|
||||
| `escrow.GenerateRecoveryCode` / `joinSafe` / `RecoveryCodeSep` | internal/escrow/wordlist.go | `GenerateRecoveryCode() (string, error)` | minting the customer recovery code R | Draws from the EFF large list **filtered of every word containing `RecoveryCodeSep`** (4 entries: drop-down, felt-tip, t-shirt, yo-yo) so a code always segments back into exactly 10 words — a hyphenated word made codes ambiguous to transcribe AND flaked the test ~1/5 (v0.93.0). Generation-only: **already-issued codes stay valid**, R is verified as a whole passphrase and never re-split. Never count words by splitting the joined string — count what the generator drew |
|
||||
| `escrow.CeremonyBinary` / `CeremonyArgs()` / `CeremonyOutput` | internal/escrow/ceremony.go | the ONE fixed sudo self-invocation argv + the `--output=json` wire object (v1) | controller-driven ceremony (v0.88.0) | SINGLE SOURCE shared by the localapi exec, the capability manifest entry, and (byte-identically) the FELHOM_ESCROW sudoers line — `TestEscrowCeremonyArgvPinned` + `TestManifestCoveredBySudoers` lock all three. Never flag-helpers, never `--`→`-` (spike §2.2) |
|
||||
| localapi escrow ceremony job | internal/localapi/escrow_ceremony.go | `POST /escrow/ceremony` + status + ONE-SHOT claim + preflight | the wizard's agent half | R lives ONLY in `Server.escrowR` (NEVER the job struct — snapshots must be structurally R-free); zeroed on claim/supersede/10-min TTL (`unclaimed_void`); in-memory BY DESIGN (restart loses R safely; re-run supersedes); subprocess stdout is SECRET-BEARING → parsed then zeroed, never logged |
|
||||
| `poke.Listener` + `poke.Port` | internal/poke/poke.go | `NewListener(resolve, trigger, port, logger)`; `poke.Port = 51822` | agent-plane immediate-sync (Direction-2a, v0.89.0) | Binds a contentless UDP socket EXCLUSIVELY to the box's WG /32 (`wgtunnel.LoadAssignedAddr`), fires the hub-loop out-of-band trigger. **Port 51822 is a SHARED cross-repo contract** — the hub poke sender + the ep0 `felhom-poke` forced-command target the SAME number; change one → change all three. Contentless (payload ignored), leading-edge debounced (`DebounceWindow`), WG-confined (kernel EKEYREJECTED refuses non-peer /32s). Wired only when `wg_tunnel.enabled` |
|
||||
| `wgtunnel.LoadAssignedAddr` | internal/wgtunnel/manager.go | `LoadAssignedAddr(stateDir) (netip.Addr, bool)` | the box's own WG /32 without a Manager | Reads `registered.json`; ok=false until registered; strips the /32 → bare addr (the poke bind target) |
|
||||
| `fasttick.Loop` + `fasttick.SourceFunc` | internal/fasttick/fasttick.go | `New(out chan<- struct{}, interval, logger, sources...)`; `Source.Unconverged() (bool, reason)` | agent-plane immediacy SECONDARY (v0.90.0, R-28): pulse the SAME out-of-band trigger every 30 s while ANY source is unconverged, self-disarm on convergence | STATE-BASED (no timer, nothing to journal). Every source MUST be a CACHED read (no exec/network per tick) — `desiredProvider.Generation()`, `reconcile.Engine.LastResult()` (Planned−Pending>0), `pbsdrLoop.PBSDRStatus().State=="waiting_secret"` ONLY, `wgtunnel.Manager.TunnelConvergence()`. The LOUD pbsdr states + destructive `pending_signature` are DELIBERATELY EXCLUDED (a stuck-loud box must not hammer). Pulses the cap-1 channel non-blocking (coalesces with poke/watchdog); NEVER touch `MinPollSeconds`/`clampInterval`/the ticker |
|
||||
| `wgtunnel.Manager.TunnelConvergence` / `reconcile.Engine.LastResult` | internal/wgtunnel/manager.go, internal/reconcile/engine.go | `TunnelConvergence() (desired, operational bool)`; `LastResult() (Result, bool)` | cached convergence snapshots for the fast-tick | TunnelConvergence is refreshed at the END of every `Apply` (its own cadence) so the fast-tick never execs `wg`/`systemctl`; LastResult is mutex-recorded per `reconcileOnce`, ok=false until the first pass |
|
||||
|
||||
## 2. Canonical patterns (copy structure from THE named file)
|
||||
|
||||
@@ -97,8 +110,13 @@
|
||||
| Anti-retarget durable-id binding | internal/localapi/wipe_reresolve.go | resolve id → re-derive + exact match → re-inspect expected state → act on RE-RESOLVED device only |
|
||||
| Atomic single-file JSON store | internal/storage/intent.go | `Open*` loads (missing=empty, corrupt=fail-loud), mutex, tmp+rename 0600, idempotent set |
|
||||
| Durable append-only log + index | internal/authz/noncestore.go (`FileNonceStore`) | fsync before returning "new"; replay into index on open; expiry-only compaction |
|
||||
| Injectable seam funcs on Server | internal/localapi/server.go (`reresolveWipe`, `deviceDurableID`, `boundCheck`) | prod default wired in `NewServer`; tests override — no real /dev in tests |
|
||||
| Injectable seam funcs on Server | internal/localapi/server.go (`reresolveWipe`, `deviceDurableID`, `boundCheck`, `deviceCheck`, `livenessCheck`, net-verify: `netTrigger`/`netMounted`/`netJournal`/`netReachable`) | prod default wired in `NewServer`; tests override — no real /dev, /proc/mounts, journalctl or TCP in tests. **For mount-table predicates prefer the DATA seams `procSelfMountinfo` / `procGuestMountinfo` (internal/localapi/intermediary.go) over `boundCheck`/`livenessCheck`**: pointing them at a captured fixture runs the real parser, the real predicate and the real handler, so the test cannot go hollow the way R-116's did |
|
||||
| `Server.devicePresent` (R-113, v0.114.0) | internal/localapi/disks.go | `devicePresent(rawMountPath) bool`; seam `deviceCheck`, default `isHostMountpoint` | the agent's DEVICE-presence signal — asks whether the drive's RAW mount is still mounted | **Use this, never the bind, to answer "is the drive there".** The raw mount is a device-bound systemd unit and dies with its device; the agent's own bind under the shared parent is NOT device-bound and outlives it as a stale shell. `BoundUnderParent` is now `boundUnderParent(...) && devicePresent(...)` at BOTH /disks construction sites — dropping either half is a regression with its own red-proof. Empty path ⇒ **true** (unknown is never absent: absent stops a customer's apps) |
|
||||
| `bindLiveness` + `BindLiveness` (R-117, v0.117.0) | internal/localapi/intermediary.go | `bindLiveness(stable, raw) BindLiveness`; seam `livenessCheck`; read verdicts ONLY via `.Usable()` | the agent's bind-LIVENESS signal — the third term of `BoundUnderParent` | **`devicePresent` and `boundUnderParent` are both PATH-PRESENCE tests and neither is liveness.** They compare only mountinfo field 5, so both stay true over a bind that names the drive that went away while the raw mount healed onto the returning one (measured: raw 8:32 /dev/sdc, bind 8:16 /dev/sdb `shutdown`, EIO both ways, payload healthy). Two dead states, and a fix needs BOTH checks: devno mismatch (the detach/return case) AND the ext4 abort tokens `shutdown`/`emergency_ro` (the steady-state case, where the devnos AGREE because the device never left). **THREE states, never a bool** — `BindUnknown` must exist and `Usable()` treats it as PRESENT (absent stops a customer's apps). **Order matters:** compare devices first and read the abort flag off the RAW mount in the stale case — abort-first classifies the real return state as aborted and refuses the re-bind that repairs it. **NO BLOCK I/O, ever** (CLAUDE.md rule; a probe on a wedged device survives SIGKILL). 6 red-proofs |
|
||||
| `AttachDrive` repair ruling (R-117, v0.117.0) | internal/localapi/intermediary.go | the `switch bindLiveness(...)` inside the `n == 1 && GuestSeesMount` arm | decides whether the existing self-heal runs | `BindStaleDevice` ⇒ **re-bind** (the raw mount is a healthy new superblock; repairs live, no guest restart). `BindAborted` ⇒ **quiet no-op** — a re-bind lands on the SAME dead superblock and this runs every 20 s, so re-binding is an infinite silent retry that also masks the state; it must surface via `BoundUnderParent=false`. `BindLive`/`BindUnknown` ⇒ no-op, unchanged. **Do not return an error for the aborted case** — the reconcile loop would log a failure every 20 s |
|
||||
| Detached IN-MEMORY verify job (single slot, deliberately unpersisted) | internal/localapi/netverifyjob.go | claim slot sync (single-flight 409) → detached pipeline off baseCtx → auto-rollback on fail; restart ⇒ slot empty ⇒ the CALLER rolls back (Scenario F) — contrast formatjob (persisted+recovered) |
|
||||
| Optional dependency degradation | internal/localapi/server.go (`Options`) | nil dep ⇒ endpoint answers "not configured" (503), never a crash |
|
||||
| Version channel (v0.82.0) | internal/localapi/server.go (`Options.AgentVersion`; `Handler()` mux wrap) | sets `X-Felhom-Agent-Version` on EVERY response (all routes/statuses, incl. auth-fail/404) — the controller's capability-comparison source; empty version ⇒ header omitted |
|
||||
| Root-file install via random temp | internal/localapi/intermediary.go (`installSharedParentUnit`) | `stageTemp` (os.CreateTemp) → sudoers-globbed `install -m` → pinned destination |
|
||||
| Detached destructive job + restart recovery | internal/localapi/formatjob.go | persist `running` → run off baseCtx → record outcome; recovery re-resolves durable id, never a path |
|
||||
| Signed-op verify pipeline | internal/authz/verifier.go (`Verify`) | armor→namespace→key-material allowlist→crypto over RAW bytes→blob→target→window→nonce LAST |
|
||||
@@ -126,18 +144,30 @@
|
||||
|---|---|---|---|
|
||||
| `proxmox.Runner` | internal/proxmox/privileged.go | `*ExecRunner` (direct/sudo) | `mockRunner` internal/proxmox/mock_test.go; runner fakes in storage tests |
|
||||
| `storage.HostOps` | internal/storage/hostops.go | `*SudoHostOps` (prod), `NoopHostOps` (degraded) | fakes in internal/storage/observe_test.go, watchdog_test.go |
|
||||
| `storage.HostReader` | internal/storage/hostread.go | `*ProcHostReader` | `fakeHostReader` internal/localapi/disks_test.go; internal/storage/role_test.go |
|
||||
| `storage.HostReader` | internal/storage/hostread.go | `*ProcHostReader` | `fakeHostReader` internal/localapi/disks_test.go; internal/storage/role_test.go. v0.87.0: `BlockSlaves(name)` lists `/sys/block/<name>/slaves` (root-free) — backs the `SystemDisks` dm/md walk (`physicalDisksOf`/`walkSlaves`, role.go); per-branch conservatism: an unresolvable slave fails the WHOLE walk → all-system fail-safe. NEVER weaken the signature test `TestSystemDisks_WalkTopologies` (root-backing disk always in the system set). |
|
||||
| `localapi.DiskOps` / `StorageGate` / `GuestAttacher` / `GuestLister` | internal/localapi/disks.go | `*storage.SudoHostOps`; `storageGateAdapter` (cmd/felhom-agent/main.go); `*GuestBinder`; `*proxmox.Client` | `fakeDiskOps`/`fakeGate`/`fakeGuestAttacher`/`fakeGuestList` internal/localapi/disks_test.go |
|
||||
| `localapi.GuestAPI` / `BackupService` / `BackupStore` / `TokenAuthority` | internal/localapi/server.go | `*proxmox.Client`, `*backup.BackupRunner`, `*backup.Store`, `*TokenStore` | `fakeGuests`/`fakeBackups`/`fakeStore` internal/localapi/server_test.go |
|
||||
| `backup.InFlight` | internal/backup/inflight.go | `TryAcquire(what) (release, busy, ok)` / `Busy()` | THE host-wide "one heavy guest operation at a time" gate — shared by the local-API backup path and the restore-test scheduler (R-85) | A **LINK** guard, not a lock one: the scratch VMID never touches the live guest's vzdump lock, but an offsite restore PULLS multi-GB over the tunnel a backup PUSHES one. Callers **DEFER, never cancel** — a deferred restore-test costs coverage, a cancelled backup costs the backup. A nil gate is ungated (pre-R-85 callers). |
|
||||
| `capability` store-grant probe (`storeGrantStatuses` / `storeGrantVerdict` / `Client.Permissions`) | cmd/felhom-agent/main.go, internal/proxmox/query.go | *"may the agent READ this backup tier?"*, one `capability.Status` per configured tier | R-185. **Never infer permission from an empty content listing** — `{"data":[]}` is what a FORBIDDEN tier and a NEWBORN tier both return, and that ambiguity hid an unreadable host tier on both demo boxes. Ask `/access/permissions` **as the agent's own token** (root always says yes). **The ungranted answer is not empty and not a 403** — it carries the privileges inherited from the box-wide `/` grant, so test for **`Datastore.AllocateSpace`** specifically; path-presence or `Datastore.Audit` reports a blinded storage healthy. Probed set comes from `BackupTiers()`, never a fixed list. Critical except the `local` fallback. Composes AROUND the sudo prober (the `poolReadStatus` precedent); `Status`'s wire shape is untouched so the hub alert is free. Unreachable PVE ⇒ degraded, never ok. |
|
||||
| `backup.RestoreTestState` | internal/backup/restoretest_state.go | `RecordSuccess(target,archive,tier,verified,t)` / `ProvenArchive(target)` / `ProvenRestoreTests(ctx)` / `LastSuccess(target)` / `OldestFirst(targets)` | Per-tier restore-test PROOF state, persisted (atomic tmp+rename) — **which archive** was proven, and when (R-86) | **Credit ONLY on success** — a permanently failing tier must keep sorting first, or it looks freshly proven and stops being retried. Ties break on target id: without it, two tiers proven in the same second rotate by Go's randomised map order. **This one NEEDS persistence unlike R-84** — R-84 had ground truth to consult (the archive is still on the storage); a restore-test destroys its scratch and leaves no artifact. **R-86: the ARCHIVE is the state, the time is metadata** — a time alone cannot answer "have we proven THIS archive", which is the due-check's whole question. A pre-R-86 file (bare RFC3339 per target) keeps its time and yields NO proven archive, so each tier is due once after the upgrade; reading a legacy time as proof of the current archive would invent a guarantee. **R-189: it is also the REPORTABLE half of the restore-test signal.** The in-memory `backup.Store` holds only this process's latest run, and under per-archive due-ness the agent will not re-test a proven archive — so a proof lost to a restart is not repeated for a whole archive generation (observed live: a passing 14.5 GB offsite restore reached no host-report). `ProvenRestoreTests` renders the stored proofs as `hub.RestoreTest` entries and the collector merges them; a record missing the archive or the tier is NOT emitted, because an unproven tier reading as proven is worse than the defect. **Only successes are stored, deliberately:** a success suppresses future work, a failure causes it. |
|
||||
| `hub.ProvenRestoreTestReporter` + `Collector.SetProvenRestoreTests` | internal/hub/collect.go | the DURABLE restore-test source, merged with the in-memory one | R-189. Merge rule: **one entry per tier, newest by `TestedAt` wins** — a fresh failure beats a stored success (the failure is the news, and it lives nowhere else), a stored success beats a stale in-memory entry after a restart, and a tier never appears twice (the hub would read two tests). An unparseable timestamp counts as OLDER, so a malformed entry cannot displace a good one. **The wiring is pinned by an AST test** — the method this replaced (`RestoreTestState.Snapshot`) carried a doc comment naming a host-report gauge and had no caller for weeks. |
|
||||
| `backup.SpecBuilder` / `backup.TierPicker` / `(*BackupRunner).PickSettledRestoreCandidateOn` | internal/backup/schedule.go, runner.go | `func(ctx,archive) RestoreTestSpec`; `func(ctx,target,notAfter) (archive,landed,error)` | The per-run restore-test spec + per-tier **settled** candidate lookup (R-85, widened by R-86) | The spec is built **PER RUN**, never frozen at construction — the pre-R-85 immediately-invoked value made the offsite tier unschedulable AND went stale on any config change. `SourceTier` comes from **the archive**, never the configured target (the v0.100.0 rule). A tier with no archive returns `("", zero, nil)` — **`""` is NOT an error**, or every fresh box looks broken for its first week. **R-86: `notAfter` is the settle cutoff** (zero = no cutoff, which is what keeps `PickRestoreCandidateOn` a one-line call into it), and the picker now skips entries failing `archivePlausiblyComplete` — under per-archive due-ness an incomplete phantom would be picked forever, fail forever, never earn proof, and make the tier due at EVERY evaluation. |
|
||||
| `localapi.BackupTier` + `normalizeBackupTiers` / `config.BackupConfig.BackupTiers` | internal/localapi/backup_tiers.go, internal/config/config.go | `normalizeBackupTiers(tiers, legacy, cadence) []BackupTier`; `BackupTiers() ([]BackupTier, []string)` | THE R-82 multi-tier resolution — one runner per tier, primary first | **The untargeted local-API contract is FROZEN**: no `?target=` ⇒ primary tier ⇒ pre-R-82 response BYTES (Target is `omitempty` and stays empty). Never default a missing cadence — reject it and log the warning at ERROR. Never share one retention knob between tiers. Jobs are keyed by (vmid,target). |
|
||||
| `localapi.StaleLockController` | internal/localapi/stalelock.go | `*staleLockController` (Client + Runner + pool) | `fakeStaleLock` (Server-level) stalelock_test.go; `fakeStaleLockAPI` (controller-level, tests the A1 pool intersect) stalelock_pool_test.go |
|
||||
| `localapi.GuestExecutor` | internal/localapi/controllerswap.go | `*GuestBinder` (pct exec) | `fakeGuestExec` internal/localapi/controllerswap_test.go |
|
||||
| `guestnet.Runner` / `guestnet.GuestSource` (R-54, v0.92.0) | internal/guestnet/{probe,watchdog}.go | `*proxmox.ExecRunner`; the POOL-VERIFIED `localapi.StaleLockController.Guests` (ListLXC ∩ felhom pool, audit A1) | `scriptedRunner` + `fakeGuests` internal/guestnet/watchdog_test.go. **Never wire a bare `ListLXC` here** — under a broad token that would run dhclient inside a co-tenant's container. Every assertion is an exec COUNT, and the load-bearing ones are the negatives: a static guest, an unprobeable guest, a boot-race guest and an unproven guest list must record **zero** heal calls |
|
||||
| `guestnet.Watchdog.SetDampers` / `now` (clock seam) | internal/guestnet/watchdog.go | config `guest_net.*`; `now` defaults to `time.Now` | tests advance a manual clock (the storage-watchdog pattern) and assert the heal ceilings EXACTLY — ≥10 min apart, ≤3/hour, and ≤30 over a scripted 10 hours of permanent failure. A damper with no test is a comment |
|
||||
| `hub.GuestNetReporter` (R-54) | internal/hub/collect.go | `*guestnet.Watchdog` (`GuestNetStatus`) | internal/hub/collect_guestnet_test.go asserts the stanza through the PRODUCTION `Collect` path AND that the `guest_net` key is ABSENT from the wire when no reporter is wired — an always-present empty stanza would make "not wired" and "found nothing" the same signal, which is the shape v0.91.0 hid behind |
|
||||
| `hub.AddressEnumerator` (v0.119.0) | internal/hub/hostaddr.go | **defaults to the REAL `systemInterfaces`** when `Collector.addrEnum` is nil — deliberately inverting the nil-reporter-means-off convention, because this stanza has no config gate and a forgotten wiring call would otherwise ship silently empty (the inert-seam shape, four instances on record) | internal/hub/hostaddr_test.go drives fixtures TRANSCRIBED from `ip -o addr show` on demo-felhom AND demo-hp, including the address-less veth/NIC rows — the "no denylist needed" claim rests on those rows really being empty, so omitting them would prove the claim by assuming it. `filterHostAddresses` keeps GLOBAL UNICAST only: one predicate that drops loopback, `fe80::/10`, and `169.254/16` — the last being the R-50 island literal, identical on every box and actively misleading if surfaced |
|
||||
| `reconcile.OpVerifier` | internal/reconcile/gate.go | `*authz.Verifier` | fake verifier in internal/reconcile gate tests |
|
||||
| `signedjobs.WipeOps` / `Executor` (`ExecutorChain`) | internal/signedjobs/wipe.go + runner.go | `*storage.SudoHostOps`; `WipeExecutor`+`DecommissionExecutor` | internal/signedjobs wipe/runner/decommission tests |
|
||||
| `hub.reporter` / `collectorIface` / `EnvelopeObserver` | internal/hub/loop.go | `*hub.Client`, `*hub.Collector`; `desired.Syncer` + `signedjobs.Runner` | `fakeReporter`/`fakeCollector` internal/hub/loop_test.go |
|
||||
| `hub.reporter` / `collectorIface` / `EnvelopeObserver` | internal/hub/loop.go | `*hub.Client`, `*hub.Collector`; `desired.Syncer` + `signedjobs.Runner` | `fakeReporter`/`fakeCollector` internal/hub/loop_test.go; `recordingReporter` loop_logtail_test.go |
|
||||
| `applog.Ring` (always-DEBUG capture ring) + fan-out `applog.New → (logger, ring)` | internal/log/log.go | wired in cmd/felhom-agent/main.go → `localapi.Options.LogRing` + `Loop.SetLogTailSource(ring.Lines)` | internal/log/log_test.go; localapi/debuglogs_test.go — v0.83.0; the byte-capped `Lines` is the heartbeat tail source |
|
||||
| `pbsdr.StorageReader` / `SecretConsumer` / `Manager.probeFP` (func seam) | internal/pbsdr/manager.go | `*proxmox.Client`; `*hub.Client`; `pbs.ProbeFingerprint` | `fakeStorage`/`fakeConsumer`/`fakeRunner` internal/pbsdr/manager_test.go (argv+stdin recorder) |
|
||||
| `capability.Runner` | internal/capability/probe.go | `*proxmox.ExecRunner` (RunnerDirect) | `fakeRunner` internal/capability/probe_test.go |
|
||||
| Cross-repo: local API ↔ controller | internal/localapi/server.go routes; contract seeded by internal/provision/doc.go (`bootstrap.json`: endpoint + leaf fingerprint + token) | felhom-controller's agentapi client | pin = served leaf cert (memory gotcha) |
|
||||
| Cross-repo: agent ↔ hub | internal/hub/report.go (`HostReport`), `ControlEnvelope`; POST `/api/v1/host-report` | hub mirrors structs field-for-field | new event/report fields need hub-side ingest changes |
|
||||
| Cross-repo: shipped host artifacts | configs/felhom-agent.sudoers, configs/felhom-mkfs-guarded.sh, shared-parent script/unit (inline in internal/localapi/intermediary.go) | deployed WITH the binary | sudoers globs must match `stageTemp` patterns + staging dirs exactly |
|
||||
| Cross-repo: shipped host artifacts | configs/felhom-agent.sudoers, configs/felhom-mkfs-guarded.sh, configs/felhom-pbs-apply, shared-parent script/unit (inline in internal/localapi/intermediary.go) | deployed WITH the binary | sudoers globs must match `stageTemp` patterns + staging dirs exactly |
|
||||
| Operator signing | internal/authz (OpBlob, SSHSIG) | cmd/felhom-opsign (offline CLI) | blob/verify tests in internal/authz |
|
||||
|
||||
## 5. Extension points (where new features plug in)
|
||||
@@ -147,6 +177,14 @@
|
||||
- **New privileged host op**: validate args (internal/storage/validate.go style) → exec via `Runner` → add a `Cmnd_Alias` to configs/felhom-agent.sudoers → add a probe vector to internal/capability/manifest.go (so degradation is visible) → ship sudoers with the binary.
|
||||
- **New reconcile action**: `ActionKind` + `classOfAction` (internal/reconcile/classify.go), plan emission in internal/reconcile/plan.go; destructive ⇒ gate handles it automatically.
|
||||
- **Hub-report field**: extend `hub.HostReport` (internal/hub/report.go) + `Collector` — hub side must mirror + allowlist it (cross-repo).
|
||||
- **DR-recipe section (host-half)**: add the field to `DRRecipeHostHalf` (internal/hub/dr_recipe.go) **AND** to
|
||||
the hub's `hostHalfShape` + `AssembledRecipe` (felhom.eu `hub/internal/store/dr_recipe.go`). Those two
|
||||
hub structs are **ALLOW-LISTS**: a section only the agent knows about is stored intact and silently
|
||||
dropped before any operator sees it — that is R-122, which cost `offsite_restic` its entire existence.
|
||||
Then update BOTH copies of `testdata/host-report.golden.json` (byte-identical, cross-repo) and extend
|
||||
`TestAssembleDRRecipe_CarriesEveryEmittedSection`. **A recipe field that cannot be resolved records an
|
||||
explicit unknown with a reason — never a default, an empty string, or a placeholder** (`DRState*` /
|
||||
`DRReason*`); a recipe read during a rebuild must not present a guess as a fact.
|
||||
- **Envelope-driven behavior**: implement `hub.EnvelopeObserver`, add to the `MultiObserver` in cmd/felhom-agent/main.go.
|
||||
- **Selftest mode**: `selftestFlag` + `runSelftest*` in cmd/felhom-agent/main.go.
|
||||
- **Config**: internal/config/config.go (`Load` + `applyEnv` `FELHOM_AGENT_*` overlay; keep secrets out of `Redacted()` output).
|
||||
@@ -155,8 +193,9 @@
|
||||
|
||||
- Two lsblk `-J` parsers with near-identical structs: `parseLsblkDevice`/`lsblkDevice` (internal/storage/hostops.go) vs `parseLsblkNodes`/`lsblkDev` (internal/storage/claim.go).
|
||||
- Two smartctl `-a -j` paths: `SudoHostOps.SMART` (internal/storage/hostops.go, parsed `hub.SmartSummary`) vs `Privileged.SMART` (internal/proxmox/privileged.go, raw map).
|
||||
- **SMART device resolution (v0.95.0):** `smartDeviceFor` (internal/storage/observe.go) resolves partition→disk AND dm/LVM→disk (`dmWholeDisk` in internal/storage/smartdev.go, via `/sys/block/<dm>/slaves`, `sysBlockRoot` test seam). `storage.SmartReader.SMARTForBacking` is the shared read the localapi `/disks` union path uses (Fix B) — do NOT re-implement smartctl parsing. The builtin-`local` SMART device comes from `containingMountDevice` (SMART-only; never feeds backing/durable_id).
|
||||
- Atomic tmp+rename JSON store implemented 3×: `IntentStore.saveLocked` (internal/storage/intent.go), `FormatJobStore.save` (internal/localapi/formatjob.go), `GuestBindStore.saveLocked` (internal/localapi/guestbindstore.go) — comments say "mirrors", no shared helper.
|
||||
- `run(ctx, name, args...) error` stderr-wrapping helper duplicated 4×: `SudoHostOps.run`, `Privileged.run`, `BackHalf.run` (internal/provision/backhalf.go), `GuestBinder.run` (internal/localapi/guestbind.go).
|
||||
- Several independent /proc mount-table readers: `SudoHostOps.mountedSet` (internal/storage/hostops.go), `ProcHostReader.Mounts` (internal/storage/hostread.go), `isHostMountpoint` + `countHostMounts` (internal/localapi/intermediary.go).
|
||||
- Several independent /proc mount-table readers: `SudoHostOps.mountedSet` (internal/storage/hostops.go), `ProcHostReader.Mounts` (internal/storage/hostread.go). **In localapi they were unified in v0.117.0**: `isHostMountpoint` and `countHostMounts` are now one-liners over `hostMountEntries`, the single parser that also yields devno/fstype/super-options for `bindLiveness`.
|
||||
- Deliberate mirror: `antiRetargetResolveExpect` (internal/localapi/wipe_reresolve.go) duplicates `WipeExecutor.Execute` steps 1–3 (internal/signedjobs/wipe.go) across packages.
|
||||
- `stableParentDir` literal duplicated in internal/provision/backhalf.go to avoid a provision→localapi import edge (commented as intentional); `trim` (internal/storage/hostops.go) vs `trimBody` (internal/proxmox/errors.go) output-truncation twins.
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// R-82 live regression (2026-07-26): the restore-test derived its tier from the CONFIGURED default
|
||||
// target instead of the archive's own storage. Restoring a `felhom-pbs:` archive on a box whose
|
||||
// primary target is "local" was classified "local" → the 10-minute local wait instead of the
|
||||
// generous PBS one → the wait expired mid-restore at 600s against a 14.46 GB WAN restore, teardown
|
||||
// fired at a still-restoring guest, and the scratch leaked.
|
||||
func TestArchiveStorageID(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"felhom-pbs:backup/ct/9201/2026-07-26T12:21:48Z", "felhom-pbs"},
|
||||
{"local:backup/vzdump-lxc-9201-2026_07_26-09_03_19.tar.zst", "local"},
|
||||
{"", ""},
|
||||
{"no-prefix", ""},
|
||||
{":leading-colon", ""}, // i>0 guard: a leading colon is not a storage id
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := archiveStorageID(c.in); got != c.want {
|
||||
t.Fatalf("archiveStorageID(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// struct (felhom-hub api.escrowUploadRequest). Cross-repo, no shared module — this is the agent
|
||||
// half of the contract guard; the hub has the mirror in its own test.
|
||||
func TestEscrowUploadContract(t *testing.T) {
|
||||
b, _ := json.Marshal(escrowUploadRequest{BlobB64: "x", KeyFingerprint: "y", Posture: "z", CreatedAt: "t"})
|
||||
b, _ := json.Marshal(escrowUploadRequest{BlobB64: "x", KeyFingerprint: "y", Posture: "z", CreatedAt: "t", ResticPwSHA256: "h"})
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -21,8 +21,16 @@ func TestEscrowUploadContract(t *testing.T) {
|
||||
got = append(got, k)
|
||||
}
|
||||
sort.Strings(got)
|
||||
want := []string{"blob_b64", "created_at", "key_fingerprint", "posture"}
|
||||
want := []string{"blob_b64", "created_at", "key_fingerprint", "posture", "restic_pw_sha256"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("escrow wire contract drift: got %v want %v (must match the hub ingest struct)", got, want)
|
||||
}
|
||||
// SLICE 3: no staged password folded in → the hash field is OMITTED on the wire (the hub stores NULL →
|
||||
// the controller never matches → stays pending; correct — the blob doesn't cover the key).
|
||||
b2, _ := json.Marshal(escrowUploadRequest{BlobB64: "x", KeyFingerprint: "y", Posture: "z", CreatedAt: "t"})
|
||||
var m2 map[string]any
|
||||
_ = json.Unmarshal(b2, &m2)
|
||||
if _, present := m2["restic_pw_sha256"]; present {
|
||||
t.Fatal("restic_pw_sha256 must be omitted when no staged password was sealed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Scenario H — THE SEAM IS WIRED IN THE PRODUCTION PATH, proven by walking the AST rather than by
|
||||
// grepping for a string.
|
||||
//
|
||||
// WHY THIS TEST EXISTS AND WHY IT IS AN AST WALK. This project's built-but-never-wired count is six,
|
||||
// and links 6 and 7 of the recovery chain were TWO of them: `UnwrapIdentityBundle` sat in the tree
|
||||
// for two months with no caller but a `--selftest`, and the hub's blob-serving endpoints have no
|
||||
// client to this day. The fix must not become the seventh. `strings.Contains` on the file would pass
|
||||
// against a commented-out line, a line inside a test helper, or a line in dead code behind a flag
|
||||
// nobody sets — so this resolves the call graph instead: `Options{EscrowRecovery: …}` must be
|
||||
// constructed inside a function that `runDaemon` reaches, and `runDaemon` must be reached by `main`.
|
||||
|
||||
func parseMain(t *testing.T) (*token.FileSet, *ast.File) {
|
||||
t.Helper()
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, parser.ParseComments)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing main.go: %v", err)
|
||||
}
|
||||
return fset, f
|
||||
}
|
||||
|
||||
// callsWithin returns the set of function names called (directly, by identifier or selector) inside
|
||||
// the named top-level function.
|
||||
func callsWithin(f *ast.File, fnName string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != fnName || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
ce, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
switch fn := ce.Fun.(type) {
|
||||
case *ast.Ident:
|
||||
out[fn.Name] = true
|
||||
case *ast.SelectorExpr:
|
||||
if x, ok := fn.X.(*ast.Ident); ok {
|
||||
out[x.Name+"."+fn.Sel.Name] = true
|
||||
}
|
||||
out[fn.Sel.Name] = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestEscrowRecoveryIsWiredIntoTheDaemon asserts the whole chain from func main() to the field.
|
||||
func TestEscrowRecoveryIsWiredIntoTheDaemon(t *testing.T) {
|
||||
_, f := parseMain(t)
|
||||
|
||||
// 1. main() reaches runDaemon.
|
||||
if !callsWithin(f, "main")["runDaemon"] {
|
||||
t.Fatal("func main() does not call runDaemon — the daemon path this test asserts is not the live one")
|
||||
}
|
||||
// 2. runDaemon reaches buildLocalAPIServer.
|
||||
if !callsWithin(f, "runDaemon")["buildLocalAPIServer"] {
|
||||
t.Fatal("runDaemon does not call buildLocalAPIServer — the local API is not built on the daemon path")
|
||||
}
|
||||
|
||||
// 3. Inside buildLocalAPIServer, a localapi.Options composite literal carries EscrowRecovery, and
|
||||
// an escrow.OffsiteKeyRecoverer is constructed there.
|
||||
var optionsHasField, recovererConstructed bool
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != "buildLocalAPIServer" || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
cl, ok := n.(*ast.CompositeLit)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := cl.Type.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
pkg, _ := sel.X.(*ast.Ident)
|
||||
if pkg == nil {
|
||||
return true
|
||||
}
|
||||
switch pkg.Name + "." + sel.Sel.Name {
|
||||
case "localapi.Options":
|
||||
for _, el := range cl.Elts {
|
||||
kv, ok := el.(*ast.KeyValueExpr)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if k, ok := kv.Key.(*ast.Ident); ok && k.Name == "EscrowRecovery" {
|
||||
optionsHasField = true
|
||||
}
|
||||
}
|
||||
case "escrow.OffsiteKeyRecoverer":
|
||||
recovererConstructed = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !recovererConstructed {
|
||||
t.Error("no escrow.OffsiteKeyRecoverer is constructed in buildLocalAPIServer — links 6→8 have no " +
|
||||
"production assembly point (the built-but-never-wired shape, seventh instance)")
|
||||
}
|
||||
if !optionsHasField {
|
||||
t.Error("localapi.Options in buildLocalAPIServer carries no EscrowRecovery field — the recoverer " +
|
||||
"exists and the route would answer 503 forever")
|
||||
}
|
||||
}
|
||||
|
||||
// The hub fetch must be the DAEMON's own hub client, not a freshly constructed one with different
|
||||
// credentials — the self-scoping that makes cross-host retrieval impossible is a property of WHICH
|
||||
// key is used.
|
||||
func TestEscrowRecoveryUsesTheDaemonHubClient(t *testing.T) {
|
||||
fset, f := parseMain(t)
|
||||
var fetchUsesHubClient bool
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != "buildLocalAPIServer" || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
ce, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := ce.Fun.(*ast.SelectorExpr)
|
||||
if !ok || sel.Sel.Name != "FetchIdentityEscrow" {
|
||||
return true
|
||||
}
|
||||
if x, ok := sel.X.(*ast.Ident); ok && x.Name == "hubClient" {
|
||||
fetchUsesHubClient = true
|
||||
} else {
|
||||
t.Errorf("FetchIdentityEscrow at %s is called on something other than the injected hub client",
|
||||
fset.Position(ce.Pos()))
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !fetchUsesHubClient {
|
||||
t.Fatal("the recoverer's fetcher does not call hubClient.FetchIdentityEscrow — either the fetch is " +
|
||||
"not wired, or it uses a client whose credentials are not this host's")
|
||||
}
|
||||
}
|
||||
|
||||
// The route itself must be registered on the local API. A handler with no route is the same defect
|
||||
// one layer down, and it has shipped here before.
|
||||
func TestRecoverRouteIsRegistered(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "../../internal/localapi/server.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing localapi/server.go: %v", err)
|
||||
}
|
||||
var registered bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
ce, ok := n.(*ast.CallExpr)
|
||||
if !ok || len(ce.Args) < 2 {
|
||||
return true
|
||||
}
|
||||
sel, ok := ce.Fun.(*ast.SelectorExpr)
|
||||
if !ok || sel.Sel.Name != "HandleFunc" {
|
||||
return true
|
||||
}
|
||||
lit, ok := ce.Args[0].(*ast.BasicLit)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(lit.Value, "/escrow/recover-offsite-password") {
|
||||
registered = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !registered {
|
||||
t.Fatal("POST /escrow/recover-offsite-password is not registered on the local API mux — the handler " +
|
||||
"exists and nothing can reach it")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// F10/rc255 (CAMPAIGN-3): a guest-hook phase body that PANICS must never crash the process — the hook
|
||||
// must return cleanly so the guest start proceeds (a nonzero exit blocks the start). runHookPhase
|
||||
// recovers the panic and returns.
|
||||
func TestRunHookPhase_PanicRecovered(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runHookPhase("9201", "pre-start", 5*time.Second, func(context.Context) {
|
||||
panic("simulated heal panic (e.g. a future Heal bug)")
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
// returned cleanly — the guest start would proceed
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("runHookPhase did not return after a panicking body (would have crashed the hook)")
|
||||
}
|
||||
}
|
||||
|
||||
// A phase body that overruns the timeout must be abandoned — the hook returns rather than hanging the
|
||||
// PVE start task. (The body's context is cancelled; the hook does not wait for the body to notice.)
|
||||
func TestRunHookPhase_TimeoutReturns(t *testing.T) {
|
||||
bodyCtxCancelled := make(chan struct{}, 1)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runHookPhase("9201", "post-start", 20*time.Millisecond, func(ctx context.Context) {
|
||||
<-ctx.Done() // simulate a body that respects cancellation eventually
|
||||
bodyCtxCancelled <- struct{}{}
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("runHookPhase did not return after the timeout (would hang the guest start)")
|
||||
}
|
||||
select {
|
||||
case <-bodyCtxCancelled:
|
||||
// the body's context was cancelled at the deadline — the intended signal
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("the phase body's context was not cancelled at the timeout")
|
||||
}
|
||||
}
|
||||
|
||||
// A body that errors (returns normally, no panic) is fine — the hook returns cleanly.
|
||||
func TestRunHookPhase_NormalBodyReturns(t *testing.T) {
|
||||
ran := false
|
||||
runHookPhase("9201", "pre-start", time.Second, func(context.Context) { ran = true })
|
||||
if !ran {
|
||||
t.Fatal("the phase body must run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The hook WIRING red-proof target: `guest-hook <vmid> post-start` must invoke the network
|
||||
// reassert with the vmid; pre-start and unknown phases must NOT. (Companion red-proof: remove the
|
||||
// PhasePostStart case from runGuestHook → the invoked assertion fails.)
|
||||
func TestRunGuestHook_PostStartInvokesNetworkReassert(t *testing.T) {
|
||||
orig := postStartNetworkReassertFn
|
||||
t.Cleanup(func() { postStartNetworkReassertFn = orig })
|
||||
|
||||
var gotVMIDs []string
|
||||
postStartNetworkReassertFn = func(_ context.Context, vmid string) {
|
||||
gotVMIDs = append(gotVMIDs, vmid)
|
||||
}
|
||||
|
||||
runGuestHook([]string{"9201", "post-start"})
|
||||
if len(gotVMIDs) != 1 || gotVMIDs[0] != "9201" {
|
||||
t.Fatalf("post-start must invoke the network reassert with vmid 9201, got %v", gotVMIDs)
|
||||
}
|
||||
|
||||
// pre-start must not touch the network reassert (it is the placeholder-heal phase; the heal
|
||||
// no-ops on a nonexistent config path and never blocks).
|
||||
runGuestHook([]string{"9201", "pre-start"})
|
||||
// unknown phases are ignored entirely.
|
||||
runGuestHook([]string{"9201", "pre-stop"})
|
||||
if len(gotVMIDs) != 1 {
|
||||
t.Fatalf("only post-start may invoke the network reassert, got %v", gotVMIDs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-54 §9 rule 6 — the seam-discipline test, and the one this repo has the most reason to write:
|
||||
// v0.91.0 shipped the PBS auth-probe seam with `main.go` never calling `SetAuthSink`, every unit
|
||||
// test green because they all injected the seam directly. The guestnet watchdog has the identical
|
||||
// shape (a component + a reporter seam + a goroutine), so its wiring is asserted here rather than
|
||||
// trusted.
|
||||
//
|
||||
// This walks the AST rather than grepping the source: a commented-out call still satisfies a
|
||||
// substring match (found while red-proofing the controller's twin of this test), and a comment is
|
||||
// not a caller.
|
||||
func TestMainWiresGuestNetWatchdog(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
|
||||
var constructed, reporterWired, started bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch node := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
switch fn := node.Fun.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
switch fn.Sel.Name {
|
||||
case "New":
|
||||
// guestnet.New(...)
|
||||
if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "guestnet" {
|
||||
constructed = true
|
||||
}
|
||||
case "SetGuestNetReporter":
|
||||
reporterWired = true
|
||||
}
|
||||
}
|
||||
case *ast.GoStmt:
|
||||
if sel, ok := node.Call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Watch" {
|
||||
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "gnWatchdog" {
|
||||
started = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if !constructed {
|
||||
t.Error("main.go never calls guestnet.New — the watchdog does not exist at runtime")
|
||||
}
|
||||
if !reporterWired {
|
||||
t.Error("main.go never calls collector.SetGuestNetReporter — the guest_net stanza would " +
|
||||
"never reach the hub (the exact v0.91.0 inert-seam defect)")
|
||||
}
|
||||
if !started {
|
||||
t.Error("main.go never starts the watchdog with `go gnWatchdog.Watch(ctx)` — it would be " +
|
||||
"constructed, reported on, and never probe anything")
|
||||
}
|
||||
}
|
||||
|
||||
// The watchdog must NOT join the errc fan-out: a guest-network watchdog that can terminate the
|
||||
// agent turns a customer's DHCP problem into an operator-plane outage. If it is ever changed to
|
||||
// `errc <- ...`, the drain bound at the bottom of main() also has to change — this catches the
|
||||
// first half of that mistake.
|
||||
func TestGuestNetWatchdogIsNotInTheErrcFanout(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
bad := false
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
send, ok := n.(*ast.SendStmt)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if ch, ok := send.Chan.(*ast.Ident); !ok || ch.Name != "errc" {
|
||||
return true
|
||||
}
|
||||
call, ok := send.Value.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Watch" {
|
||||
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "gnWatchdog" {
|
||||
bad = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if bad {
|
||||
t.Fatal("the guestnet watchdog was added to the errc fan-out — a watchdog over customer " +
|
||||
"guests must never be able to bring the agent down (and the drain bound in main() " +
|
||||
"would now be off by one, hanging shutdown)")
|
||||
}
|
||||
}
|
||||
+1230
-89
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,156 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-86 Scenario I — the seam-discipline test for the due-check.
|
||||
//
|
||||
// A due-check is worth nothing if the daemon still wires the OLD picker: every unit test in
|
||||
// internal/backup would stay green (they inject the seam directly), the scheduler would ask for the
|
||||
// newest archive with no settle cutoff, and the per-archive rule would run against a candidate that
|
||||
// changes every time a backup lands. That is the same shape as the v0.91.0 inert seam — built,
|
||||
// tested, never called — and this repo has shipped it four times.
|
||||
//
|
||||
// It walks main.go's AST rather than grepping: a commented-out call still satisfies a substring
|
||||
// match, and a comment is not a caller.
|
||||
func TestMainWiresTheSettleAwareTierPicker(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var settlePicker, oldPicker, settleWired, evalInterval bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch node := n.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
// runner.PickSettledRestoreCandidateOn passed as a value (not called).
|
||||
switch node.Sel.Name {
|
||||
case "PickSettledRestoreCandidateOn":
|
||||
settlePicker = true
|
||||
case "PickRestoreCandidateOn":
|
||||
oldPicker = true
|
||||
}
|
||||
case *ast.KeyValueExpr:
|
||||
key, ok := node.Key.(*ast.Ident)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if key.Name == "Settle" {
|
||||
settleWired = true
|
||||
}
|
||||
case *ast.CallExpr:
|
||||
if sel, ok := node.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "RestoreTestEvalInterval" {
|
||||
evalInterval = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if !settlePicker {
|
||||
t.Error("main.go never passes runner.PickSettledRestoreCandidateOn as the scheduler's TierPick — " +
|
||||
"the due-check would run without a settle cutoff, i.e. against an archive that may still be being written")
|
||||
}
|
||||
if oldPicker {
|
||||
t.Error("main.go still wires the pre-R-86 PickRestoreCandidateOn as a tier picker — " +
|
||||
"two pickers means the one under test is not the one running")
|
||||
}
|
||||
if !settleWired {
|
||||
t.Error("main.go never sets SchedulerOptions.Settle — the settle lag would default to 0 in the daemon " +
|
||||
"and every freshly-landed archive would be an immediate candidate")
|
||||
}
|
||||
if !evalInterval {
|
||||
t.Error("main.go never calls cfg.Backup.RestoreTestEvalInterval() — the scheduler would be driven by " +
|
||||
"the retired cadence knob")
|
||||
}
|
||||
}
|
||||
|
||||
// The two R-85 guarantees the due-check must not have quietly dropped: the spec is still built PER
|
||||
// RUN, and the shared heavy-operation gate is still handed to the scheduler.
|
||||
func TestMainStillWiresTheHeavyOperationGateAndPerRunSpec(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var inFlightWired, specIsAFunc bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
kv, ok := n.(*ast.KeyValueExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
key, ok := kv.Key.(*ast.Ident)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
switch key.Name {
|
||||
case "InFlight":
|
||||
inFlightWired = true
|
||||
case "Spec":
|
||||
// A FuncLit means it is evaluated per run; anything else is a frozen value.
|
||||
if _, isFunc := kv.Value.(*ast.FuncLit); isFunc {
|
||||
specIsAFunc = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if !inFlightWired {
|
||||
t.Error("main.go no longer hands the scheduler the shared InFlight gate — a restore-test could pull a " +
|
||||
"multi-GB archive over the same tunnel an offsite backup is pushing one over (Scenario F)")
|
||||
}
|
||||
if !specIsAFunc {
|
||||
t.Error("SchedulerOptions.Spec is no longer a function literal — a frozen spec is the R-85 defect " +
|
||||
"(the tier and its timeout evaluated once at daemon start, forever)")
|
||||
}
|
||||
}
|
||||
|
||||
func parseMainForWiring(t *testing.T) *ast.File {
|
||||
t.Helper()
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// R-189 Scenario I — the DURABLE proof source must actually be wired into the collector.
|
||||
//
|
||||
// This test exists because the method it feeds is the project's own cautionary tale:
|
||||
// `RestoreTestState.Snapshot` carried the doc comment "for the host-report gauge" from the day it
|
||||
// was written and **had no caller at all** — a seam built, documented and never connected, found
|
||||
// only when a live restore-test's PASS reached no host-report. The fix must not become the next
|
||||
// instance, so the wiring is asserted rather than trusted.
|
||||
//
|
||||
// AST, not grep: a commented-out call still contains the string (proven yesterday, when commenting
|
||||
// out the tier-picker line failed this test while a `strings.Contains` check would have passed).
|
||||
func TestMainWiresTheDurableRestoreTestProof(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var wired, feedsState bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok || sel.Sel.Name != "SetProvenRestoreTests" {
|
||||
return true
|
||||
}
|
||||
wired = true
|
||||
// ...and it must be fed the PERSISTED state, not the in-memory store.
|
||||
if len(call.Args) == 1 {
|
||||
if id, ok := call.Args[0].(*ast.Ident); ok && id.Name == "rtState" {
|
||||
feedsState = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if !wired {
|
||||
t.Error("main.go never calls collector.SetProvenRestoreTests — the persisted proof would never " +
|
||||
"reach the hub, which is the R-189 defect exactly: a passing restore-test that vanishes on restart")
|
||||
}
|
||||
if wired && !feedsState {
|
||||
t.Error("collector.SetProvenRestoreTests is not fed rtState — the in-memory store is the thing " +
|
||||
"that does NOT survive a restart, so wiring it here would fix nothing")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
||||
)
|
||||
|
||||
// COMPILE-TIME WITNESSES for OPTIONAL interfaces that are satisfied by a RUNTIME type assertion.
|
||||
//
|
||||
// WHY THIS FILE EXISTS. `localapi.BackupArchiveLister` is asserted at server.go's `newestArchiveOn`
|
||||
// via `tier.Service.(BackupArchiveLister)`. A failed assertion does not error — it degrades to
|
||||
// `archiveAbsent`, i.e. the pre-R-84 "ask the in-memory record only" behaviour. That degrade is
|
||||
// SILENT and it is behaviour-relevant: it is exactly the R-84 bug (a cold store after a restart
|
||||
// reading as "no backup ever") coming back, with nothing in any log to say so.
|
||||
//
|
||||
// The precedent is not hypothetical. During R-88 Part 2 the controller's `quiesceBackend` stopped
|
||||
// satisfying `quiesce.TieredBackend` when a signature changed, and `go build` AND `go vet` both
|
||||
// passed — because the interface is only ever asserted at runtime. Every box would have degraded to
|
||||
// the single-tier path, losing R-82's multi-tier backups, with no error anywhere. It was caught by
|
||||
// accident.
|
||||
//
|
||||
// A witness costs one line and converts that class of failure from a silent production degrade into
|
||||
// a compile error.
|
||||
//
|
||||
// THIS DOES NOT MAKE THE INTERFACE REQUIRED. The optionality is deliberate — it is what lets a
|
||||
// BackupService without a lister still work. The witness pins the IMPLEMENTATION (this concrete type
|
||||
// really does satisfy it), not the CONTRACT.
|
||||
var _ localapi.BackupArchiveLister = (*backup.BackupRunner)(nil)
|
||||
@@ -0,0 +1,417 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"go/ast"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
|
||||
)
|
||||
|
||||
// R-185 — a tier the box cannot READ must say so.
|
||||
//
|
||||
// THE OBSERVATION (demo-felhom, 2026-08-03, reproduced at the start of this session): root lists
|
||||
// three archives on `felhom-backup`; the agent's own token gets `{"data":[]}` from the same
|
||||
// endpoint; and `local`, which has the grant, lists through that same token. The token is the
|
||||
// variable, not the storage.
|
||||
//
|
||||
// The defect is NOT the missing grant — that is one command. It is that an empty content listing is
|
||||
// what a FORBIDDEN tier and a NEWBORN tier both return, so the box could not tell them apart and
|
||||
// said nothing. These tests pin the distinction.
|
||||
|
||||
// permAnswer is the shape /access/permissions really returns, taken from the live measurement:
|
||||
// an UNGRANTED path answers with the privileges inherited from the box-wide grant — NOT empty, and
|
||||
// NOT a 403.
|
||||
var (
|
||||
permGranted = map[string]int{"Datastore.Allocate": 1, "Datastore.AllocateSpace": 1}
|
||||
permUngranted = map[string]int{"Sys.Audit": 1, "SDN.Use": 1, "Datastore.Audit": 1}
|
||||
)
|
||||
|
||||
// probeWith calls the PRODUCTION decision with a permissions answer. **Naming the seam:** everything
|
||||
// below is true up to `storeGrantVerdict`; that the live call feeds it the real API answer is what
|
||||
// Part 0's measurement established and what the live run on the box demonstrates. An earlier draft
|
||||
// of this file re-implemented the branch here — it passed, and would have kept passing while
|
||||
// production diverged, which is the hollow shape this project keeps catching in its own tests.
|
||||
func probeWith(privs map[string]int, targetID string, critical bool) capability.Status {
|
||||
return storeGrantVerdict(targetID, critical, privs, nil)
|
||||
}
|
||||
|
||||
// ── SCENARIO A — a forbidden storage is REPORTED, not passed over ────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-03): delete the store-grant probes from `probeAll` in
|
||||
// main.go — i.e. restore `append(capProber.Probe(ctx), poolReadStatus(ctx, px))` — and
|
||||
// TestMainWiresTheStoreGrantProbe fails with "main.go never calls storeGrantStatuses". That is
|
||||
// today's behaviour on the live box: complete silence about a tier it cannot read.
|
||||
func TestStoreGrant_ForbiddenStorageIsDegradedAndNamed(t *testing.T) {
|
||||
s := probeWith(permUngranted, "felhom-backup", true)
|
||||
|
||||
if s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("a storage the agent may not read must be DEGRADED, not %q — silence is the defect", s.Status)
|
||||
}
|
||||
if !s.Critical {
|
||||
t.Fatal("it must be CRITICAL: the hub alerts only on critical, so a non-critical entry is the same silence with extra steps")
|
||||
}
|
||||
if !strings.Contains(s.Reason, "felhom-backup") {
|
||||
t.Fatalf("the reason must NAME the storage — 'a grant is missing' costs a diagnosis at 07:00; got %q", s.Reason)
|
||||
}
|
||||
if !strings.Contains(s.Reason, "FelhomAgentStore") {
|
||||
t.Fatalf("the reason must name the ROLE to grant, so the fix is in the alert; got %q", s.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// THE TRAP THE LIVE MEASUREMENT CAUGHT, pinned so it cannot be re-introduced: the ungranted answer
|
||||
// is not empty and not a 403 — it carries the INHERITED box-wide privileges. A probe that asked
|
||||
// "did the path come back?" or "does it have Datastore.Audit?" would report the blinded storage
|
||||
// healthy.
|
||||
func TestStoreGrant_InheritedPrivilegesAreNotAGrant(t *testing.T) {
|
||||
if len(permUngranted) == 0 {
|
||||
t.Fatal("fixture wrong: the ungranted answer is NOT empty — that is the whole trap")
|
||||
}
|
||||
if permUngranted["Datastore.Audit"] != 1 {
|
||||
t.Fatal("fixture wrong: the ungranted path DOES carry Datastore.Audit, inherited box-wide")
|
||||
}
|
||||
if s := probeWith(permUngranted, "felhom-backup", true); s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("checking for the wrong privilege reports a blinded storage healthy; got %q", s.Status)
|
||||
}
|
||||
// ...and the privilege actually checked is the one whose absence was measured to blind listing.
|
||||
if storeGrantRequiredPriv != "Datastore.AllocateSpace" {
|
||||
t.Fatalf("the probed privilege changed to %q — re-measure before trusting it", storeGrantRequiredPriv)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO B — a newborn tier is still silent ──────────────────────────────────────────────
|
||||
//
|
||||
// A storage the agent IS allowed to read but which simply holds no archives yet is HEALTHY. The
|
||||
// probe must not look at content at all, or every freshly provisioned box alarms and the signal dies.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): make the probe degrade on an empty content listing instead of on
|
||||
// the permission — a granted-but-empty storage then reports degraded, i.e. every newborn box alarms.
|
||||
func TestStoreGrant_GrantedButEmptyIsHealthy(t *testing.T) {
|
||||
s := probeWith(permGranted, "felhom-pbs", true)
|
||||
if s.Status != capability.StatusOK {
|
||||
t.Fatalf("a readable tier is healthy whether or not it holds archives yet; got %q (%s)", s.Status, s.Reason)
|
||||
}
|
||||
if s.Reason != "" {
|
||||
t.Fatalf("a healthy probe carries no reason; got %q", s.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO C — the two states are distinguishable at a glance ──────────────────────────────
|
||||
func TestStoreGrant_ForbiddenAndNewbornAreDistinguishable(t *testing.T) {
|
||||
forbidden := probeWith(permUngranted, "felhom-backup", true)
|
||||
newborn := probeWith(permGranted, "felhom-pbs", true)
|
||||
|
||||
if forbidden.Status == newborn.Status {
|
||||
t.Fatalf("the two states must differ — today both read as 'no settled archive yet'; got %q for both", forbidden.Status)
|
||||
}
|
||||
if forbidden.Name == newborn.Name {
|
||||
t.Fatalf("each tier needs its own capability id, or one tier's fault hides another's; got %q twice", forbidden.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// §8.3, weighed once and pinned: a box with NO dedicated target ("local" — host-install's own
|
||||
// DEGRADED fallback) must not turn an ordinary configuration into an operator page. It is still
|
||||
// probed and still reported; only the paging differs.
|
||||
func TestStoreGrant_TheFallbackTargetIsNotCritical(t *testing.T) {
|
||||
if storeGrantCritical("local") {
|
||||
t.Fatal("a box whose backup target is the 'local' fallback must not page the operator about " +
|
||||
"an ordinary, documented configuration")
|
||||
}
|
||||
for _, dedicated := range []string{"felhom-backup", "felhom-pbs", "some-nvme"} {
|
||||
if !storeGrantCritical(dedicated) {
|
||||
t.Fatalf("a DEDICATED target that cannot be read is user-facing and must be critical; %q was not", dedicated)
|
||||
}
|
||||
}
|
||||
// The fallback is still reported — silence for it would be the original defect, scoped smaller.
|
||||
if s := probeWith(permUngranted, "local", storeGrantCritical("local")); s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("the fallback target must still report degraded when unreadable; got %q", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// A probe that cannot ask must never answer "ok" — unknown reported as healthy is worse than no
|
||||
// probe, because it looks like coverage.
|
||||
func TestStoreGrant_UnreachablePVEIsDegradedNotOK(t *testing.T) {
|
||||
s := storeGrantStatus(context.Background(), nil, "felhom-backup", true, nil)
|
||||
if s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("an unaskable probe must be DEGRADED, never ok; got %q", s.Status)
|
||||
}
|
||||
if s.Reason == "" {
|
||||
t.Fatal("it must say why it could not ask")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO H — the seam ────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// This project's "built but never wired" count reached six last week. The fix for a SILENCE must not
|
||||
// itself be silent. AST, not grep: a commented-out call still contains the string.
|
||||
func TestMainWiresTheStoreGrantProbe(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var wired bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "storeGrantStatuses" {
|
||||
wired = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !wired {
|
||||
t.Error("main.go never calls storeGrantStatuses — the probe would exist and report to nobody, " +
|
||||
"which is precisely the silence R-185 is about")
|
||||
}
|
||||
}
|
||||
|
||||
// ── R-190 — the grant repairs itself, and the repair is VISIBLE ──────────────────────────────
|
||||
//
|
||||
// R-190 is a storage grant that demonstrably worked at 04:44 on 2026-08-03 and was gone by 09:24,
|
||||
// with a host reinstall, logged `pveum` activity and cluster-log entries all ruled out. The cause is
|
||||
// open; the resilience is not conditional on it.
|
||||
//
|
||||
// The half that matters is the RECORD. R-190's own words: the probe sees the state, nothing sees the
|
||||
// transition. A self-repair that leaves only "ok" behind destroys the only evidence a loss happened,
|
||||
// so a recurring loss becomes undetectable forever — strictly worse than the fault it fixes.
|
||||
|
||||
// fakeRepairRunner records wrapper invocations and can be made to fail.
|
||||
type fakeRepairRunner struct {
|
||||
calls [][]string
|
||||
fail bool
|
||||
}
|
||||
|
||||
func (f *fakeRepairRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
f.calls = append(f.calls, append([]string{name}, args...))
|
||||
if f.fail {
|
||||
return nil, []byte("pveum: refused"), errors.New("exit status 2")
|
||||
}
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func newRepairer(f *fakeRepairRunner) *storeGrantRepairer {
|
||||
return &storeGrantRepairer{run: f.Run, log: slog.New(slog.NewTextHandler(io.Discard, nil))}
|
||||
}
|
||||
|
||||
// ── SCENARIO F — the repair is BOUNDED ───────────────────────────────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-04): make mayAttempt always return true (drop the
|
||||
// storeGrantRepairMinInterval check) →
|
||||
//
|
||||
// --- FAIL: TestGrantRepair_IsBounded
|
||||
// storegrant_test.go: a repair must not run on every cycle; 5 cycles produced 5 attempt(s)
|
||||
//
|
||||
// which is a re-grant every report cycle, forever, against a fault an ACL cannot fix. Restored.
|
||||
func TestGrantRepair_IsBounded(t *testing.T) {
|
||||
f := &fakeRepairRunner{}
|
||||
r := newRepairer(f)
|
||||
// Jittered, so the series never lands exactly on the interval boundary — a perfectly regular
|
||||
// series is how a threshold test passes its own mutation, which has happened here before.
|
||||
base := time.Date(2026, 8, 4, 9, 17, 43, 0, time.UTC)
|
||||
offsets := []time.Duration{0, 13*time.Minute + 7*time.Second, 27*time.Minute + 51*time.Second,
|
||||
41*time.Minute + 19*time.Second, 55*time.Minute + 3*time.Second}
|
||||
attempts := 0
|
||||
for _, off := range offsets {
|
||||
if r.mayAttempt("felhom-backup", base.Add(off)) {
|
||||
attempts++
|
||||
}
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Fatalf("a repair must not run on every cycle; %d cycles produced %d attempt(s) within %s",
|
||||
len(offsets), attempts, storeGrantRepairMinInterval)
|
||||
}
|
||||
// ...and once the interval has genuinely passed, it may try again — a bound is not a ban.
|
||||
if !r.mayAttempt("felhom-backup", base.Add(storeGrantRepairMinInterval+2*time.Minute+11*time.Second)) {
|
||||
t.Fatal("after the interval a repair must be allowed again — otherwise one failure disables the repair forever")
|
||||
}
|
||||
// A DIFFERENT tier is not throttled by this one's attempt.
|
||||
if !r.mayAttempt("felhom-pbs", base.Add(time.Minute)) {
|
||||
t.Fatal("the bound must be per tier — one tier's attempt must not suppress another's")
|
||||
}
|
||||
}
|
||||
|
||||
// A nil repairer (or one with no runner) never attempts, and never panics.
|
||||
func TestGrantRepair_NilIsSafe(t *testing.T) {
|
||||
var r *storeGrantRepairer
|
||||
if r.mayAttempt("felhom-backup", time.Now()) {
|
||||
t.Fatal("a nil repairer must never claim an attempt")
|
||||
}
|
||||
if (&storeGrantRepairer{}).mayAttempt("felhom-backup", time.Now()) {
|
||||
t.Fatal("a repairer with no runner must never claim an attempt")
|
||||
}
|
||||
}
|
||||
|
||||
// The repair calls the EXISTING wrapper verb, with the storage id — no new privileged surface.
|
||||
func TestGrantRepair_CallsTheExistingWrapperVerb(t *testing.T) {
|
||||
f := &fakeRepairRunner{}
|
||||
r := newRepairer(f)
|
||||
if err := r.repair(context.Background(), "felhom-backup"); err != nil {
|
||||
t.Fatalf("repair should succeed with a healthy runner: %v", err)
|
||||
}
|
||||
if len(f.calls) != 1 {
|
||||
t.Fatalf("exactly one wrapper invocation expected; got %d", len(f.calls))
|
||||
}
|
||||
got := f.calls[0]
|
||||
want := []string{"/usr/local/sbin/felhom-backup-target-apply", "grant", "felhom-backup"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("wrapper argv = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("wrapper argv = %v, want %v — the sudoers vector is `grant *`; anything else is a policy change", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A repair that FAILS must surface the failure, not swallow it (Scenario E's precondition).
|
||||
func TestGrantRepair_FailureIsReturned(t *testing.T) {
|
||||
f := &fakeRepairRunner{fail: true}
|
||||
if err := newRepairer(f).repair(context.Background(), "felhom-backup"); err == nil {
|
||||
t.Fatal("a failed wrapper run must return its error — a repair that cannot run must never read as done")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO D (the half that matters) — the REPAIR MUST BE VISIBLE ──────────────────────────
|
||||
//
|
||||
// A repair that leaves only "ok" behind is worse than the fault: the tier works, and the fact that a
|
||||
// permission vanished is gone with it. R-190 exists because nothing saw the transition.
|
||||
//
|
||||
// The channel is the hub's EXISTING ok→degraded→ok edge (§8.5) — nothing new was built. That only
|
||||
// works if the agent deliberately reports ONE degraded cycle after repairing, and if the explanation
|
||||
// rides the field the hub actually puts in the operator's e-mail. The hub's message is built from the
|
||||
// capability NAME and FEATURE (`internal/monitor/host_capability.go` emitTransition) — **not** from
|
||||
// Reason — so the Feature must carry it.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-04): after a successful repair, report ok instead —
|
||||
//
|
||||
// s.Status = capability.StatusOK; s.Feature unchanged
|
||||
//
|
||||
// → --- FAIL: TestGrantRepair_ARepairedGrantIsReportedAsATransition
|
||||
//
|
||||
// storegrant_test.go: a self-repair must still report DEGRADED for one cycle so the hub raises
|
||||
// its edge; got "ok" — the loss would be invisible
|
||||
//
|
||||
// i.e. exactly the silence R-190 is about. Restored.
|
||||
func TestGrantRepair_ARepairedGrantIsReportedAsATransition(t *testing.T) {
|
||||
// THE PRODUCTION verdict, not a copy of it. An earlier draft of this test built the Status
|
||||
// itself and asserted its own construction — it would have passed while production reported ok,
|
||||
// which is precisely the silence being guarded against.
|
||||
if pre := probeWith(permUngranted, "felhom-backup", true); pre.Status != capability.StatusDegraded {
|
||||
t.Fatalf("precondition: a missing grant is degraded; got %q", pre.Status)
|
||||
}
|
||||
s := storeGrantRepairedVerdict("felhom-backup", true)
|
||||
|
||||
if s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("a self-repair must still report DEGRADED for one cycle so the hub raises its edge; "+
|
||||
"got %q — the loss would be invisible", s.Status)
|
||||
}
|
||||
// The hub e-mails the FEATURE text. If the explanation is not there, the operator is told a
|
||||
// capability was degraded and never learns it repaired itself or that anything vanished.
|
||||
for _, want := range []string{"MISSING", "RESTORED", "felhom-backup", "R-190"} {
|
||||
if !strings.Contains(s.Feature, want) {
|
||||
t.Fatalf("the Feature text is what the hub puts in the operator's e-mail; it must contain %q. Got: %s", want, s.Feature)
|
||||
}
|
||||
}
|
||||
if !s.Critical {
|
||||
t.Fatal("the transition must be CRITICAL or the hub does not alert on it at all")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO H — the seam ────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// The wrapper's `grant` verb is itself a "built but never wired" example: it exists, is
|
||||
// sudoers-permitted for any id, and had only ever been called at storage CREATION. The repair must
|
||||
// not become the seventh instance. AST, not grep — a commented-out call still contains the string.
|
||||
func TestMainWiresTheGrantRepair(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var built, passed bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch node := n.(type) {
|
||||
case *ast.CompositeLit:
|
||||
if id, ok := node.Type.(*ast.Ident); ok && id.Name == "storeGrantRepairer" {
|
||||
built = true
|
||||
}
|
||||
case *ast.CallExpr:
|
||||
if id, ok := node.Fun.(*ast.Ident); ok && id.Name == "storeGrantStatuses" && len(node.Args) == 4 {
|
||||
if a, ok := node.Args[3].(*ast.Ident); ok && a.Name == "grantRepairer" {
|
||||
passed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !built {
|
||||
t.Error("main.go never constructs a storeGrantRepairer — nothing would ever repair a lost grant")
|
||||
}
|
||||
if !passed {
|
||||
t.Error("storeGrantStatuses is not passed the repairer — the probe would detect the loss and " +
|
||||
"leave it, which is v0.123.0's behaviour and not R-190's mitigation")
|
||||
}
|
||||
}
|
||||
|
||||
// The transition must survive a probe that is NOT the one feeding the hub.
|
||||
//
|
||||
// MEASURED LIVE 2026-08-04, and this test exists because the first implementation failed it in
|
||||
// production while every unit test passed: `probeAll` is called independently by the self-check LOG
|
||||
// and by the collector building a host-report. The repairing call was the log's; the report three
|
||||
// seconds later found the grant present and reported `ok`. The agent's journal had the record and the
|
||||
// hub had nothing — the exact silence R-190 is about, re-created inside its own mitigation.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): delete the `recentlyRepaired` branch from the healthy path →
|
||||
//
|
||||
// --- FAIL: TestGrantRepair_TransitionSurvivesALaterProbe
|
||||
// storegrant_test.go: a probe AFTER the repair must still report the transition; got "ok" —
|
||||
// the host-report would carry ok and the operator would never learn the grant vanished
|
||||
//
|
||||
// Restored.
|
||||
func TestGrantRepair_TransitionSurvivesALaterProbe(t *testing.T) {
|
||||
r := newRepairer(&fakeRepairRunner{})
|
||||
// Jittered, never landing on the window boundary.
|
||||
repairedAt := time.Date(2026, 8, 4, 9, 39, 34, 0, time.UTC)
|
||||
r.noteRepaired("felhom-backup", repairedAt)
|
||||
|
||||
// The DECISION a later probe makes — the production function, not the helper it calls. An
|
||||
// earlier draft asserted `recentlyRepaired` directly and its red-proof PASSED, because removing
|
||||
// the latch's USE left the helper untouched.
|
||||
healthy := probeWith(permGranted, "felhom-backup", true)
|
||||
if healthy.Status != capability.StatusOK {
|
||||
t.Fatalf("precondition: a granted tier is ok; got %q", healthy.Status)
|
||||
}
|
||||
got := storeGrantHealthyVerdict("felhom-backup", true,
|
||||
healthy, r.recentlyRepaired("felhom-backup", repairedAt.Add(3*time.Second)))
|
||||
if got.Status != capability.StatusDegraded {
|
||||
t.Fatalf("a probe AFTER the repair must still report the transition; got %q — the host-report "+
|
||||
"would carry ok and the operator would never learn the grant vanished", got.Status)
|
||||
}
|
||||
if !strings.Contains(got.Feature, "RESTORED") {
|
||||
t.Fatalf("the later probe must carry the explanation into the hub's e-mail; got: %s", got.Feature)
|
||||
}
|
||||
// Outside the window it reports plain ok again.
|
||||
late := storeGrantHealthyVerdict("felhom-backup", true,
|
||||
healthy, r.recentlyRepaired("felhom-backup", repairedAt.Add(storeGrantRepairReportWindow+time.Minute)))
|
||||
if late.Status != capability.StatusOK {
|
||||
t.Fatalf("outside the window a healthy tier reports ok; got %q — a permanent degraded state "+
|
||||
"would be its own false alarm", late.Status)
|
||||
}
|
||||
if !r.recentlyRepaired("felhom-backup", repairedAt.Add(14*time.Minute+37*time.Second)) {
|
||||
t.Fatal("the latch must outlast the 900s hub report interval, or the record never reaches the hub")
|
||||
}
|
||||
// ...and it clears on its own rather than latching a box degraded forever.
|
||||
if r.recentlyRepaired("felhom-backup", repairedAt.Add(storeGrantRepairReportWindow+time.Minute+7*time.Second)) {
|
||||
t.Fatal("the latch must clear — a permanent degraded state would be its own false alarm")
|
||||
}
|
||||
// It is per tier.
|
||||
if r.recentlyRepaired("felhom-pbs", repairedAt.Add(time.Second)) {
|
||||
t.Fatal("one tier's repair must not latch another tier's status")
|
||||
}
|
||||
// The window MUST exceed the report interval — the property, asserted rather than assumed.
|
||||
if storeGrantRepairReportWindow <= 15*time.Minute {
|
||||
t.Fatalf("the report window (%s) must exceed the 900s hub report interval, or a transition can "+
|
||||
"be missed entirely", storeGrantRepairReportWindow)
|
||||
}
|
||||
}
|
||||
@@ -48,10 +48,16 @@
|
||||
},
|
||||
"local_api": {
|
||||
"enable": true,
|
||||
"listen_addr": "192.168.0.162:8443",
|
||||
"listen_addr": "169.254.253.1:8443",
|
||||
"cert_file": "/var/lib/felhom-agent/local-api.crt",
|
||||
"key_file": "/var/lib/felhom-agent/local-api.key",
|
||||
"token_store": "/var/lib/felhom-agent/local-tokens.log"
|
||||
"token_store": "/var/lib/felhom-agent/local-tokens.log",
|
||||
"island_bridge": "vmbr9",
|
||||
"island_guest_addr": "169.254.253.2/30"
|
||||
},
|
||||
"lan_resolver": {
|
||||
"enable": true,
|
||||
"host_ip": "192.168.0.162"
|
||||
},
|
||||
"log_level": "info"
|
||||
}
|
||||
|
||||
+146
-52
@@ -26,20 +26,42 @@
|
||||
# Build-time registry login for the controller pull (used ONCE inside the build guest, then logged
|
||||
# out — never baked): set REGISTRY_USER + REGISTRY_TOKEN in the environment.
|
||||
#
|
||||
# OS / Docker-data SPLIT (storage-split slice): the golden is built with a SMALL OS rootfs and a
|
||||
# SEPARATE Docker-data volume mounted at /var/lib/docker (mp0, backup=1). The baked controller +
|
||||
# infra images land on that volume and travel INSIDE the golden archive — so provisioned guests boot
|
||||
# from baked images with no registry pull. The split is for RESILIENCE: an isolated OS rootfs stays
|
||||
# bootable + agent-recoverable if the Docker volume fills (the controller's prevention layer keeps it
|
||||
# from filling). Sizes are env-overridable (OS_SIZE_GB / GOLDEN_DOCKER_GB); provision GROWS the data
|
||||
# volume to the per-customer target (bringup.go DataVolGrowGB). backup=1 is MANDATORY on the data mp:
|
||||
# without it vzdump EXCLUDES the volume (extra LXC mountpoints default backup=0 — storage-split B3),
|
||||
# so the archive would carry NO images and provisioned guests would boot imageless.
|
||||
# OS / DATA SPLIT, and since v3.0.0 ONE DATA VOLUME (R-165, decision D-a + variant V-c).
|
||||
#
|
||||
# The golden is built with a SMALL OS rootfs and a SINGLE data volume (mp0, backup=1) mounted at a
|
||||
# NEUTRAL path, /var/lib/felhom. Both consumer paths are binds of subdirectories of it:
|
||||
#
|
||||
# /var/lib/felhom/docker --bind--> /var/lib/docker (Docker's data-root)
|
||||
# /var/lib/felhom/sys_drive --bind--> /mnt/sys_drive (the controller's system_data_path)
|
||||
#
|
||||
# WHAT THIS REPLACED, AND WHY. Until v2.1.0 these were TWO volumes (mp0 16 G at /var/lib/docker,
|
||||
# mp1 8 G at /mnt/sys_drive, grown separately at provision). The second one was a fixed ceiling: an
|
||||
# app whose local recovery unit outgrew it stopped being backed up even with free space next door.
|
||||
# D-a removed the wall rather than moving it — one volume, one free-space figure, no ceiling.
|
||||
#
|
||||
# WHY A NEUTRAL MOUNT AND NOT SIMPLY NESTING ONE PATH INSIDE THE OTHER. Both simpler shapes were
|
||||
# built and measured (SPIKE-r165-phase0-2026-08-03.md); both boot and reboot cleanly, and each breaks
|
||||
# a different documented guarantee:
|
||||
# * volume at /var/lib/docker -> customer backups live INSIDE Docker's data-root, so `du` there
|
||||
# stops meaning what it says and the ordinary "clear /var/lib/docker to fix Docker" reflex
|
||||
# destroys every local recovery unit on the box;
|
||||
# * volume at /mnt/sys_drive -> Docker's ENTIRE data-root lands under /mnt, which the controller
|
||||
# container mounts wholesale (`-v /mnt:/mnt:rslave`). Measured: the container then sees
|
||||
# /mnt/sys_drive/docker. The bootstrap's own claim that /mnt "holds only Felhom's
|
||||
# felhom-data-namespace mounts" would become false.
|
||||
# The neutral mount breaks neither, for one extra path and one extra fstab line.
|
||||
#
|
||||
# The split from the OS rootfs is still for RESILIENCE: an isolated rootfs stays bootable +
|
||||
# agent-recoverable if the data volume fills (the controller's prevention layer, and since
|
||||
# controller v0.192.0 the capture floor, keep it from filling). Size is env-overridable
|
||||
# (OS_SIZE_GB / GOLDEN_VOLUME_GB); provision GROWS the one volume (bringup.go DataVolGrowGB).
|
||||
# backup=1 is MANDATORY: without it vzdump EXCLUDES the volume (extra LXC mountpoints default
|
||||
# backup=0 — storage-split B3), so the archive would carry no images AND no user data.
|
||||
set -euo pipefail
|
||||
|
||||
# Script provenance — logged into every bake transcript next to the baked controller tag, so an
|
||||
# archive can always be traced to the script that produced it. Bump on any behavior change.
|
||||
GOLDEN_SCRIPT_VERSION="2.0.0"
|
||||
GOLDEN_SCRIPT_VERSION="3.0.0"
|
||||
|
||||
VMID="${1:-9100}"
|
||||
TEMPLATE="${2:-local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst}"
|
||||
@@ -59,25 +81,28 @@ if [ -z "$CONTROLLER_IMAGE" ]; then
|
||||
exit 1
|
||||
fi
|
||||
REGISTRY_HOST="${CONTROLLER_IMAGE%%/*}"
|
||||
# OS rootfs size (GiB) and the golden's Docker-data volume size (GiB). Keep GOLDEN_DOCKER_GB just
|
||||
# large enough for the baked images + headroom; provision grows it to the per-customer target.
|
||||
# OS rootfs size (GiB) and the golden's SINGLE data volume size (GiB).
|
||||
#
|
||||
# ONE VOLUME MEANS ONE NUMBER (v3.0.0). The retired GOLDEN_SYSDATA_GB has no successor: there is
|
||||
# nothing left to size separately. Keep GOLDEN_VOLUME_GB just large enough for the baked images plus
|
||||
# headroom for the controller's felhom-data skeleton; provision grows the one volume to the
|
||||
# per-customer target (bringup.go DataVolGrowGB).
|
||||
OS_SIZE_GB="${OS_SIZE_GB:-32}"
|
||||
GOLDEN_DOCKER_GB="${GOLDEN_DOCKER_GB:-16}"
|
||||
# The golden's SSD user-data volume (GiB) mounted at /mnt/sys_drive (mp1, backup=1) — the controller's
|
||||
# system_data_path. Ships small + near-empty (the controller creates <sys_drive>/felhom-data itself once
|
||||
# it's a real mountpoint); provision GROWS it to the per-customer target (bringup.go SysDataGrowGB). Like
|
||||
# mp0, backup=1 is MANDATORY: without it vzdump EXCLUDES the volume (extra mountpoints default backup=0 —
|
||||
# storage-split B3) and the user-data area would silently fall out of PBS coverage.
|
||||
GOLDEN_SYSDATA_GB="${GOLDEN_SYSDATA_GB:-8}"
|
||||
# 24 = the retired pair's 16 (docker) + 8 (user-data), so a golden archive carries the same content it
|
||||
# did before the merge. It is deliberately NOT a per-customer size: provision grows it.
|
||||
GOLDEN_VOLUME_GB="${GOLDEN_VOLUME_GB:-24}"
|
||||
# The neutral mount path of the single volume. Both consumer paths are binds of subdirectories of it.
|
||||
GOLDEN_VOLUME_MP="/var/lib/felhom"
|
||||
|
||||
echo "[golden] build-golden.sh v${GOLDEN_SCRIPT_VERSION} — baking controller ${CONTROLLER_IMAGE}"
|
||||
echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged; rootfs ${OS_SIZE_GB}G + Docker-data ${GOLDEN_DOCKER_GB}G @ /var/lib/docker + user-data ${GOLDEN_SYSDATA_GB}G @ /mnt/sys_drive, both backup=1) …"
|
||||
echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged; rootfs ${OS_SIZE_GB}G + ONE data volume ${GOLDEN_VOLUME_GB}G @ ${GOLDEN_VOLUME_MP}, backup=1) …"
|
||||
# ONE mpN slot. There is deliberately no mp1: that slot held the retired user-data volume, and the
|
||||
# whole point of R-165 is that it stops existing rather than being made bigger.
|
||||
pct create "$VMID" "$TEMPLATE" \
|
||||
--hostname felhom-golden --unprivileged 1 \
|
||||
--features nesting=1,keyctl=1 \
|
||||
--rootfs "${ROOTFS_STORAGE}:${OS_SIZE_GB}" --cores 2 --memory 2048 \
|
||||
--mp0 "${ROOTFS_STORAGE}:${GOLDEN_DOCKER_GB},mp=/var/lib/docker,backup=1" \
|
||||
--mp1 "${ROOTFS_STORAGE}:${GOLDEN_SYSDATA_GB},mp=/mnt/sys_drive,backup=1" \
|
||||
--mp0 "${ROOTFS_STORAGE}:${GOLDEN_VOLUME_GB},mp=${GOLDEN_VOLUME_MP},backup=1" \
|
||||
--net0 "name=eth0,bridge=${BRIDGE},ip=dhcp" --onboot 0
|
||||
|
||||
echo "[golden] starting + installing Docker (official repo, trixie channel) …"
|
||||
@@ -106,8 +131,10 @@ echo "[golden] baking daemon.json: classic overlay2 driver (containerd-snapshott
|
||||
# The classic overlay2 driver stores EVERYTHING (images + overlay + volumes) under data-root
|
||||
# (/var/lib/docker) = the data volume, which is exactly what "one data-root = one partition for all
|
||||
# images + overlay" requires. It also makes the controller's statfs("/") (its overlay root) report the
|
||||
# DATA volume, which the prevention layer depends on. /var/lib/docker is the mp0 mount (mounted empty
|
||||
# before docker installs), so data-root needs no override. Log caps kill the most common runaway.
|
||||
# DATA volume, which the prevention layer depends on — MEASURED to still hold under the v3.0.0 merged
|
||||
# layout (a container's `df /` reports the single volume, phase-0 spike). Since v3.0.0 /var/lib/docker
|
||||
# is a BIND of <volume>/docker rather than the mp0 mount itself, wired immediately below; data-root
|
||||
# still needs no override because the path is unchanged. Log caps kill the most common runaway.
|
||||
pct exec "$VMID" -- bash -c 'mkdir -p /etc/docker; cat > /etc/docker/daemon.json <<JSON
|
||||
{
|
||||
"features": { "containerd-snapshotter": false },
|
||||
@@ -115,6 +142,32 @@ pct exec "$VMID" -- bash -c 'mkdir -p /etc/docker; cat > /etc/docker/daemon.json
|
||||
"log-opts": { "max-size": "10m", "max-file": "3" }
|
||||
}
|
||||
JSON'
|
||||
echo "[golden] wiring the single data volume (R-165 variant V-c): ${GOLDEN_VOLUME_MP}/{docker,sys_drive} -> binds …"
|
||||
# docker-ce has already populated /var/lib/docker ON THE ROOTFS by now (it auto-starts on install), so
|
||||
# the content is MOVED onto the volume before the bind is laid over the top. Doing it the other way
|
||||
# round would hide those files under the bind and silently ship a golden whose baked images are on the
|
||||
# rootfs — the exact failure class the assertions below exist to catch.
|
||||
#
|
||||
# /etc/fstab, not a hand-run mount: systemd's fstab generator orders both binds under local-fs.target,
|
||||
# which precedes basic.target and therefore docker.service. MEASURED across 3 reboots per variant in
|
||||
# the phase-0 spike — the ordering worry that motivated the probe did not materialise.
|
||||
pct exec "$VMID" -- bash -c "
|
||||
set -e
|
||||
systemctl stop docker docker.socket containerd 2>/dev/null || true
|
||||
mkdir -p '${GOLDEN_VOLUME_MP}/docker' '${GOLDEN_VOLUME_MP}/sys_drive'
|
||||
if [ -d /var/lib/docker ] && [ -n \"\$(ls -A /var/lib/docker 2>/dev/null)\" ]; then
|
||||
cp -a /var/lib/docker/. '${GOLDEN_VOLUME_MP}/docker'/
|
||||
rm -rf /var/lib/docker/*
|
||||
fi
|
||||
mkdir -p /var/lib/docker /mnt/sys_drive
|
||||
printf '%s /var/lib/docker none bind 0 0\n' '${GOLDEN_VOLUME_MP}/docker' >> /etc/fstab
|
||||
printf '%s /mnt/sys_drive none bind 0 0\n' '${GOLDEN_VOLUME_MP}/sys_drive' >> /etc/fstab
|
||||
systemctl daemon-reload
|
||||
mount /var/lib/docker
|
||||
mount /mnt/sys_drive
|
||||
systemctl start containerd
|
||||
"
|
||||
|
||||
echo "[golden] verifying Docker works in the build guest (storage driver should be overlay2 on the ext4 data volume) …"
|
||||
# RESTART (not start): docker-ce auto-starts on install with the DEFAULT config, so it is already
|
||||
# running by now; only a restart picks up the daemon.json just written (overlay2 + log caps).
|
||||
@@ -122,12 +175,20 @@ pct exec "$VMID" -- bash -c 'systemctl restart docker; sleep 3; docker run --rm
|
||||
# Guard: the image store MUST be on the data volume now. /var/lib/containerd holding the images would
|
||||
# mean containerd-snapshotter is still on (the split would leave images on the rootfs).
|
||||
pct exec "$VMID" -- bash -c 'drv=$(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"); [ "$drv" = "overlay2" ] || { echo "[golden] FATAL: storage driver is $drv, expected overlay2 — images would not land on the data volume"; exit 1; }'
|
||||
# Confirm /var/lib/docker is genuinely the dedicated volume, not the rootfs (catch a silent mp miss).
|
||||
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /var/lib/docker | grep -q . && echo " /var/lib/docker is a separate mount: $(findmnt -no SOURCE,FSTYPE /var/lib/docker)" || { echo "[golden] FATAL: /var/lib/docker is NOT a separate mount — the mp0 split did not take"; exit 1; }'
|
||||
# Same guard for the SSD user-data volume (mp1): /mnt/sys_drive must be its own mount, not the rootfs
|
||||
# device — otherwise the controller's system_data_path lands on the OS drive and it warns (the whole
|
||||
# point of this volume is to clear that warning).
|
||||
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /mnt/sys_drive | grep -q . && echo " /mnt/sys_drive is a separate mount: $(findmnt -no SOURCE,FSTYPE /mnt/sys_drive)" || { echo "[golden] FATAL: /mnt/sys_drive is NOT a separate mount — the mp1 split did not take"; exit 1; }'
|
||||
# ASSERTION 1 (RETARGETED v3.0.0, not removed). /var/lib/docker must be a real mount — now the V-c
|
||||
# bind of <volume>/docker rather than the mp0 mount itself. Still fails closed on the same failure:
|
||||
# if the bind did not take, Docker's data-root silently sits on the OS rootfs and the golden ships
|
||||
# its baked images there.
|
||||
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /var/lib/docker | grep -q . && echo " /var/lib/docker is a real mount: $(findmnt -no SOURCE,FSTYPE /var/lib/docker | head -1)" || { echo "[golden] FATAL: /var/lib/docker is NOT a mount — the V-c docker bind did not take, so the baked images would land on the OS rootfs"; exit 1; }'
|
||||
# ASSERTION 2 (RETARGETED v3.0.0). /mnt/sys_drive must be a real mount — now the V-c bind of
|
||||
# <volume>/sys_drive. Otherwise the controller's system_data_path lands on the OS drive and it warns
|
||||
# (clearing that warning is the whole point of the volume).
|
||||
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /mnt/sys_drive | grep -q . && echo " /mnt/sys_drive is a real mount: $(findmnt -no SOURCE,FSTYPE /mnt/sys_drive | head -1)" || { echo "[golden] FATAL: /mnt/sys_drive is NOT a mount — the V-c sys_drive bind did not take, so the controller system_data_path would be the OS rootfs"; exit 1; }'
|
||||
# ASSERTION 2b (NEW v3.0.0 — the invariant the merge is FOR). Both paths must be backed by the SAME
|
||||
# device, i.e. ONE filesystem with ONE free-space figure. Two devices here is the S2 shape the R-165
|
||||
# spike ranked strictly WORSE than the split it replaced: every assertion satisfied, the ceiling still
|
||||
# there, and a shared pool neither `df` can see coming.
|
||||
pct exec "$VMID" -- bash -c 'n=$(df --output=source /var/lib/docker /mnt/sys_drive | tail -n +2 | sort -u | wc -l); [ "$n" = "1" ] && echo " both paths are ONE filesystem: $(df --output=source,avail /var/lib/docker | tail -1)" || { echo "[golden] FATAL: /var/lib/docker and /mnt/sys_drive are on $n DIFFERENT filesystems — that is the S2 shape (two ceilings), not the R-165 merge"; exit 1; }'
|
||||
|
||||
echo "[golden] baking the in-guest controller image $CONTROLLER_IMAGE (no registry cred at deploy) …"
|
||||
# docker login is used ONCE here on the trusted build host, then logged out before archiving so
|
||||
@@ -136,29 +197,52 @@ if [ -n "${REGISTRY_USER:-}" ] && [ -n "${REGISTRY_TOKEN:-}" ]; then
|
||||
pct exec "$VMID" -- bash -c "systemctl start docker; sleep 1; echo '$REGISTRY_TOKEN' | docker login '$REGISTRY_HOST' -u '$REGISTRY_USER' --password-stdin >/dev/null"
|
||||
fi
|
||||
pct exec "$VMID" -- bash -c "docker pull '$CONTROLLER_IMAGE'"
|
||||
pct exec "$VMID" -- bash -c "docker logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; rm -f /root/.docker/config.json"
|
||||
# Record the baked image ref for the bootstrap unit (so the unit needs no login/pull).
|
||||
pct exec "$VMID" -- bash -c "printf '%s\n' '$CONTROLLER_IMAGE' > /etc/felhom-controller-image"
|
||||
|
||||
# Bake the base-infrastructure images (traefik, cloudflared, filebrowser) so the controller's
|
||||
# first-boot bring-up (EnsureBaseStack) is OFFLINE-capable — no registry pull at deploy. These are
|
||||
# PUBLIC Docker Hub images (no cred needed). The PINNED tags MUST match the controller's
|
||||
# internal/infra constants (TraefikImage / CloudflaredImage / FileBrowserImage); a drift means the
|
||||
# golden bakes one image and the controller requests another (→ a pull at deploy, defeating the goal).
|
||||
INFRA_IMAGES=(
|
||||
"traefik:v3.6.7"
|
||||
"cloudflare/cloudflared:2026.6.0"
|
||||
"gtstef/filebrowser:1.3.3-stable"
|
||||
)
|
||||
echo "[golden] baking base-infra images: ${INFRA_IMAGES[*]} …"
|
||||
# Bake EVERY controller-managed infra image so enabling an infra stack on a fresh box is near-instant
|
||||
# instead of a multi-minute silent registry pull.
|
||||
#
|
||||
# THE LIST COMES FROM THE CONTROLLER WE JUST PULLED, not from a copy here. This script used to carry
|
||||
# a hand-maintained array of three tags with a comment telling the reader to keep it in sync with the
|
||||
# controller's internal/infra constants. It drifted the moment a fourth stack was added: felhom-samba
|
||||
# was never added here, so the golden baked 3 of 4 and turning on Megosztás pulled from the registry
|
||||
# with zero UI feedback — observed live, twice. Asking the binary (`--print-infra-images`, backed by
|
||||
# infra.Images()) makes golden-vs-controller drift structurally impossible: the golden bakes exactly
|
||||
# what THIS controller version will request.
|
||||
echo "[golden] asking the controller which infra images it manages …"
|
||||
INFRA_LIST="$(pct exec "$VMID" -- bash -c "docker run --rm --entrypoint /usr/local/bin/felhom-controller '$CONTROLLER_IMAGE' --print-infra-images 2>/dev/null" || true)"
|
||||
mapfile -t INFRA_IMAGES < <(printf '%s\n' "$INFRA_LIST" | grep -E '^[a-z0-9._/-]+:[A-Za-z0-9._-]+$' || true)
|
||||
if [ "${#INFRA_IMAGES[@]}" -eq 0 ]; then
|
||||
# Controllers older than v0.147.0 have no --print-infra-images. Fall back to the historical list so
|
||||
# an old-controller bake still works — but say so loudly, because the fallback is exactly the
|
||||
# drift-prone thing this change removed, and it CANNOT know about stacks added after it was written.
|
||||
echo "[golden] WARN: '$CONTROLLER_IMAGE' does not support --print-infra-images (pre-0.147.0)."
|
||||
echo "[golden] WARN: falling back to the historical 3-image list. felhom-samba will NOT be baked,"
|
||||
echo "[golden] WARN: so enabling Megosztás on this golden will pull at runtime. Bake >=0.147.0."
|
||||
INFRA_IMAGES=(
|
||||
"traefik:v3.6.7"
|
||||
"cloudflare/cloudflared:2026.6.0"
|
||||
"gtstef/filebrowser:1.3.3-stable"
|
||||
)
|
||||
fi
|
||||
echo "[golden] baking infra images (${#INFRA_IMAGES[@]}): ${INFRA_IMAGES[*]} …"
|
||||
for img in "${INFRA_IMAGES[@]}"; do
|
||||
# Hard gate: fail loudly BEFORE pulling if a pinned tag doesn't resolve (a bad pin otherwise fails
|
||||
# mid-bake with a confusing error).
|
||||
pct exec "$VMID" -- bash -c "docker manifest inspect '$img' >/dev/null 2>&1" \
|
||||
|| { echo "[golden] FATAL: pinned base-infra image does not resolve: $img"; exit 1; }
|
||||
|| { echo "[golden] FATAL: pinned infra image does not resolve: $img"; exit 1; }
|
||||
pct exec "$VMID" -- bash -c "docker pull '$img'"
|
||||
done
|
||||
|
||||
# Log out LAST — not right after the controller pull. felhom-samba lives on the same private registry
|
||||
# as the controller, so the infra loop above needs the credential; logging out first made that pull
|
||||
# 401. The credential is still never archived: the logout + config.json removal happen before the
|
||||
# guest is stopped and templated.
|
||||
pct exec "$VMID" -- bash -c "docker logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; rm -f /root/.docker/config.json"
|
||||
pct exec "$VMID" -- bash -c '[ ! -s /root/.docker/config.json ]' \
|
||||
|| { echo "[golden] FATAL: registry credential still present in the guest — refusing to archive"; exit 1; }
|
||||
|
||||
echo "[golden] baking the controller-bootstrap unit (deploys the BAKED controller from the config mount) …"
|
||||
pct push "$VMID" /dev/stdin /usr/local/sbin/felhom-controller-bootstrap.sh --perms 700 <<'BOOTSH'
|
||||
#!/bin/bash
|
||||
@@ -284,26 +368,36 @@ pct exec "$VMID" -- bash -c '
|
||||
|
||||
echo "[golden] stop + archive …"
|
||||
pct stop "$VMID"
|
||||
# --mode stop with mp0 + mp1 backup=1 → BOTH the Docker-data volume (baked images) and the
|
||||
# /mnt/sys_drive user-data volume are INCLUDED. The log below MUST show "including mount point mp0"
|
||||
# AND "including mount point mp1" — if either shows "excluding … (disabled)" the backup flag was lost
|
||||
# and the archive carries no images / no user-data volume (storage-split B3 trap).
|
||||
# --mode stop with mp0 backup=1 → the SINGLE data volume (baked images AND the user-data area) is
|
||||
# INCLUDED. The log MUST show "including mount point mp0" and must NOT show it being excluded — an
|
||||
# exclusion means the backup flag was lost and the archive carries neither (storage-split B3 trap).
|
||||
# Since v3.0.0 there is no mp1; the guard that covered it is retargeted below rather than deleted,
|
||||
# because a guard whose pattern can no longer match is a guard that has silently stopped guarding.
|
||||
vzdump "$VMID" --storage "$ARCHIVE_STORAGE" --mode stop --compress zstd 2>&1 | tee /tmp/golden-vzdump.log | grep -iE "including mount point|excluding|archive file size|Finished Backup" || true
|
||||
if grep -q "excluding volume mount point mp0" /tmp/golden-vzdump.log; then
|
||||
echo "[golden] FATAL: mp0 (/var/lib/docker) was EXCLUDED from the archive — backup=1 was lost; the golden would carry no images. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
if grep -q "excluding volume mount point mp1" /tmp/golden-vzdump.log; then
|
||||
echo "[golden] FATAL: mp1 (/mnt/sys_drive) was EXCLUDED from the archive — backup=1 was lost; the golden would carry no user-data volume. Aborting."
|
||||
# ASSERTION 4 (RETARGETED v3.0.0). The mp1 guard used to catch "the user-data volume fell out of the
|
||||
# archive". After the merge there is no mp1 — so the same failure now looks like the volume being
|
||||
# mounted at the WRONG PATH, which would carry the images but not the user-data area. Assert the
|
||||
# inclusion line names the volume's actual mount path.
|
||||
if ! grep -q "including mount point mp0 ('${GOLDEN_VOLUME_MP}')" /tmp/golden-vzdump.log; then
|
||||
echo "[golden] FATAL: the archive's mp0 is not ${GOLDEN_VOLUME_MP} — the single data volume is mounted somewhere unexpected, so the archive would not carry both the baked images and the user-data area. Aborting."
|
||||
grep -iE "mount point" /tmp/golden-vzdump.log || true
|
||||
exit 1
|
||||
fi
|
||||
# ASSERTION 5 (RETARGETED v3.0.0). There must be NO mp1 in the archive at all. A leftover second
|
||||
# volume means the merge did not take and this golden would ship the very ceiling R-165 removed.
|
||||
if grep -qE "mount point mp1" /tmp/golden-vzdump.log; then
|
||||
echo "[golden] FATAL: the archive still carries an mp1 — the R-165 merge did not take and this golden would ship a second, ceilinged volume. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
grep -q "including mount point mp0" /tmp/golden-vzdump.log \
|
||||
|| echo "[golden] WARN: could not confirm mp0 inclusion in the vzdump log — verify manually before using this archive."
|
||||
grep -q "including mount point mp1" /tmp/golden-vzdump.log \
|
||||
|| echo "[golden] WARN: could not confirm mp1 inclusion in the vzdump log — verify manually before using this archive."
|
||||
|
||||
VOLID=$(pvesm list "$ARCHIVE_STORAGE" --content backup 2>/dev/null | awk -v v="$VMID" '$1 ~ ("vzdump-lxc-" v "-") {print $1}' | sort | tail -1)
|
||||
echo "[golden] DONE. golden archive volid: ${VOLID:-<check ${ARCHIVE_STORAGE} dump dir>} (rootfs ${OS_SIZE_GB}G + Docker-data ${GOLDEN_DOCKER_GB}G + user-data ${GOLDEN_SYSDATA_GB}G, all in the archive)"
|
||||
echo "[golden] DONE. golden archive volid: ${VOLID:-<check ${ARCHIVE_STORAGE} dump dir>} (rootfs ${OS_SIZE_GB}G + ONE data volume ${GOLDEN_VOLUME_GB}G @ ${GOLDEN_VOLUME_MP}, all in the archive)"
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Publish to Gitea (BUNDLE slice) — make this golden fetchable by the host-bootstrap script.
|
||||
|
||||
@@ -133,6 +133,28 @@ Cmnd_Alias FELHOM_CONTROLLERSWAP = \
|
||||
Cmnd_Alias FELHOM_STALELOCK = \
|
||||
/usr/sbin/pct unlock [0-9]*
|
||||
|
||||
# Restore-test scratch teardown (F-LEAK, Campaign 8, v0.110.0). A restore-test whose restore FAILS
|
||||
# leaves a scratch guest the API token CANNOT destroy: `FelhomAgentGuest` is granted at /pool/felhom and
|
||||
# a guest joins that pool only when its restore COMPLETES, so a failed restore leaves a pool-less guest
|
||||
# out of reach (403 VM.Allocate) holding its disks until a human removes it.
|
||||
#
|
||||
# TWO API-SIDE FIXES WERE TRIED AND BOTH REFUTED LIVE on 2026-07-28, which is why this grant exists:
|
||||
# 1. Adopt the stranded guest into the pool, then retry. `PUT /pools/{pool}` ALSO requires
|
||||
# VM.Allocate on the VM being added — pool membership cannot bootstrap its own authority.
|
||||
# 2. Grant FelhomAgentGuest per-path at /vms/990000..990009. Durable for exactly one use per slot:
|
||||
# PVE's own destroy path calls `AccessControl::remove_vm_access($vmid)` (LXC.pm:906), which DELETES
|
||||
# every ACL at /vms/<vmid> (AccessControl.pm:1898). The grant is consumed by the operation it
|
||||
# authorises, so after ten teardowns the band is ungranted and the defect returns.
|
||||
#
|
||||
# WHY THIS IS THE TIGHTEST AVAILABLE FENCE, not a widening: sudo matches the vmid LITERALLY, so
|
||||
# `99000[0-9]` is exactly the ten-slot scratch band the restore-test picks from — nothing else. There is
|
||||
# no `[0-9]*` coarse allowlist here on purpose: unlike `pct unlock`, this op DESTROYS, so the band must
|
||||
# be in the policy and not merely validated in the agent. Even a compromised agent asking for
|
||||
# `pct destroy 9201` is refused by sudo itself. Unlike an ACL, a sudoers rule is not consumed by use.
|
||||
# The agent re-checks the band in code before exec (defence in depth); this is the outer fence.
|
||||
Cmnd_Alias FELHOM_SCRATCH_TEARDOWN = \
|
||||
/usr/sbin/pct destroy 99000[0-9] --purge
|
||||
|
||||
# Network storage / NAS (Part A1, SPIKE-nas-storage-2026-06-29). The agent mounts a customer NAS share
|
||||
# HOST-SIDE under /mnt/felhom-drives/<name> via a systemd .automount (+ .mount) pair so it propagates
|
||||
# into the guest through the existing shared bind (an unprivileged LXC cannot mount NFS/CIFS itself).
|
||||
@@ -143,11 +165,23 @@ Cmnd_Alias FELHOM_STALELOCK = \
|
||||
# `.automount` variants + the unit-file removal. The unit FILE name is the systemd-escaped mountpoint,
|
||||
# which always begins `mnt-felhom` (the mountpoint is /mnt/felhom-drives/<name>), so the rm glob is scoped
|
||||
# to felhom mount units only. mkdir of the mountpoint reuses FELHOM_INTERMEDIARY's /mnt/felhom-drives/*.
|
||||
# CAMPAIGN-3 additions (loud, per the no-widening rule):
|
||||
# - `systemctl reset-failed -- mnt-felhom*`: F10 (CRITICAL) — a NAS automount that hit
|
||||
# mount-start-limit-hit during an outage was re-armable by NO platform path; the reassert now
|
||||
# reset-failed's the stuck unit before `enable --now` (which the start-limit otherwise refuses),
|
||||
# and RemoveNetworkMount clears failed-state residue (F2). Scoped to felhom mount units (the unit
|
||||
# name is the systemd-escaped mountpoint, always beginning `mnt-felhom`). reset-failed only clears
|
||||
# a unit's failed latch — it cannot start/stop/alter anything.
|
||||
# - `rmdir /mnt/felhom-drives/*`: F1 — remove the now-empty mountpoint dir a removed share leaves
|
||||
# behind (the campaign accumulated 10 stub-shaped leftovers). rmdir ONLY (never rm -rf): it refuses
|
||||
# a non-empty dir, so unexpected data is preserved, not destroyed — a fail-safe grant.
|
||||
Cmnd_Alias FELHOM_NETMOUNT = \
|
||||
/usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/units/* /etc/systemd/system/*.automount, \
|
||||
/usr/bin/systemctl enable --now -- *.automount, \
|
||||
/usr/bin/systemctl disable -- *.automount, \
|
||||
/usr/bin/systemctl stop -- *.automount, \
|
||||
/usr/bin/systemctl reset-failed -- mnt-felhom*, \
|
||||
/usr/bin/rmdir /mnt/felhom-drives/*, \
|
||||
/usr/bin/rm -f /etc/systemd/system/mnt-felhom*
|
||||
|
||||
# Offsite WG tunnel (S3, doc 06 §3.3). The agent manages wg-quick@wg-felhom as an agent-managed
|
||||
@@ -178,4 +212,97 @@ Cmnd_Alias FELHOM_SELFUPDATE = \
|
||||
/usr/local/sbin/felhom-selfupdate-guarded commit, \
|
||||
/usr/local/sbin/felhom-selfupdate-guarded rollback
|
||||
|
||||
felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE
|
||||
# Dedicated OOB sshd (TASK H1). The agent manages felhom-sshd like wg-felhom/dnsmasq: it RENDERS the
|
||||
# config (Port from its claim) + the operator's authorized_keys, validates with `sshd -t`, and reloads
|
||||
# (never restart-on-change [SF-2]). Both install SOURCES are the agent-owned staged files under
|
||||
# StateDir; both DESTINATIONS are FIXED. `sshd -t/-T` are the validate/discover reads. The
|
||||
# systemctl verbs are SCOPED to felhom-sshd only. reset-failed precedes a deliberate restart [SF-5].
|
||||
# NOTHING here can touch the stock sshd, :22, or /etc/ssh.
|
||||
Cmnd_Alias FELHOM_SSHD = \
|
||||
/usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/felhom-sshd/sshd_config /etc/felhom-sshd/sshd_config, \
|
||||
/usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/felhom-sshd/authorized_keys.felhom-op /etc/felhom-sshd/authorized_keys/felhom-op, \
|
||||
/usr/sbin/sshd -t -f /var/lib/felhom-agent/felhom-sshd/sshd_config, \
|
||||
/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config, \
|
||||
/usr/sbin/sshd -T -f /etc/felhom-sshd/sshd_config, \
|
||||
/usr/bin/systemctl enable --now felhom-sshd, \
|
||||
/usr/bin/systemctl reload felhom-sshd, \
|
||||
/usr/bin/systemctl restart felhom-sshd, \
|
||||
/usr/bin/systemctl reset-failed felhom-sshd, \
|
||||
/usr/bin/wg show wg-felhom latest-handshakes
|
||||
|
||||
# PBS DR tier apply (slice 2, SPIKE-pbs-tier-provisioning-2026-07-10 §2b). Storage-entry
|
||||
# lifecycle is /storage-ROOT-gated in the PVE API (spike Probe 1: create/modify/delete all check
|
||||
# Datastore.Allocate on /storage), so the agent token cannot do it — this wrapper is the pinned
|
||||
# vector. THE SET-ONLY LAW: the wrapper contains NO deletion path (entry deletion destroys the
|
||||
# client encryption key = un-decryptable backups); verbs are create/reconcile/grant only. The
|
||||
# token secret rides the wrapper's STDIN — sudo logs argv, so it must never appear here. The
|
||||
# agent fine-validates every field (charset + descriptor equality) before exec; these globs are
|
||||
# the coarse allowlist.
|
||||
#
|
||||
# `read` (R-39 leg b, agent v0.91.0) is the ONE added verb. It prints a token secret to stdout and
|
||||
# performs no mutation. It exists because the agent writes that file through this wrapper but could
|
||||
# never read it back (/etc/pve/priv is 0700 root:www-data), leaving its PBS verify loop permanently
|
||||
# blind to an `applied`-but-401 tier. It is NOT a general file-read: the wrapper pins the directory
|
||||
# and prefix-asserts the resolved path, and the id grammar admits no slash. The secret goes to
|
||||
# STDOUT, never argv — sudo logs argv.
|
||||
# E-2a: the backup-target storage shim. Creating a PVE storage needs Datastore.Allocate at /storage
|
||||
# and the grant needs Permissions.Modify -- the agent holds NEITHER by design (blast-radius
|
||||
# containment; Permissions.Modify would let it rewrite its own authority). Both live behind this
|
||||
# fixed-vocabulary root shim instead, exactly like the mkfs and pbs-apply wrappers. The wrapper has
|
||||
# NO storage-removal path, enforces is_mountpoint 1, and refuses a target on the root device.
|
||||
Cmnd_Alias FELHOM_BACKUPTARGET = \
|
||||
/usr/local/sbin/felhom-backup-target-apply create *, \
|
||||
/usr/local/sbin/felhom-backup-target-apply grant *
|
||||
|
||||
Cmnd_Alias FELHOM_PBSDR = \
|
||||
/usr/local/sbin/felhom-pbs-apply create *, \
|
||||
/usr/local/sbin/felhom-pbs-apply reconcile *, \
|
||||
/usr/local/sbin/felhom-pbs-apply grant *, \
|
||||
/usr/local/sbin/felhom-pbs-apply read *
|
||||
|
||||
# OOB nft belt (TASK H1). The STATIC table `inet felhom_oob` is installed once by host-install; the
|
||||
# agent mutates ONLY its two SETS — @operator_ips (the operator /32) + @ssh_port (the claimed port).
|
||||
# SET ELEMENTS ONLY [trap 4]: NO `nft add rule`, NO `nft -f`, NO `flush ruleset/table` — a rule grant
|
||||
# would let the agent firewall anything. The agent fine-validates every element (netip / int range)
|
||||
# before exec; the trailing wildcards are the coarse allowlist (values only).
|
||||
Cmnd_Alias FELHOM_OOB = \
|
||||
/usr/sbin/nft list set inet felhom_oob operator_ips, \
|
||||
/usr/sbin/nft list set inet felhom_oob ssh_port, \
|
||||
/usr/sbin/nft flush set inet felhom_oob operator_ips, \
|
||||
/usr/sbin/nft flush set inet felhom_oob ssh_port, \
|
||||
/usr/sbin/nft add element inet felhom_oob operator_ips *, \
|
||||
/usr/sbin/nft add element inet felhom_oob ssh_port *
|
||||
|
||||
# Escrow ceremony (controller-driven, TASK 2026-07-13; mechanics validated by
|
||||
# SPIKE-controller-escrow-2026-07-13). ONE fixed argv — sudoers matches the argument vector
|
||||
# byte-for-byte (spike §2.2): any alteration (value, extra flag, order, config path) is refused.
|
||||
# --config pinned: env_reset strips FELHOM_AGENT_CONFIG and the pin closes alternate-config
|
||||
# injection. R rides the subprocess stdout pipe only; sudo logs argv = secrets-free. The argv
|
||||
# MUST stay byte-identical to escrow.CeremonyArgs() (internal/escrow/ceremony.go) — the
|
||||
# capability manifest entry + TestManifestCoveredBySudoers lock the three copies together.
|
||||
Cmnd_Alias FELHOM_ESCROW = \
|
||||
/usr/local/bin/felhom-agent --config /etc/felhom-agent/agent.json --selftest=escrow-create --upload --output=json
|
||||
|
||||
# Node self-heal (CAMPAIGN-3 Part 6, F12-class defense in depth). The ONE fixed unit the appliance
|
||||
# watchdog may (re)start when a boot leaves networking down — the exact command the morning recovery
|
||||
# ran by hand after the F12 host loss. FIXED unit, no glob: this grant alone cannot harm — starting
|
||||
# networking.service is precisely what the boot should have done. The remedy is ALSO code-gated on
|
||||
# deployment_mode="appliance" (the Manager refuses to invoke it on a byo host); the sudoers grant is
|
||||
# the coarse floor, the mode gate is the fine one.
|
||||
Cmnd_Alias FELHOM_SELFHEAL = \
|
||||
/usr/bin/systemctl start networking.service
|
||||
|
||||
# Guest-network watchdog (internal/guestnet, R-54). The guest's DHCP client is unsupervised — when it
|
||||
# died on 2026-07-20 the box lost its address ~80 minutes later and went off the internet for 1h15m
|
||||
# (INCIDENT-guest-dhclient-killed-2026-07-20). Four FIXED read vectors plus ONE fixed heal vector; the
|
||||
# heal is the incident's own restored invocation, byte for byte. This is NOT a general `pct exec`: every
|
||||
# argument after the numeric vmid is a literal, so the grant cannot be widened by anything the guest or
|
||||
# the hub says. The address read is deliberately NOT duplicated here — it is already FELHOM_DNSMASQ's,
|
||||
# and the same command must not be granted twice under two names.
|
||||
Cmnd_Alias FELHOM_GUESTNET = \
|
||||
/usr/sbin/pct exec [0-9]* -- ip route show default, \
|
||||
/usr/sbin/pct exec [0-9]* -- cat /etc/network/interfaces, \
|
||||
/usr/sbin/pct exec [0-9]* -- pgrep -x dhclient, \
|
||||
/usr/sbin/pct exec [0-9]* -- dhclient -pf /run/dhclient.eth0.pid -lf /var/lib/dhcp/dhclient.eth0.leases eth0
|
||||
|
||||
felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_BACKUPTARGET, FELHOM_SELFHEAL, FELHOM_ESCROW, FELHOM_GUESTNET, FELHOM_SCRATCH_TEARDOWN
|
||||
|
||||
Executable
+111
@@ -0,0 +1,111 @@
|
||||
#!/bin/bash
|
||||
#===============================================================================
|
||||
# felhom-backup-target-apply — the ONLY path the felhom-agent sudoers permits for creating the
|
||||
# whole-guest backup TARGET storage and granting the agent access to it (E-2a).
|
||||
#
|
||||
# WHY A WRAPPER AT ALL. Creating a PVE storage needs `Datastore.Allocate` at `/storage`, and the ACL
|
||||
# grant needs `Permissions.Modify`. The agent holds NEITHER by design — its token is scoped per
|
||||
# storage path for blast-radius containment, and `Permissions.Modify` would let it rewrite its own
|
||||
# authority. Widening the PVE role to make the move possible would trade the entire containment model
|
||||
# for one feature. So the privileged half lives here: a minimal, auditable root shim with a fixed
|
||||
# vocabulary, exactly like felhom-mkfs-guarded and felhom-pbs-apply.
|
||||
#
|
||||
# THE NO-DELETE LAW (inherited from felhom-pbs-apply, same reasoning class). This wrapper contains NO
|
||||
# storage-removal path of any kind. `pvesm remove` on a dir storage does not delete the archives, but
|
||||
# it DOES silently orphan a configured backup tier, and a "cleanup" verb here would be reachable by
|
||||
# any bug in the agent. Retiring a target is a deliberate operator op, not this tool. Grep-assertable;
|
||||
# do not add one.
|
||||
#
|
||||
# THE TWO LAWS E-1 PAID FOR ON LIVE HARDWARE, both enforced here rather than trusted to the caller:
|
||||
#
|
||||
# F-1 the storage path must BE the drive's own mountpoint. A subdirectory fails the agent's
|
||||
# exactMount check, so the target reports `disconnected` FOREVER and its durable id degrades
|
||||
# off the filesystem UUID. Enforced: `mountpoint -q` must pass on the exact path given.
|
||||
#
|
||||
# F-2 --is_mountpoint 1 is not optional. Without it, an unplugged or late-mounting drive leaves a
|
||||
# bare directory on the ROOT filesystem and vzdump writes the whole-guest backup onto the
|
||||
# system drive — the exact device the whole change exists to escape — while PVE reports the
|
||||
# storage `active` and advertises the root filesystem's free space. Proven live: the unguarded
|
||||
# form had already created dump/ on pve-root. Hardcoded below; not a caller-supplied flag.
|
||||
#
|
||||
# Ops (all non-secret; nothing here touches a credential, so nothing arrives on stdin):
|
||||
# create <id> <mountpoint>
|
||||
# Create a `dir` storage with content=backup at <mountpoint>, is_mountpoint 1.
|
||||
# IDEMPOTENT: an existing entry with the SAME path is accepted (re-run safe, and the
|
||||
# installer re-run path depends on it). An existing entry with a DIFFERENT path is REFUSED
|
||||
# — silently repointing a live backup target is the failure this whole arc closes.
|
||||
# grant <id>
|
||||
# The dual grant: FelhomAgentStore on /storage/<id> to the agent user AND token (privsep
|
||||
# intersection — a token's rights are the intersection, so granting one is granting neither).
|
||||
# Without it every backup 403s on first run (E-1 finding F-3, found by the first real backup).
|
||||
#===============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
die() { echo "felhom-backup-target-apply: REFUSED: $*" >&2; exit 1; }
|
||||
|
||||
op="${1:-}"; id="${2:-}"
|
||||
[[ -n "$op" && -n "$id" ]] || die "usage: felhom-backup-target-apply <create|grant> <storage-id> [mountpoint]"
|
||||
|
||||
# Storage id: PVE grammar, conservative. Also the ACL path component — no slashes possible.
|
||||
[[ "$id" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,27}$ ]] || die "bad storage id ($id)"
|
||||
|
||||
STORECFG=/etc/pve/storage.cfg
|
||||
|
||||
# current_path_of <id> — the configured `path` of dir storage <id>, or "" when absent/not-a-dir.
|
||||
current_path_of() {
|
||||
awk -v want="dir: $1" '
|
||||
$0 == want { found=1; next }
|
||||
found && /^[a-z]+: / { exit }
|
||||
found && $1 == "path" { print $2; exit }
|
||||
' "$STORECFG" 2>/dev/null || true
|
||||
}
|
||||
|
||||
case "$op" in
|
||||
create)
|
||||
[[ $# -eq 3 ]] || die "create takes <id> <mountpoint>"
|
||||
mp="$3"
|
||||
# Absolute, normalized, no traversal, no shell metacharacters. The value reaches pvesm and the
|
||||
# filesystem, so it is validated here rather than assumed well-formed.
|
||||
[[ "$mp" = /* ]] || die "mountpoint must be absolute ($mp)"
|
||||
[[ "$mp" != *".."* ]] || die "mountpoint must not contain .. ($mp)"
|
||||
[[ "$mp" =~ ^[A-Za-z0-9/_.-]+$ ]] || die "mountpoint has unexpected characters ($mp)"
|
||||
[[ "$mp" != "/" ]] || die "refusing / as a backup target"
|
||||
|
||||
# F-1 + F-2, checked as one: the path must BE a mountpoint right now. A bare directory here is
|
||||
# precisely the silent-retarget shape, and is_mountpoint would make PVE refuse it later anyway —
|
||||
# better to refuse now, with a reason, than to create a storage that can never activate.
|
||||
mountpoint -q "$mp" || die "$mp is not a mountpoint — the backup target must be the drive's OWN mountpoint (F-1), and an unmounted path would silently retarget onto the system drive (F-2)"
|
||||
|
||||
# Never the system disk: a target on the root filesystem is not drive-loss protection, it is the
|
||||
# thing we are escaping. The root device and the candidate's device are compared, not their paths.
|
||||
root_dev="$(findmnt -no SOURCE / 2>/dev/null || true)"
|
||||
mp_dev="$(findmnt -no SOURCE "$mp" 2>/dev/null || true)"
|
||||
[[ -n "$mp_dev" ]] || die "could not resolve the backing device of $mp"
|
||||
[[ "$mp_dev" != "$root_dev" ]] || die "$mp is backed by the ROOT device ($root_dev) — a backup target there protects against corruption only, never drive loss"
|
||||
|
||||
existing="$(current_path_of "$id")"
|
||||
if [[ -n "$existing" ]]; then
|
||||
if [[ "$existing" == "$mp" ]]; then
|
||||
echo "felhom-backup-target-apply: storage $id already exists at $mp — nothing to do (idempotent)" >&2
|
||||
exit 0
|
||||
fi
|
||||
die "storage $id already exists at $existing — refusing to repoint it at $mp (a live backup target is never silently moved)"
|
||||
fi
|
||||
|
||||
# is_mountpoint 1 is HARDCODED (F-2). content=backup only: this storage exists for vzdump archives
|
||||
# and must never become a place guests are allocated on.
|
||||
pvesm add dir "$id" --path "$mp" --content backup --is_mountpoint 1 >&2
|
||||
echo "felhom-backup-target-apply: created dir storage $id at $mp (content=backup, is_mountpoint 1)" >&2
|
||||
;;
|
||||
grant)
|
||||
[[ $# -eq 2 ]] || die "grant takes only <id>"
|
||||
# BOTH, always. A privsep token's rights are the intersection of the user's and the token's ACLs,
|
||||
# so granting one of the two grants nothing usable.
|
||||
pveum acl modify "/storage/$id" --users felhom-agent@pve --roles FelhomAgentStore >&2
|
||||
pveum acl modify "/storage/$id" --tokens 'felhom-agent@pve!agent' --roles FelhomAgentStore >&2
|
||||
echo "felhom-backup-target-apply: granted FelhomAgentStore on /storage/$id (user + token)" >&2
|
||||
;;
|
||||
*)
|
||||
die "unknown op ($op)"
|
||||
;;
|
||||
esac
|
||||
@@ -1,20 +1,30 @@
|
||||
# felhom-agent local API — host firewall narrowing (doc 03 §6, slice 8A)
|
||||
# felhom-agent local API — host firewall narrowing (doc 03 §6; R-50 island update 2026-07-25)
|
||||
#
|
||||
# Defense-in-depth for the per-guest local API (the controller→agent channel on the host
|
||||
# bridge). The PER-GUEST BEARER TOKEN is the authorization gate; this firewall rule is an
|
||||
# ADDITIONAL layer that limits who can even reach the port. The slice-8A spike found no rule
|
||||
# was needed for reachability on the demo (PVE firewall off) — this narrows exposure so that
|
||||
# only guests on the bridge subnet (not arbitrary LAN hosts) can open a connection.
|
||||
# Defense-in-depth for the per-guest local API (the controller→agent channel). The PER-GUEST BEARER
|
||||
# TOKEN + the served-leaf pin are the authorization gate; a firewall rule is only an ADDITIONAL layer
|
||||
# limiting who can even open the port.
|
||||
#
|
||||
# The agent already binds the listener to the host BRIDGE IP (local_api.listen_addr), not
|
||||
# 0.0.0.0. This file adds the subnet restriction. Apply it at HOST SETUP (it is a host-level
|
||||
# packet-filter change, intentionally OUTSIDE the agent's 3-exception privileged fence — the
|
||||
# agent never mutates the host firewall at runtime).
|
||||
# === R-50 ISLAND INSTALL (the default on a fresh appliance) =================================
|
||||
# The agent binds local_api.listen_addr on the HOST-INTERNAL island bridge — 169.254.253.1:8443 on
|
||||
# vmbr9, a bridge with NO physical port (bridge-ports none). That bind is the security win:
|
||||
# * Nothing listens on the LAN IP at all, so no LAN host (or off-site attacker on the LAN) can
|
||||
# reach the local API — the LAN:8443 surface is CLOSED by the bind, not by a rule.
|
||||
# * vmbr9 has no uplink, so 169.254.253.1:8443 is reachable ONLY from the one guest wired to the
|
||||
# /30 (169.254.253.2) — the controller. The portless bridge is the isolation.
|
||||
# So on an island install NO firewall rule is required for exposure; the topology provides it. If you
|
||||
# want belt-and-suspenders, restrict the port to the island bridge (it changes nothing, since nothing
|
||||
# off-bridge can route to a portless bridge anyway):
|
||||
#
|
||||
# Replace the bridge IP (192.168.0.162), port (8443), and the guest bridge subnet
|
||||
# (192.168.0.0/24) with this host's values.
|
||||
# nft add rule inet filter input iifname != "vmbr9" ip daddr 169.254.253.1 tcp dport 8443 drop
|
||||
#
|
||||
# Verify: from the guest, a TLS connect to 169.254.253.1:8443 succeeds; there is no LAN listener to
|
||||
# probe (`ss -lnt 'sport = :8443'` shows only the island IP).
|
||||
#
|
||||
# === LEGACY LAN BIND (byo, --no-island, or an explicit --bridge-ip) =========================
|
||||
# When the agent still binds a LAN bridge IP (e.g. 192.168.0.162:8443), the port is exposed to the
|
||||
# whole LAN and the subnet-narrowing rule below is worth applying. Replace the bridge IP, port, and
|
||||
# the guest bridge subnet with this host's values.
|
||||
#
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Option A — nftables (recommended on PVE 8/9; inet filter table). Insert ABOVE any accept:
|
||||
#
|
||||
# nft add rule inet filter input ip daddr 192.168.0.162 tcp dport 8443 \
|
||||
@@ -22,13 +32,11 @@
|
||||
# nft add rule inet filter input ip daddr 192.168.0.162 tcp dport 8443 \
|
||||
# ip saddr 192.168.0.0/24 accept
|
||||
#
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Option B — iptables:
|
||||
#
|
||||
# iptables -A INPUT -d 192.168.0.162 -p tcp --dport 8443 -s 192.168.0.0/24 -j ACCEPT
|
||||
# iptables -A INPUT -d 192.168.0.162 -p tcp --dport 8443 -j DROP
|
||||
#
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# Option C — PVE host firewall (/etc/pve/nodes/<node>/host.fw), if the PVE firewall is enabled.
|
||||
# Add under [RULES] (and ensure the firewall is enabled in cluster.fw / host.fw):
|
||||
#
|
||||
@@ -36,5 +44,6 @@
|
||||
# IN ACCEPT -source 192.168.0.0/24 -dport 8443 -proto tcp -log nolog
|
||||
# IN DROP -dport 8443 -proto tcp -log nolog
|
||||
#
|
||||
# Verify after applying: from a guest ON the bridge, a TLS connect to <bridge-ip>:8443 succeeds;
|
||||
# from an OFF-bridge host it is refused/dropped. (The token + leaf-pin still gate the request.)
|
||||
# Apply at HOST SETUP — a host-level packet-filter change, intentionally OUTSIDE the agent's
|
||||
# 3-exception privileged fence (the agent never mutates the host firewall at runtime). The token +
|
||||
# leaf-pin still gate the request regardless of which bind is in force.
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# felhom-mgmt-watchdog.service (TASK G1) — install as /etc/systemd/system/felhom-mgmt-watchdog.service.
|
||||
#
|
||||
# LAYER 2 (auto-heal without login): a DUMB oneshot that runs /usr/local/sbin/felhom-mgmt-watchdog,
|
||||
# triggered every ~60s by felhom-mgmt-watchdog.timer. It recreates a missing /run/sshd and clears a
|
||||
# start-limited stock sshd — so a management-plane lockout self-corrects in ≤1 tick with NOBODY logged
|
||||
# in and WITHOUT the felhom-agent binary running (the agent only OBSERVES/reports; the heal must not
|
||||
# depend on it — SPIKE-felhom-sshd-2026-07-05 finding #9 / TASK G1 trap 1).
|
||||
#
|
||||
# CRITICAL: this unit MUST NOT declare `RuntimeDirectory=` — that directive (with value `sshd`) is the
|
||||
# incident it exists to fix. It writes only to /run (dir + heal marker) as root; no state dir needed.
|
||||
[Unit]
|
||||
Description=Felhom management-plane watchdog (privsep-dir auto-heal; agent-independent)
|
||||
# Ordering only — the timer drives cadence; no hard dependency so it runs even in degraded boots.
|
||||
After=systemd-tmpfiles-setup.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/felhom-mgmt-watchdog
|
||||
# Deliberately NO RuntimeDirectory=, NO PrivateTmp, NO sandboxing that would hide /run/sshd from it.
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
# felhom-mgmt-watchdog — the DUMB, agent-INDEPENDENT management-plane healer (TASK G1).
|
||||
#
|
||||
# WHY THIS EXISTS: /run/sshd is OpenSSH's compiled-in privilege-separation directory, SHARED by
|
||||
# every sshd on the host. If it goes missing, sshd's listener stays up but every new session RESETS
|
||||
# right after SSH2_MSG_KEXINIT — a management-plane lockout on an otherwise-healthy box
|
||||
# (SPIKE-felhom-sshd-2026-07-05 §8, caused live by a second unit's `RuntimeDirectory=sshd` cleanup).
|
||||
# The tmpfiles.d entry (felhom-privsep.tmpfiles) PREVENTS the known cause; this script AUTO-HEALS any
|
||||
# novel cause with ZERO login and ZERO dependency on the felhom-agent binary (so it self-corrects even
|
||||
# when the agent is down — the whole point). It is run by felhom-mgmt-watchdog.timer every ~60s.
|
||||
#
|
||||
# CONTRACT (all four are load-bearing):
|
||||
# 1. STAT-FIRST / IDEMPOTENT — a HEALTHY /run/sshd is NEVER touched (no mkdir/chmod/chown call), so
|
||||
# steady state is zero mutation and the dir's mtime is preserved. Only a MISSING or
|
||||
# wrong-mode/owner dir is corrected.
|
||||
# 2. reset-failed the STOCK sshd ONLY when it is in the `failed` state (a start-limit lockout) —
|
||||
# never otherwise, and NEVER `restart` it (mkdir alone restores it; PID stays, spike-proven).
|
||||
# 3. HEAL MARKER — writes an RFC3339 UTC timestamp to $MARKER only on a real heal, so the agent can
|
||||
# REPORT the condition to the hub (a recurring clobber must surface BEFORE it becomes a lockout).
|
||||
# 4. NO `RuntimeDirectory=` anywhere in this feature (that IS the incident cause) — the unit that
|
||||
# runs this script must not declare one either.
|
||||
#
|
||||
# Touches host /run + the stock sshd unit ONLY — no guests (pool-scoping is moot here). Fail-safe:
|
||||
# any single step failing is logged and does not abort the others (`|| true`); the next tick retries.
|
||||
|
||||
set -u
|
||||
|
||||
PRIVSEP=/run/sshd
|
||||
MARKER=/run/felhom-mgmt-watchdog.healed
|
||||
SSHD_UNIT=ssh.service # Debian/PVE stock sshd unit name (sshd.service is an alias)
|
||||
healed=0
|
||||
|
||||
log() { logger -t felhom-mgmt-watchdog "$*" 2>/dev/null || true; }
|
||||
|
||||
# 1. Privsep dir — stat-first: create only when missing; correct mode/owner only when wrong.
|
||||
if [ ! -d "$PRIVSEP" ]; then
|
||||
if mkdir -p "$PRIVSEP" && chown root:root "$PRIVSEP" && chmod 0755 "$PRIVSEP"; then
|
||||
healed=1
|
||||
log "recreated missing privsep dir $PRIVSEP (0755 root:root)"
|
||||
else
|
||||
log "ERROR: failed to recreate $PRIVSEP"
|
||||
fi
|
||||
else
|
||||
mode=$(stat -c %a "$PRIVSEP" 2>/dev/null || echo "")
|
||||
owner=$(stat -c %U:%G "$PRIVSEP" 2>/dev/null || echo "")
|
||||
if [ "$mode" != "755" ]; then
|
||||
chmod 0755 "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP mode ($mode -> 755)"
|
||||
fi
|
||||
if [ "$owner" != "root:root" ]; then
|
||||
chown root:root "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP owner ($owner -> root:root)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Stock sshd — clear a start-limit lockout ONLY when the unit is genuinely `failed`. Never restart
|
||||
# it (unnecessary — a recreated privsep dir is picked up by the next fork; the listener never died).
|
||||
if systemctl is-failed --quiet "$SSHD_UNIT" 2>/dev/null; then
|
||||
if systemctl reset-failed "$SSHD_UNIT" 2>/dev/null; then
|
||||
healed=1
|
||||
log "reset-failed $SSHD_UNIT (was in the failed state)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. Heal marker — only on a real heal, so the agent reports the condition (hub raises a warning).
|
||||
if [ "$healed" = "1" ]; then
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ > "$MARKER" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,18 @@
|
||||
# felhom-mgmt-watchdog.timer (TASK G1) — install as /etc/systemd/system/felhom-mgmt-watchdog.timer.
|
||||
#
|
||||
# Drives felhom-mgmt-watchdog.service on a ~60s cadence (the auto-heal tick). OnBootSec fires shortly
|
||||
# after boot (belt-and-suspenders with the tmpfiles layer); OnUnitActiveSec=60s gives the ≤1-tick
|
||||
# heal budget the G1 acceptance drill measures. Persistent=true runs a missed tick immediately after a
|
||||
# resume/late boot. The oneshot is idempotent, so a fast cadence never churns a healthy host.
|
||||
[Unit]
|
||||
Description=Felhom management-plane watchdog timer (~60s privsep-dir auto-heal tick)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitActiveSec=60s
|
||||
AccuracySec=5s
|
||||
Persistent=true
|
||||
Unit=felhom-mgmt-watchdog.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,19 @@
|
||||
# felhom-oob-nft.service (TASK H1) — install as /etc/systemd/system/felhom-oob-nft.service.
|
||||
#
|
||||
# Loads the STATIC felhom_oob belt table on boot (the agent then fills its sets each tick). Oneshot +
|
||||
# RemainAfterExit so `systemctl status` reflects "loaded". Independent of pve-firewall/nftables.service
|
||||
# (the table is a self-contained `inet felhom_oob`, additive — it never flushes the ruleset).
|
||||
[Unit]
|
||||
Description=Felhom OOB belt — load the static felhom_oob nft table (H1)
|
||||
After=network-pre.target
|
||||
Before=network.target felhom-agent.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/sbin/nft -f /etc/felhom-oob.nft
|
||||
# On stop, remove the table (best-effort) so an uninstall/disable leaves nft clean.
|
||||
ExecStop=-/usr/sbin/nft delete table inet felhom_oob
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/sbin/nft -f
|
||||
# felhom OOB belt — STATIC table (TASK H1). Installed by host-install; loaded on boot by
|
||||
# felhom-oob-nft.service. The agent mutates ONLY the two SETS (@operator_ips, @ssh_port) via the
|
||||
# narrow FELHOM_OOB sudoers — it NEVER touches the rules [trap 4], so the enforcement semantics are
|
||||
# fixed here and cannot be changed by the agent.
|
||||
#
|
||||
# Enforcement (scoped to iifname "wg-felhom" — the offsite tunnel — so :22 and every other interface
|
||||
# are UNTOUCHED [SF-3/OF-5]):
|
||||
# - ESTABLISHED,RELATED replies always pass (covers the PBS path + reply traffic; PMTU-safe).
|
||||
# - felhom-sshd's port is reachable from the operator /32 (in @operator_ips) over the tunnel ONLY.
|
||||
# - any other tunnel source to that port is DROPPED at the host (defense-in-depth vs the endpoint).
|
||||
# - the port is unreachable OFF the tunnel entirely.
|
||||
# Empty sets (before the agent's first sync) = the port rules match nothing = no enforcement yet
|
||||
# (felhom-sshd is not started until the agent renders its config either). priority -5 sits just above
|
||||
# the default; policy accept so this table only ever ADDS drops for the felhom-sshd port.
|
||||
#
|
||||
# Idempotent load: create-then-delete-then-define (a re-run/boot yields a clean table with EMPTY sets;
|
||||
# the agent refills them within one tick).
|
||||
|
||||
table inet felhom_oob
|
||||
delete table inet felhom_oob
|
||||
|
||||
table inet felhom_oob {
|
||||
set operator_ips {
|
||||
type ipv4_addr
|
||||
}
|
||||
set ssh_port {
|
||||
type inet_service
|
||||
}
|
||||
chain input {
|
||||
type filter hook input priority -5; policy accept;
|
||||
ct state established,related accept
|
||||
iifname "wg-felhom" tcp dport @ssh_port ip saddr @operator_ips accept
|
||||
iifname "wg-felhom" tcp dport @ssh_port drop
|
||||
tcp dport @ssh_port iifname != "wg-felhom" drop
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
# felhom-op operator sudoers (TASK H1) — install as /etc/sudoers.d/felhom-op (0440 root:root).
|
||||
#
|
||||
# The DEFAULT operator identity (login via felhom-sshd only; key in /etc/felhom-sshd/authorized_keys/
|
||||
# felhom-op, outside ~/.ssh, so the customer's sshd never honours it [SF-3]). felhom-op is a normal
|
||||
# login user with NO ambient privilege — it escalates ONLY through these explicit repair verbs, so a
|
||||
# leaked operator key's blast radius is this verb set, not the box (locked decision §6). Deep repairs
|
||||
# that genuinely need root (pct, deeper systemctl) remain available via `root` as a tunnel-gated
|
||||
# break-glass account (PermitRootLogin prohibit-password on felhom-sshd), but the day-to-day identity
|
||||
# is felhom-op.
|
||||
Cmnd_Alias FELHOM_OP_REPAIR = \
|
||||
/usr/bin/mkdir -p /run/sshd, \
|
||||
/usr/bin/systemctl reset-failed wg-quick@wg-felhom, \
|
||||
/usr/bin/systemctl restart wg-quick@wg-felhom, \
|
||||
/usr/bin/systemctl start felhom-agent, \
|
||||
/usr/bin/systemctl restart felhom-agent, \
|
||||
/usr/bin/systemctl reset-failed felhom-sshd, \
|
||||
/usr/bin/systemctl restart felhom-sshd, \
|
||||
/usr/sbin/pct list, \
|
||||
/usr/sbin/pct start [0-9]*, \
|
||||
/usr/sbin/pct stop [0-9]*, \
|
||||
/usr/sbin/pct unlock [0-9]*
|
||||
|
||||
felhom-op ALL=(root) NOPASSWD: FELHOM_OP_REPAIR
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/bin/bash
|
||||
#===============================================================================
|
||||
# felhom-pbs-apply — the ONLY storage-entry path the felhom-agent sudoers permits for the PBS DR
|
||||
# tier (slice 2; SPIKE-pbs-tier-provisioning-2026-07-10 §2b). The guarded-mkfs shape: the agent's
|
||||
# in-process validation is the PRIMARY gate (descriptor-field charset + equality checks BEFORE
|
||||
# exec); this wrapper is the minimal, auditable second gate as root.
|
||||
#
|
||||
# THE SET-ONLY LAW (spike §4, data-loss class): `pvesm` entry deletion DESTROYS the client
|
||||
# encryption key file (<id>.enc = K) — un-decryptable backups. This wrapper therefore contains
|
||||
# NO deletion path of any kind, and a re-apply is `pvesm set`-only. Grep-assertable; do not add
|
||||
# a "cleanup" verb here, ever. Deprovision is a deliberate future operator op, not this tool.
|
||||
#
|
||||
# SECRET DISCIPLINE (spike §2b): sudo logs its full argv to auth.log → the PBS token secret
|
||||
# arrives on STDIN, never as an argument to this wrapper. Inside, it is passed to pvesm's
|
||||
# --password (root-local, transient ps exposure — the accepted spike posture); it is never
|
||||
# echoed, never written anywhere except by pvesm itself (the 0600 .pw store).
|
||||
#
|
||||
# Ops (non-secret args on argv):
|
||||
# create <id> <server> <datastore> <namespace> <token-id> <fingerprint> <secret-dir>
|
||||
# secret on stdin (required). Creates the pbs entry with --encryption-key autogen
|
||||
# (K born at /etc/pve/priv/storage/<id>.enc), then places .pw/.enc copies in
|
||||
# <secret-dir> when it differs (the §4b WARN-fix dir; escrow-create's PBSEncKeyPath
|
||||
# must find K there — spike §4 flag).
|
||||
# reconcile <id> <server> <namespace> <token-id> <fingerprint> <secret-dir>
|
||||
# secret on stdin (optional; empty = no credential change). `pvesm set` ONLY:
|
||||
# server/fingerprint (+ --password when a secret is fed, e.g. after a hub re-issue).
|
||||
# NOTE datastore is deliberately NOT settable, and namespace/token-id are accepted
|
||||
# for validation parity but NOT applied — tenancy identity is adopt-only (the
|
||||
# demo's live entry must never be repointed at a different namespace).
|
||||
# read <id> <secret-dir>
|
||||
# R-39 leg (b): print the storage's token secret to STDOUT and nothing else.
|
||||
# The non-root agent WRITES this file through this wrapper but could never read it
|
||||
# back (/etc/pve/priv is 0700 root:www-data and there is no read verb), so its
|
||||
# 15-minute PBS verify loop was permanently blind to the one failure it exists to
|
||||
# catch — an `applied` tier authenticating 401. This verb is that missing read.
|
||||
# It is deliberately the narrowest thing that works: no network, no mutation, no
|
||||
# logging of the value, one file, prefix-asserted under the given secret dir.
|
||||
#
|
||||
# grant <id>
|
||||
# The Part-0-evidenced dual-grant: FelhomAgentStore on /storage/<id> to the agent
|
||||
# user AND token (privsep intersection). Datastore.Audit reads ride the base role.
|
||||
#===============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
die() { echo "felhom-pbs-apply: REFUSED: $*" >&2; exit 1; }
|
||||
|
||||
op="${1:-}"; id="${2:-}"
|
||||
[[ -n "$op" && -n "$id" ]] || die "usage: felhom-pbs-apply <create|reconcile|grant|read> <storage-id> ..."
|
||||
|
||||
# Storage id: PVE grammar, conservative. Also the ACL path component — no slashes possible.
|
||||
[[ "$id" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,27}$ ]] || die "bad storage id ($id)"
|
||||
|
||||
STORECFG=/etc/pve/storage.cfg
|
||||
PRIVDIR=/etc/pve/priv/storage
|
||||
|
||||
entry_exists() { grep -Eq "^pbs: ${id}\$" "$STORECFG"; }
|
||||
|
||||
val_server() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,62}$ ]] || die "bad server ($1)"; }
|
||||
val_datastore() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,31}$ ]] || die "bad datastore ($1)"; }
|
||||
val_ns() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,31}$ ]] || die "bad namespace ($1)"; }
|
||||
val_tok() { [[ "$1" =~ ^[A-Za-z0-9_.-]+@[A-Za-z0-9]+![A-Za-z0-9_.-]+$ ]] || die "bad token id ($1)"; }
|
||||
val_fp() { [[ "$1" =~ ^([A-Fa-f0-9]{2}:){31}[A-Fa-f0-9]{2}$ ]] || die "bad fingerprint"; }
|
||||
val_sdir() {
|
||||
case "$1" in
|
||||
/etc/pve/priv/storage) : ;;
|
||||
/var/lib/felhom-agent/*) [[ "$1" != *..* ]] || die "secret-dir traversal" ;;
|
||||
*) die "secret-dir must be $PRIVDIR or under /var/lib/felhom-agent ($1)" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
read_secret() { # → SECRET (may be empty when optional)
|
||||
local s
|
||||
s=$(head -c 256 || true)
|
||||
s="${s%$'\n'}"; s="${s%$'\r'}"
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
place_copies() { # secret-dir — the §4b WARN-fix placement (non-root agent can't read /etc/pve/priv)
|
||||
local sdir="$1"
|
||||
[[ "$sdir" == "$PRIVDIR" ]] && return 0
|
||||
install -d -o felhom-agent -g felhom-agent -m 0700 "$sdir"
|
||||
[[ -f "$PRIVDIR/$id.pw" ]] && install -o felhom-agent -g felhom-agent -m 0600 "$PRIVDIR/$id.pw" "$sdir/$id.pw"
|
||||
# K's copy: escrow-create stats PBSEncKeyPath(<secret-dir>/<id>.enc) — the spike §4 flag.
|
||||
[[ -f "$PRIVDIR/$id.enc" ]] && install -o root -g felhom-agent -m 0640 "$PRIVDIR/$id.enc" "$sdir/$id.enc"
|
||||
return 0
|
||||
}
|
||||
|
||||
case "$op" in
|
||||
create)
|
||||
[[ $# -eq 8 ]] || die "create needs 7 args: <id> <server> <datastore> <namespace> <token-id> <fingerprint> <secret-dir>"
|
||||
server="$3"; datastore="$4"; ns="$5"; tok="$6"; fp="$7"; sdir="$8"
|
||||
val_server "$server"; val_datastore "$datastore"; val_ns "$ns"; val_tok "$tok"; val_fp "$fp"; val_sdir "$sdir"
|
||||
entry_exists && die "entry $id already exists (reconcile is the re-apply path — set-only law)"
|
||||
SECRET=$(read_secret)
|
||||
[[ -n "$SECRET" ]] || die "create requires the token secret on stdin"
|
||||
pvesm add pbs "$id" \
|
||||
--server "$server" --datastore "$datastore" --namespace "$ns" \
|
||||
--username "$tok" --password "$SECRET" --fingerprint "$fp" \
|
||||
--content backup --encryption-key autogen >&2
|
||||
[[ -f "$PRIVDIR/$id.enc" ]] || die "pvesm add succeeded but K ($PRIVDIR/$id.enc) was not born"
|
||||
place_copies "$sdir"
|
||||
echo "felhom-pbs-apply: created $id (K born; encryption-key autogen)" >&2
|
||||
;;
|
||||
reconcile)
|
||||
[[ $# -eq 7 ]] || die "reconcile needs 6 args: <id> <server> <namespace> <token-id> <fingerprint> <secret-dir>"
|
||||
server="$3"; ns="$4"; tok="$5"; fp="$6"; sdir="$7"
|
||||
val_server "$server"; val_ns "$ns"; val_tok "$tok"; val_fp "$fp"; val_sdir "$sdir"
|
||||
entry_exists || die "entry $id does not exist (create is the fresh path)"
|
||||
SECRET=$(read_secret)
|
||||
# R-39 (2026-07-18): NEVER pass --server to `pvesm set`. PVE treats `server` as a CREATE-ONLY
|
||||
# parameter and rejects the ENTIRE call — "can't change value of fixed parameter 'server'" —
|
||||
# even when the value is byte-identical to the stored one. That made every reconcile exit 255,
|
||||
# so each hub-re-issued one-time secret was consumed-then-burned and the tier stayed pinned to
|
||||
# a revoked credential (401 forever). Proven live on the N100 demo host: `pvesm set <id>
|
||||
# --server <same> --fingerprint <same>` -> rejected; the same call without --server -> rc 0.
|
||||
# The server address is immutable by construction (relocating a PBS endpoint needs a fresh
|
||||
# create), so there is nothing here to reconcile. Guarded by
|
||||
# TestReconcileNeverPassesServerToPvesmSet.
|
||||
args=(--fingerprint "$fp")
|
||||
[[ -n "$SECRET" ]] && args+=(--password "$SECRET")
|
||||
pvesm set "$id" "${args[@]}" >&2
|
||||
place_copies "$sdir"
|
||||
echo "felhom-pbs-apply: reconciled $id (set-only; tenancy identity untouched)" >&2
|
||||
;;
|
||||
read)
|
||||
# R-39(b): the missing read path. Prints the secret to STDOUT, nothing else — no stderr note (it
|
||||
# would be the only verb whose success line could be confused with the value), no mutation.
|
||||
#
|
||||
# Traversal is refused three times over, because this is the one verb that EXFILTRATES a file and
|
||||
# its argv is attacker-shaped if the agent is ever compromised:
|
||||
# 1. `id` already matched ^[A-Za-z][A-Za-z0-9_.-]{0,27}$ at the top — it cannot start with a dot
|
||||
# and cannot contain a slash, so "../../etc/shadow" never reaches here;
|
||||
# 2. val_sdir pins the directory to PRIVDIR or under /var/lib/felhom-agent, rejecting "..";
|
||||
# 3. the RESOLVED path is prefix-asserted under that directory below, so even a future change to
|
||||
# either grammar cannot walk out.
|
||||
[[ $# -eq 3 ]] || die "read needs 2 args: <id> <secret-dir>"
|
||||
sdir="$3"
|
||||
val_sdir "$sdir"
|
||||
target="$sdir/$id.pw"
|
||||
# Belt: resolve and re-check the prefix (guards a symlinked <id>.pw pointing outside the dir).
|
||||
resolved=$(readlink -f -- "$target" 2>/dev/null || true)
|
||||
[[ -n "$resolved" ]] || die "secret file not found ($target)"
|
||||
case "$resolved" in
|
||||
"$sdir"/*) : ;;
|
||||
*) die "resolved secret path escapes the secret dir" ;;
|
||||
esac
|
||||
[[ -f "$resolved" ]] || die "secret file not found ($target)"
|
||||
cat -- "$resolved"
|
||||
;;
|
||||
grant)
|
||||
[[ $# -eq 2 ]] || die "grant takes only <id>"
|
||||
pveum acl modify "/storage/$id" --users felhom-agent@pve --roles FelhomAgentStore >&2
|
||||
pveum acl modify "/storage/$id" --tokens 'felhom-agent@pve!agent' --roles FelhomAgentStore >&2
|
||||
echo "felhom-pbs-apply: granted FelhomAgentStore on /storage/$id (user + token)" >&2
|
||||
;;
|
||||
*)
|
||||
die "unknown op ($op)"
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,15 @@
|
||||
# felhom privsep-dir persistence (TASK G1) — install as /etc/tmpfiles.d/felhom-privsep.conf.
|
||||
#
|
||||
# LAYER 1 (prevent): make OpenSSH's shared privilege-separation directory /run/sshd boot-persistent
|
||||
# and owned by NO systemd unit's lifecycle. This closes the exact incident cause from
|
||||
# SPIKE-felhom-sshd-2026-07-05 §8: a second sshd unit declaring `RuntimeDirectory=sshd` had systemd
|
||||
# REMOVE the shared /run/sshd on that unit's stop/failure, taking the stock sshd on :22 down with it
|
||||
# (sessions reset right after SSH2_MSG_KEXINIT). A tmpfiles.d entry recreates the dir at every boot
|
||||
# independently of any unit, so no unit's RuntimeDirectory cleanup can be the sole owner.
|
||||
#
|
||||
# systemd-tmpfiles is idempotent: `systemd-tmpfiles --create` re-run is a no-op on an existing,
|
||||
# correct dir (it only creates/fixes, never churns). Complemented at runtime by the
|
||||
# felhom-mgmt-watchdog timer (layer 2), which re-heals a dir removed AFTER boot.
|
||||
#
|
||||
# Type d = create the directory if absent (leaves an existing one, only fixing mode/owner).
|
||||
d /run/sshd 0755 root root -
|
||||
@@ -0,0 +1,28 @@
|
||||
# felhom-sshd.service (TASK H1) — install as /etc/systemd/system/felhom-sshd.service.
|
||||
#
|
||||
# The dedicated OOB sshd instance: a SECOND sshd on a claimed non-22 port, COEXISTING with the
|
||||
# customer's/stock sshd on :22 (never touched). Config is AGENT-RENDERED at /etc/felhom-sshd/sshd_config
|
||||
# (Port from the agent's claim); the agent reloads on change (never restart-on-change [SF-2]).
|
||||
#
|
||||
# CRITICAL [SF-1]: this unit MUST NOT declare `RuntimeDirectory=` — that directive (value `sshd`)
|
||||
# removed the SHARED /run/sshd and broke the stock sshd live (SPIKE-felhom-sshd §8). /run/sshd is
|
||||
# guaranteed by G1 (tmpfiles + watchdog); the ExecStartPre mkdir is a harmless extra guarantee. The
|
||||
# host-install RuntimeDirectory guard will REJECT this unit if the directive is ever added.
|
||||
[Unit]
|
||||
Description=Felhom OOB sshd (dedicated instance, agent-managed; H1)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
ExecStartPre=-/usr/bin/mkdir -p /run/sshd
|
||||
ExecStartPre=/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config
|
||||
ExecStart=/usr/sbin/sshd -D -f /etc/felhom-sshd/sshd_config
|
||||
# Config-change path: validate then HUP — a bad reload is REFUSED and the running daemon SURVIVES
|
||||
# [SF-2]. The agent always writes→sshd -t→reload; it never `restart`s on a config change.
|
||||
ExecReload=/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
KillMode=process
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,243 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// F-CRIT-2 (Campaign 8): a failed backup must not look like a fresh one.
|
||||
//
|
||||
// Every fixture below is a VERBATIM shape captured from the live PVE API on 2026-07-28
|
||||
// (`pvesh get /nodes/<node>/storage/<store>/content`), not a hand-invented struct. That matters:
|
||||
// the `unparseable` path in this package went untested for months behind a JSON shape that did not
|
||||
// match production, and the whole point of this fix is that presence != validity.
|
||||
|
||||
// phantomEntry is the artefact a PBS daemon killed mid-upload leaves behind: listed as a restorable
|
||||
// backup, 1 byte, NEWEST, and carrying no `verification`/`encrypted`/`notes` at all because it has
|
||||
// no manifest (`index.json.blob` is absent on disk).
|
||||
func phantomEntry() proxmox.StorageContent {
|
||||
return proxmox.StorageContent{
|
||||
VolID: "felhom-pbs:backup/ct/9201/2026-07-28T05:31:14Z",
|
||||
Content: "backup",
|
||||
Format: "pbs-ct",
|
||||
Size: 1,
|
||||
CTime: 1785216674,
|
||||
VMID: 9201,
|
||||
}
|
||||
}
|
||||
|
||||
// goodPBSEntry is a real, complete offsite snapshot (demo-hp, 2026-07-28T03:40:42Z).
|
||||
func goodPBSEntry() proxmox.StorageContent {
|
||||
return proxmox.StorageContent{
|
||||
VolID: "felhom-pbs:backup/ct/9201/2026-07-28T03:40:42Z",
|
||||
Content: "backup",
|
||||
Format: "pbs-ct",
|
||||
Size: 4353457559,
|
||||
CTime: 1785210042,
|
||||
VMID: 9201,
|
||||
}
|
||||
}
|
||||
|
||||
// goodLocalEntry is a real, complete LOCAL vzdump (demo-hp). Note it legitimately has no
|
||||
// `verification` and no `encrypted` on the wire — a dir storage has no such concept — which is
|
||||
// exactly why those fields must never be used as completeness discriminators.
|
||||
func goodLocalEntry() proxmox.StorageContent {
|
||||
return proxmox.StorageContent{
|
||||
VolID: "local:backup/vzdump-lxc-9201-2026_07_28-07_29_54.tar.zst",
|
||||
Content: "backup",
|
||||
Format: "tar.zst",
|
||||
Size: 1590431865,
|
||||
CTime: 1785216594,
|
||||
VMID: 9201,
|
||||
}
|
||||
}
|
||||
|
||||
func runnerWithContent(t *testing.T, buf *bytes.Buffer, content []proxmox.StorageContent) *BackupRunner {
|
||||
t.Helper()
|
||||
lg := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
||||
return NewBackupRunner(&fakeBackupAPI{content: content}, "felhom-pbs", proxmox.ModeSnapshot, "", "", lg)
|
||||
}
|
||||
|
||||
// Group A — the phantom must NOT set tier freshness, even though it is the newest entry.
|
||||
//
|
||||
// RED-PROOF: restore the old predicate in NewestArchiveTime
|
||||
// (`if e.Content == "backup" && e.VMID == vmid && e.CTime > best`) → the phantom's ctime
|
||||
// (1785216674) wins over the good snapshot's (1785210042) and this test fails with
|
||||
// "got 1785216674, want 1785210042" — i.e. the exact F-CRIT-2 defect.
|
||||
func TestNewestArchiveTime_PhantomIsNotCounted(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
// phantom deliberately listed FIRST and is also the newest by ctime.
|
||||
r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), goodPBSEntry()})
|
||||
|
||||
got, found, err := r.NewestArchiveTime(context.Background(), 9201)
|
||||
if err != nil {
|
||||
t.Fatalf("NewestArchiveTime: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("found=false — the GOOD snapshot must still be counted; rejecting everything is the thrash path")
|
||||
}
|
||||
if got.Unix() != goodPBSEntry().CTime {
|
||||
t.Errorf("freshness came from the wrong entry: got ctime %d, want %d (the good snapshot)", got.Unix(), goodPBSEntry().CTime)
|
||||
}
|
||||
if got.Unix() == phantomEntry().CTime {
|
||||
t.Error("the 1-byte manifest-less phantom set tier freshness — this is F-CRIT-2")
|
||||
}
|
||||
}
|
||||
|
||||
// Group A — with ONLY a phantom present the tier must report "no backup", not a fresh one.
|
||||
// That is what lets the controller see age_state=absent and fire its first-backup valve.
|
||||
func TestNewestArchiveTime_OnlyPhantomReportsNotFound(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry()})
|
||||
|
||||
_, found, err := r.NewestArchiveTime(context.Background(), 9201)
|
||||
if err != nil {
|
||||
t.Fatalf("NewestArchiveTime: %v", err)
|
||||
}
|
||||
if found {
|
||||
t.Error("found=true with only a phantom present — the tier would report fresh and go silent for a full cadence")
|
||||
}
|
||||
}
|
||||
|
||||
// Group B — THE SCENARIO-D GUARD. A valid snapshot on EITHER tier must still be counted.
|
||||
//
|
||||
// This is what makes Group A safe. A filter that is too aggressive does not merely lose safety
|
||||
// margin: the tier reports absent on every poll, backs up every cycle, and the R-88 breaker cannot
|
||||
// save it because those backups SUCCEED. That is a continuous multi-GB write loop across the fleet.
|
||||
//
|
||||
// RED-PROOF: make archivePlausiblyComplete return `false, "reject everything"` unconditionally →
|
||||
// both subtests fail with found=false.
|
||||
func TestNewestArchiveTime_ValidSnapshotsAreStillCounted(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
entry proxmox.StorageContent
|
||||
}{
|
||||
{"pbs offsite (has verification+encrypted on the wire)", goodPBSEntry()},
|
||||
{"local dir vzdump (has NEITHER verification NOR encrypted — and must still count)", goodLocalEntry()},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
r := runnerWithContent(t, &buf, []proxmox.StorageContent{tc.entry})
|
||||
|
||||
got, found, err := r.NewestArchiveTime(context.Background(), 9201)
|
||||
if err != nil {
|
||||
t.Fatalf("NewestArchiveTime: %v", err)
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("a REAL %s backup was rejected — this is the backup-thrash path, not extra safety", tc.name)
|
||||
}
|
||||
if got.Unix() != tc.entry.CTime {
|
||||
t.Errorf("got ctime %d, want %d", got.Unix(), tc.entry.CTime)
|
||||
}
|
||||
if strings.Contains(buf.String(), "INCOMPLETE archive") {
|
||||
t.Errorf("a valid archive was announced as incomplete:\n%s", buf.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Group B — the smallest REAL backup measured anywhere on the fleet (612,397,450 B, a guest-9100
|
||||
// vzdump) must clear the floor with room to spare. If someone ever raises
|
||||
// minPlausibleArchiveBytes past this, that is the fleet-thrash bug and this test is the tripwire.
|
||||
func TestMinPlausibleArchiveBytes_LeavesHeadroomBelowTheSmallestRealBackup(t *testing.T) {
|
||||
const smallestObservedRealBackup int64 = 612397450 // fleet survey 2026-07-28
|
||||
if minPlausibleArchiveBytes >= smallestObservedRealBackup {
|
||||
t.Fatalf("floor %d B is not below the smallest real backup ever observed (%d B) — this WILL reject real archives",
|
||||
minPlausibleArchiveBytes, smallestObservedRealBackup)
|
||||
}
|
||||
if ratio := smallestObservedRealBackup / minPlausibleArchiveBytes; ratio < 100 {
|
||||
t.Errorf("floor %d B leaves only %dx headroom below the smallest real backup (%d B) — too tight",
|
||||
minPlausibleArchiveBytes, ratio, smallestObservedRealBackup)
|
||||
}
|
||||
}
|
||||
|
||||
// Group C — UNDECIDABLE ⇒ NOT COUNTED (the fail-safe direction).
|
||||
//
|
||||
// A zero/absent size is not evidence of a good backup; it is absence of evidence. Erring toward
|
||||
// "not fresh" costs one extra backup. Erring the other way is F-CRIT-2.
|
||||
//
|
||||
// RED-PROOF: flip the comparison in archivePlausiblyComplete to `e.Size > minPlausibleArchiveBytes
|
||||
// || e.Size == 0` (i.e. treat unknown as complete) → the size-0 case reports ok=true and this fails.
|
||||
func TestArchivePlausiblyComplete_UndecidableIsNotCounted(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
size int64
|
||||
}{
|
||||
{"the observed phantom", 1},
|
||||
{"absent size field (unmarshals to 0)", 0},
|
||||
{"just under the floor", minPlausibleArchiveBytes - 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
e := phantomEntry()
|
||||
e.Size = tc.size
|
||||
ok, why := archivePlausiblyComplete(e)
|
||||
if ok {
|
||||
t.Errorf("size %d counted as a complete backup — undecidable must fail safe", tc.size)
|
||||
}
|
||||
if why == "" {
|
||||
t.Error("rejection carried no reason — a silent rejection is a new quiet path")
|
||||
}
|
||||
})
|
||||
}
|
||||
if ok, why := archivePlausiblyComplete(goodPBSEntry()); !ok {
|
||||
t.Errorf("a real snapshot was rejected: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// Group D — the rejection is announced ONCE per snapshot, not once per due-check.
|
||||
//
|
||||
// The due-check runs every 5 minutes and a phantom persists indefinitely (server-side prune does
|
||||
// not collect it), so per-poll logging would emit ~288 identical lines a day and bury the signal.
|
||||
//
|
||||
// RED-PROOF: delete the `if seen { return }` guard in warnRejectedArchiveOnce → this test reports
|
||||
// "logged 5 times, want 1".
|
||||
func TestNewestArchiveTime_RejectionLoggedOncePerSnapshot(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), goodPBSEntry()})
|
||||
|
||||
const polls = 5
|
||||
for i := 0; i < polls; i++ {
|
||||
if _, _, err := r.NewestArchiveTime(context.Background(), 9201); err != nil {
|
||||
t.Fatalf("poll %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
n := strings.Count(buf.String(), "INCOMPLETE archive")
|
||||
if n != 1 {
|
||||
t.Errorf("rejection logged %d times across %d polls, want exactly 1:\n%s", n, polls, buf.String())
|
||||
}
|
||||
out := buf.String()
|
||||
if !strings.Contains(out, phantomEntry().VolID) {
|
||||
t.Errorf("the log line does not NAME the rejected snapshot:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "below the") {
|
||||
t.Errorf("the log line does not say WHY it was rejected:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "level=WARN") {
|
||||
t.Errorf("rejection was not logged at WARN:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// Group D — a SECOND, distinct phantom is announced separately. The dedupe must be per snapshot,
|
||||
// not a one-shot latch that hides every later phantom.
|
||||
func TestNewestArchiveTime_DistinctPhantomsEachAnnounced(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
second := phantomEntry()
|
||||
second.VolID = "felhom-pbs:backup/ct/9201/2026-07-29T05:31:14Z"
|
||||
second.CTime = phantomEntry().CTime + 86400
|
||||
r := runnerWithContent(t, &buf, []proxmox.StorageContent{phantomEntry(), second, goodPBSEntry()})
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, _, err := r.NewestArchiveTime(context.Background(), 9201); err != nil {
|
||||
t.Fatalf("poll %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if n := strings.Count(buf.String(), "INCOMPLETE archive"); n != 2 {
|
||||
t.Errorf("got %d rejection lines for 2 distinct phantoms across 3 polls, want 2:\n%s", n, buf.String())
|
||||
}
|
||||
}
|
||||
@@ -135,10 +135,11 @@ func TestBackup_VzdumpFailureReturnsFailedRecord(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPickRestoreCandidate_NewestOrEmpty(t *testing.T) {
|
||||
const big = 4 << 30 // a plausible whole-guest archive
|
||||
api := &fakeBackupAPI{content: []proxmox.StorageContent{
|
||||
{VolID: "a", Content: "backup", CTime: 10},
|
||||
{VolID: "b", Content: "backup", CTime: 99},
|
||||
{VolID: "iso", Content: "iso", CTime: 999}, // not a backup → ignored
|
||||
{VolID: "a", Content: "backup", CTime: 10, Size: big},
|
||||
{VolID: "b", Content: "backup", CTime: 99, Size: big},
|
||||
{VolID: "iso", Content: "iso", CTime: 999, Size: big}, // not a backup → ignored
|
||||
}}
|
||||
r := NewBackupRunner(api, "local", "", "", "", quiet())
|
||||
vol, err := r.PickRestoreCandidate(context.Background())
|
||||
@@ -152,6 +153,26 @@ func TestPickRestoreCandidate_NewestOrEmpty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// R-86: the NEWEST entry is not a candidate if it cannot be a complete archive. An incomplete
|
||||
// artefact (F-CRIT-2's 1-byte phantom, which server-side prune does not collect) would otherwise be
|
||||
// picked forever, fail its restore forever, never earn proof, and so leave the tier due at every
|
||||
// evaluation — turning the evaluation interval into the retry rate for a multi-GB restore.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): drop the `archivePlausiblyComplete` guard from
|
||||
// PickSettledRestoreCandidateOn and this fails with
|
||||
// `pick = "phantom" want the newest COMPLETE archive 'real'`.
|
||||
func TestPickRestoreCandidate_SkipsImplausibleArchives(t *testing.T) {
|
||||
api := &fakeBackupAPI{content: []proxmox.StorageContent{
|
||||
{VolID: "real", Content: "backup", CTime: 10, Size: 4 << 30},
|
||||
{VolID: "phantom", Content: "backup", CTime: 99, Size: 1}, // newest, and impossible
|
||||
}}
|
||||
r := NewBackupRunner(api, "local", "", "", "", quiet())
|
||||
vol, err := r.PickRestoreCandidate(context.Background())
|
||||
if err != nil || vol != "real" {
|
||||
t.Fatalf("pick = %q,%v want the newest COMPLETE archive 'real'", vol, err)
|
||||
}
|
||||
}
|
||||
|
||||
// --- scheduler ---
|
||||
|
||||
type fakeRTRunner struct {
|
||||
@@ -168,9 +189,12 @@ func TestScheduler_TickRunsAndRecords(t *testing.T) {
|
||||
store := NewStore()
|
||||
rt := &fakeRTRunner{res: reconcile.RestoreTestResult{Archive: "vol", Pass: true, Verified: "boot+running", Duration: time.Second}}
|
||||
s := NewScheduler(SchedulerOptions{
|
||||
Runner: rt,
|
||||
Pick: func(context.Context) (string, error) { return "vol", nil },
|
||||
Store: store,
|
||||
Runner: rt,
|
||||
Pick: func(context.Context) (string, error) { return "vol", nil },
|
||||
Store: store,
|
||||
Spec: func(context.Context, string) reconcile.RestoreTestSpec {
|
||||
return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009}
|
||||
},
|
||||
Cadence: time.Hour,
|
||||
Logger: quiet(),
|
||||
})
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package backup
|
||||
|
||||
import "sync"
|
||||
|
||||
// InFlight is the host-wide "one heavy guest operation at a time" gate.
|
||||
//
|
||||
// R-85 (Scenario F). The operator's R-82 ruling was "one backup at a time per guest"; a restore-test
|
||||
// must JOIN that single-flight rather than sit outside it. It is not a lock-contention concern —
|
||||
// a restore-test uses a scratch VMID, so it never touches the live guest's vzdump lock. It is a
|
||||
// LINK concern: an offsite restore PULLS a multi-GB archive while an offsite backup PUSHES one, over
|
||||
// the same WireGuard tunnel. On the demo fleet that link runs at ~33 MB/min upstream; running both
|
||||
// at once makes each slower and pushes both toward their timeouts, which is how a healthy tier ends
|
||||
// up recorded as failed.
|
||||
//
|
||||
// It is deliberately host-wide and coarse rather than per-guest: these boxes carry one customer
|
||||
// guest, and the resource being protected (the uplink) is shared by everything on the host anyway.
|
||||
//
|
||||
// The gate is ADVISORY in one direction only — it never cancels anything already running. A caller
|
||||
// that cannot acquire DEFERS to its next cadence. Deferring a restore-test costs a few hours of
|
||||
// coverage; cancelling a running backup costs the backup.
|
||||
//
|
||||
// CORRECTED 2026-07-28 (F-A1). That "DEFERS" was true of the restore-test caller and NOT of the
|
||||
// backup caller, and the comment did not say so. The controller's start path had no 409 branch, so
|
||||
// a refusal here was recorded as a tier FAILURE: the R-88 breaker armed and the operator was
|
||||
// emailed "Whole-guest backup FAILED" about a backup that was merely waiting its turn. Campaign 8
|
||||
// observed it on both demo boxes in the same minute.
|
||||
//
|
||||
// Fixed on the CONTROLLER side (v0.179.0), which is where the misreading lived — this gate's
|
||||
// behaviour was correct throughout and is unchanged. The controller now maps HTTP 409 to a
|
||||
// contention path: it defers the tier, keeps it DUE, and alarms only if contention outlives the
|
||||
// agent's own restore-test ceiling. Nothing here needs to change; the claim above is simply now
|
||||
// true of both callers.
|
||||
type InFlight struct {
|
||||
mu sync.Mutex
|
||||
what string // "" = idle
|
||||
}
|
||||
|
||||
// TryAcquire claims the gate for `what`. ok=false means something else holds it, and `busy` names
|
||||
// it — the name matters, because "deferred" with no reason is indistinguishable from "broken".
|
||||
func (g *InFlight) TryAcquire(what string) (release func(), busy string, ok bool) {
|
||||
if g == nil {
|
||||
// Not wired (older call sites, tests) → no gating, previous behaviour.
|
||||
return func() {}, "", true
|
||||
}
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.what != "" {
|
||||
return nil, g.what, false
|
||||
}
|
||||
g.what = what
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
g.mu.Lock()
|
||||
g.what = ""
|
||||
g.mu.Unlock()
|
||||
})
|
||||
}, "", true
|
||||
}
|
||||
|
||||
// Busy reports what currently holds the gate ("" = idle).
|
||||
func (g *InFlight) Busy() string {
|
||||
if g == nil {
|
||||
return ""
|
||||
}
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.what
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// R-86 — a restore-test follows the BACKUP, not the clock.
|
||||
//
|
||||
// ── WHAT WAS WRONG ───────────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// The trigger was `time.NewTicker(cadence)` started at daemon start, and the tier was chosen by
|
||||
// oldest-proven rotation. Its phase was therefore the PROCESS'S UPTIME: agent deploys are routine,
|
||||
// so the test drifted to an arbitrary time of day every week; a fresh archive could sit unproven
|
||||
// while an older one was re-tested; and a weekly tier was tested on the same rhythm as a daily one,
|
||||
// sometimes twice on the same archive.
|
||||
//
|
||||
// ── THE RULE, AND THE TRAP IN ITS OBVIOUS FORM ───────────────────────────────────────────────
|
||||
//
|
||||
// R-86's ask reads "test a tier ~24 h after its own newest archive". Implemented literally —
|
||||
// *"due when the newest archive is at least `settle` old"* — a DAILY tier is NEVER due: a new
|
||||
// archive lands every day, so the newest archive's age resets to zero long before it reaches 24 h.
|
||||
// The naive rule silently switches restore-testing off for the tier that matters most, and it is
|
||||
// the version a reasonable person would write. It has a red-proof of its own
|
||||
// (TestDue_NaiveNewestArchiveAgeRuleNeverFiresOnADailyTier).
|
||||
//
|
||||
// The rule implemented here:
|
||||
//
|
||||
// Let A = the newest archive on this tier that is at least `settle` old.
|
||||
// The tier is DUE when A exists and A HAS NOT ALREADY BEEN PROVEN.
|
||||
//
|
||||
// daily tier → A is yesterday's archive; a new one settles each day → proved once per day
|
||||
// weekly tier → A is last week's until the next settles → proved once per week
|
||||
// newborn tier → A does not exist → UNKNOWN, never a fault
|
||||
//
|
||||
// Per-archive due-ness IS the pacing: one test per archive generation and no more. There is
|
||||
// deliberately no second rate limiter on top of it (§8.4) — two independent pacing mechanisms
|
||||
// produce a cadence nobody can predict from either.
|
||||
//
|
||||
// ── WHAT DID NOT CHANGE ──────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// The one-heavy-operation gate, the success-only proof credit, the oldest-proven ordering (now the
|
||||
// tie-break between two DUE tiers), the restore-test itself, its journal and its scratch band. Only
|
||||
// the trigger changed.
|
||||
|
||||
// DueVerdict is one tier's due-ness, and the evidence for it. Every field is logged: a due-check
|
||||
// that cannot say WHY is a quiet path, and quiet paths are what this monitor family keeps shipping.
|
||||
type DueVerdict struct {
|
||||
Target string // the tier's storage target id
|
||||
|
||||
// Due is true only when Archive is set and has not been proven.
|
||||
Due bool
|
||||
// Archive is the settled candidate A ("" when the tier holds none).
|
||||
Archive string
|
||||
// Landed is when A landed on the tier (zero when Archive is "").
|
||||
Landed time.Time
|
||||
// ProvenArchive is what the state says was last proven on this tier ("" = nothing/legacy).
|
||||
ProvenArchive string
|
||||
// Err is a candidate-lookup failure. A tier whose archives cannot be listed is UNKNOWN — it is
|
||||
// NEVER reported as "not due", which would silently retire a tier the moment its storage
|
||||
// stopped answering. Due stays false (we have no archive to test) and the error travels.
|
||||
Err error
|
||||
// Reason is the one-line human account of this verdict.
|
||||
Reason string
|
||||
}
|
||||
|
||||
// String renders a verdict for the operator log / selftest output.
|
||||
func (v DueVerdict) String() string {
|
||||
return fmt.Sprintf("tier=%s due=%v archive=%q reason=%s", v.Target, v.Due, v.Archive, v.Reason)
|
||||
}
|
||||
|
||||
// EvaluateDue returns the due verdict for every configured tier, ordered oldest-proven first.
|
||||
//
|
||||
// Ordering is the R-85 rotation, demoted to a TIE-BREAK: it no longer decides whether a test
|
||||
// happens (due-ness does), only which of several due tiers goes first. Keeping it means a tier can
|
||||
// still never be starved — a tier that has waited longest is served first — and keeping it as the
|
||||
// order rather than as the trigger is the whole of this change.
|
||||
func (s *Scheduler) EvaluateDue(ctx context.Context) []DueVerdict {
|
||||
if !s.rotating() {
|
||||
return nil
|
||||
}
|
||||
order := s.tiers
|
||||
if s.rtState != nil {
|
||||
order = s.rtState.OldestFirst(s.tiers)
|
||||
}
|
||||
cutoff := s.settleCutoff()
|
||||
out := make([]DueVerdict, 0, len(order))
|
||||
for _, target := range order {
|
||||
out = append(out, s.evaluateTier(ctx, target, cutoff))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// settleCutoff is the newest landing time an archive may have and still count as settled.
|
||||
func (s *Scheduler) settleCutoff() time.Time {
|
||||
if s.settle <= 0 {
|
||||
return time.Time{} // no settle requirement configured → any archive is a candidate
|
||||
}
|
||||
return s.now().Add(-s.settle)
|
||||
}
|
||||
|
||||
// evaluateTier is the per-tier due-check. PURE given the picker and the state, so the rule is
|
||||
// unit-tested directly rather than inferred from whether a fake runner happened to be called.
|
||||
func (s *Scheduler) evaluateTier(ctx context.Context, target string, cutoff time.Time) DueVerdict {
|
||||
v := DueVerdict{Target: target}
|
||||
archive, landed, err := s.tierPick(ctx, target, cutoff)
|
||||
if err != nil {
|
||||
// UNKNOWN, never "not due", and never silent.
|
||||
v.Err = err
|
||||
v.Reason = fmt.Sprintf("candidate lookup FAILED (%v) — tier is unknown this evaluation, not proven and not dismissed", err)
|
||||
return v
|
||||
}
|
||||
v.Archive, v.Landed = archive, landed
|
||||
if archive == "" {
|
||||
v.Reason = "no settled archive yet — nothing to prove (newborn or still settling)"
|
||||
return v
|
||||
}
|
||||
proven, ok := "", false
|
||||
if s.rtState != nil {
|
||||
proven, ok = s.rtState.ProvenArchive(target)
|
||||
}
|
||||
v.ProvenArchive = proven
|
||||
if ok && proven == archive {
|
||||
v.Reason = fmt.Sprintf("newest settled archive (landed %s) is already proven", landed.Format(time.RFC3339))
|
||||
return v
|
||||
}
|
||||
v.Due = true
|
||||
switch {
|
||||
case !ok && proven == "":
|
||||
v.Reason = fmt.Sprintf("newest settled archive (landed %s) has not been proven; nothing proven on this tier yet", landed.Format(time.RFC3339))
|
||||
default:
|
||||
v.Reason = fmt.Sprintf("newest settled archive (landed %s) has not been proven (last proven archive was a different one)", landed.Format(time.RFC3339))
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// EvaluateDueTier is EvaluateDue for ONE named tier — the selftest's per-tier cost probe, so the
|
||||
// WAN leg of an offsite lookup is attributable rather than buried in an aggregate.
|
||||
func (s *Scheduler) EvaluateDueTier(ctx context.Context, target string) DueVerdict {
|
||||
return s.evaluateTier(ctx, target, s.settleCutoff())
|
||||
}
|
||||
|
||||
// verdictSummary renders one compact line of per-tier verdicts for the "nothing due" log.
|
||||
//
|
||||
// It re-evaluates rather than threading the verdicts out of pickForThisRun, and that is a
|
||||
// deliberate trade: this runs only on the path where NOTHING is due, so the cost is one extra
|
||||
// storage listing per tier on an otherwise idle evaluation (measured 18 ms local / 392 ms offsite,
|
||||
// R-86 Part 1.4), and in exchange the logging path cannot drift from the deciding path by holding a
|
||||
// stale copy of it. If that cost ever matters, pass the verdicts in — do not let the two diverge.
|
||||
func (s *Scheduler) verdictSummary(ctx context.Context) string {
|
||||
out := ""
|
||||
for _, v := range s.EvaluateDue(ctx) {
|
||||
if out != "" {
|
||||
out += "; "
|
||||
}
|
||||
switch {
|
||||
case v.Err != nil:
|
||||
out += v.Target + ": UNKNOWN (" + v.Err.Error() + ")"
|
||||
default:
|
||||
out += v.Target + ": " + v.Reason
|
||||
}
|
||||
}
|
||||
if out == "" {
|
||||
return "no tiers configured"
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
)
|
||||
|
||||
// R-86 — the restore-test follows the BACKUP, not the clock.
|
||||
//
|
||||
// Every test here DRIVES time (`s.now` is injected and stepped) rather than waiting for it. A test
|
||||
// that slept could not say anything about a 24-hour rule in under 24 hours, and one that only
|
||||
// asserted "no error" would pass against a scheduler that never ran anything at all — which is
|
||||
// precisely the failure mode §8.1's trap produces. So the assertions are: did a test run, on WHICH
|
||||
// archive, and did a second evaluation correctly run NOTHING.
|
||||
|
||||
// ── the fake tier storage ────────────────────────────────────────────────────────────────────
|
||||
|
||||
// archiveStub is one archive on a tier: its volid and when it landed.
|
||||
type archiveStub struct {
|
||||
volid string
|
||||
landed time.Time
|
||||
}
|
||||
|
||||
// tierStorage is a TierPicker over per-tier archive lists. It implements the SAME contract as the
|
||||
// production picker (*BackupRunner).PickSettledRestoreCandidateOn — newest archive that landed at
|
||||
// or before the cutoff — which is itself covered against a fake PVE API in backup_test.go, and
|
||||
// end-to-end by the live run. Naming the seam explicitly: everything below is true up to this
|
||||
// picker; that the real picker obeys the same rule is asserted there, not here.
|
||||
type tierStorage struct {
|
||||
archives map[string][]archiveStub
|
||||
err map[string]error // target → lookup failure
|
||||
}
|
||||
|
||||
func (ts *tierStorage) pick(_ context.Context, target string, notAfter time.Time) (string, time.Time, error) {
|
||||
if e, ok := ts.err[target]; ok && e != nil {
|
||||
return "", time.Time{}, e
|
||||
}
|
||||
var best archiveStub
|
||||
for _, a := range ts.archives[target] {
|
||||
if !notAfter.IsZero() && a.landed.After(notAfter) {
|
||||
continue // not settled yet
|
||||
}
|
||||
if best.volid == "" || a.landed.After(best.landed) {
|
||||
best = a
|
||||
}
|
||||
}
|
||||
return best.volid, best.landed, nil
|
||||
}
|
||||
|
||||
// dueHarness is a scheduler with a driven clock over a fake tier storage.
|
||||
type dueHarness struct {
|
||||
s *Scheduler
|
||||
rr *rotRunner
|
||||
st *RestoreTestState
|
||||
ts *tierStorage
|
||||
clock time.Time
|
||||
path string
|
||||
}
|
||||
|
||||
func newDueHarness(t *testing.T, start time.Time, settle time.Duration, pass bool, tiers []string, ts *tierStorage) *dueHarness {
|
||||
t.Helper()
|
||||
return newDueHarnessAt(t, filepath.Join(t.TempDir(), "rt.json"), start, settle, pass, tiers, ts)
|
||||
}
|
||||
|
||||
func newDueHarnessAt(t *testing.T, statePath string, start time.Time, settle time.Duration, pass bool, tiers []string, ts *tierStorage) *dueHarness {
|
||||
t.Helper()
|
||||
h := &dueHarness{rr: &rotRunner{pass: pass}, ts: ts, clock: start, path: statePath}
|
||||
h.st = NewRestoreTestState(statePath)
|
||||
h.s = NewScheduler(SchedulerOptions{
|
||||
Runner: h.rr,
|
||||
Store: NewStore(),
|
||||
Spec: func(_ context.Context, archive string) reconcile.RestoreTestSpec {
|
||||
return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009}
|
||||
},
|
||||
Cadence: time.Hour,
|
||||
Settle: settle,
|
||||
Logger: quiet(),
|
||||
Tiers: tiers,
|
||||
TierPick: ts.pick,
|
||||
State: h.st,
|
||||
InFlight: &InFlight{},
|
||||
})
|
||||
h.s.now = func() time.Time { return h.clock }
|
||||
return h
|
||||
}
|
||||
|
||||
// advance steps the clock by step, evaluating once at every step — the scheduler's real shape.
|
||||
func (h *dueHarness) advance(step, total time.Duration) {
|
||||
for elapsed := time.Duration(0); elapsed < total; elapsed += step {
|
||||
h.clock = h.clock.Add(step)
|
||||
h.s.tick(context.Background())
|
||||
}
|
||||
}
|
||||
|
||||
var day0 = time.Date(2026, 8, 1, 2, 0, 0, 0, time.UTC)
|
||||
|
||||
// dailyArchives lands one archive a day at 02:00 for n days, starting at day0.
|
||||
func dailyArchives(tier string, n int) []archiveStub {
|
||||
out := make([]archiveStub, 0, n)
|
||||
for d := 0; d < n; d++ {
|
||||
out = append(out, archiveStub{
|
||||
volid: fmt.Sprintf("%s:backup/vzdump-lxc-9201-day%d.tar.zst", tier, d),
|
||||
landed: day0.AddDate(0, 0, d),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ── SCENARIO A — a daily tier is proved daily, on its own archive ────────────────────────────
|
||||
//
|
||||
// THE TRAP THIS PINS (§8.1). R-86 reads "trigger a tier ~24 h after its own newest archive", and
|
||||
// the literal implementation of that — *due when the newest archive is at least `settle` old* — is
|
||||
// NEVER true on a daily tier: a new archive lands every day, so the newest archive's age resets to
|
||||
// zero long before it reaches 24 h. The literal reading silently switches restore-testing OFF for
|
||||
// the tier that matters most.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-03). In Scheduler.evaluateTier, the per-archive comparison
|
||||
// was replaced by the naive age rule:
|
||||
//
|
||||
// - if ok && proven == archive { … not due … }
|
||||
// - if s.now().Sub(landed) < s.settle { … not due … } // and the proven-archive check deleted
|
||||
//
|
||||
// and the picker cutoff was removed (`cutoff := time.Time{}`), i.e. exactly "is the newest archive
|
||||
// old enough". Result:
|
||||
//
|
||||
// --- FAIL: TestDue_DailyTierIsProvedDailyOnItsOwnArchive
|
||||
// restoretest_due_test.go: a daily tier must be proved once per day; got 0 run(s) over 5 days
|
||||
//
|
||||
// Zero runs — restore-testing off. Restored immediately afterwards.
|
||||
func TestDue_DailyTierIsProvedDailyOnItsOwnArchive(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 6)}}
|
||||
h := newDueHarness(t, day0.Add(time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
||||
|
||||
// Five days, evaluated hourly.
|
||||
h.advance(time.Hour, 5*24*time.Hour)
|
||||
|
||||
got := h.rr.seen()
|
||||
if len(got) != 5 {
|
||||
t.Fatalf("a daily tier must be proved once per day; got %d run(s) over 5 days: %v", len(got), got)
|
||||
}
|
||||
// And each run must be on the archive that settled that day — day0's on day 1, and so on.
|
||||
for i, a := range got {
|
||||
want := fmt.Sprintf("local:backup/vzdump-lxc-9201-day%d.tar.zst", i)
|
||||
if a != want {
|
||||
t.Fatalf("run %d tested %q, want %q — the test is not following the archive", i+1, a, want)
|
||||
}
|
||||
}
|
||||
// The newest archive is NEVER the one tested: it has not settled.
|
||||
if last := got[len(got)-1]; last == "local:backup/vzdump-lxc-9201-day5.tar.zst" {
|
||||
t.Fatal("the still-settling archive was tested — the settle cutoff is not being applied")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO B — a weekly tier is proved weekly, not every other day ─────────────────────────
|
||||
func TestDue_WeeklyTierIsProvedOncePerArchive(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": {
|
||||
{volid: "felhom-pbs:backup/ct/9201/w0", landed: day0},
|
||||
{volid: "felhom-pbs:backup/ct/9201/w1", landed: day0.AddDate(0, 0, 7)},
|
||||
{volid: "felhom-pbs:backup/ct/9201/w2", landed: day0.AddDate(0, 0, 14)},
|
||||
}}}
|
||||
h := newDueHarness(t, day0.Add(time.Hour), 24*time.Hour, true, []string{"felhom-pbs"}, ts)
|
||||
|
||||
// Three weeks, evaluated every 6 hours — 84 evaluations.
|
||||
h.advance(6*time.Hour, 21*24*time.Hour)
|
||||
|
||||
got := h.rr.seen()
|
||||
want := []string{
|
||||
"felhom-pbs:backup/ct/9201/w0",
|
||||
"felhom-pbs:backup/ct/9201/w1",
|
||||
"felhom-pbs:backup/ct/9201/w2",
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("a weekly tier must be proved ONCE PER ARCHIVE (3 archives over 3 weeks); got %d run(s): %v", len(got), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("run %d tested %q, want %q", i+1, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO C — an agent restart does not change the schedule ───────────────────────────────
|
||||
//
|
||||
// This is the defect a person actually notices: today every deploy restarts the ticker, so a
|
||||
// restore-test runs one interval after each deploy regardless of what has already been proven.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-03): revert the state to per-tier TIME by making
|
||||
// ProvenArchive ignore the stored archive —
|
||||
//
|
||||
// - if !ok || p.Archive == "" { return "", false }
|
||||
// - return "", false // per-tier time only, the pre-R-86 state
|
||||
//
|
||||
// → --- FAIL: TestDue_RestartRunsNothing
|
||||
//
|
||||
// restoretest_due_test.go:226: an agent restart must not trigger a restore-test; 2 restart(s)
|
||||
// produced 4 run(s)
|
||||
//
|
||||
// Four: the same already-proven archive re-tested on EVERY evaluation after EVERY restart, which is
|
||||
// today's behaviour with the ticker's phase reset by the deploy. Restored.
|
||||
func TestDue_RestartRunsNothing(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "rt.json")
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 2)}}
|
||||
start := day0.AddDate(0, 0, 1).Add(time.Hour) // day 1, 03:00 — day0's archive has settled
|
||||
|
||||
h := newDueHarnessAt(t, path, start, 24*time.Hour, true, []string{"local"}, ts)
|
||||
h.s.tick(context.Background())
|
||||
if n := len(h.rr.seen()); n != 1 {
|
||||
t.Fatalf("precondition: the settled archive should have been proved once; got %d run(s)", n)
|
||||
}
|
||||
|
||||
// --- two restarts: brand-new scheduler + brand-new state object over the SAME file ---
|
||||
total := 0
|
||||
for i := 0; i < 2; i++ {
|
||||
h2 := newDueHarnessAt(t, path, start.Add(time.Duration(i+1)*time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
||||
h2.s.tick(context.Background())
|
||||
h2.s.tick(context.Background())
|
||||
total += len(h2.rr.seen())
|
||||
}
|
||||
if total != 0 {
|
||||
t.Fatalf("an agent restart must not trigger a restore-test; 2 restart(s) produced %d run(s)", total)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO D — a new archive makes a tier due even if it was tested yesterday ──────────────
|
||||
func TestDue_NewSettledArchiveMakesAProvedTierDueAgain(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 2)}}
|
||||
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
||||
|
||||
h.s.tick(context.Background()) // proves day0's archive
|
||||
h.s.tick(context.Background()) // nothing new has settled → nothing
|
||||
if n := len(h.rr.seen()); n != 1 {
|
||||
t.Fatalf("want exactly 1 run before the new archive settles, got %d: %v", n, h.rr.seen())
|
||||
}
|
||||
|
||||
// Day 2, 03:00 — day1's archive has now settled.
|
||||
h.clock = day0.AddDate(0, 0, 2).Add(time.Hour)
|
||||
h.s.tick(context.Background())
|
||||
|
||||
got := h.rr.seen()
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("a newly settled archive must make the tier due again; got %v", got)
|
||||
}
|
||||
if got[1] != "local:backup/vzdump-lxc-9201-day1.tar.zst" {
|
||||
t.Fatalf("the NEW archive must be the one tested; got %q", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO E — a failing tier keeps being retried, and earns no proof ──────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-03): give credit on failure in Scheduler.tick —
|
||||
//
|
||||
// - if rt.Pass && s.rtState != nil && target != "" {
|
||||
// - if s.rtState != nil && target != "" {
|
||||
//
|
||||
// → --- FAIL: TestDue_FailingTierIsRetriedAndNeverProven
|
||||
//
|
||||
// restoretest_due_test.go: a failing tier must keep being retried; got 1 run(s) over 3
|
||||
// evaluations
|
||||
//
|
||||
// A single failure would have retired the archive as proven — a permanently broken DR tier looking
|
||||
// freshly verified, which is the loudest signal this system produces going silent. Restored.
|
||||
func TestDue_FailingTierIsRetriedAndNeverProven(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"local": dailyArchives("local", 1)}}
|
||||
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, false, []string{"local"}, ts)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
h.s.tick(context.Background())
|
||||
}
|
||||
|
||||
got := h.rr.seen()
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("a failing tier must keep being retried; got %d run(s) over 3 evaluations: %v", len(got), got)
|
||||
}
|
||||
if _, ok := h.st.ProvenArchive("local"); ok {
|
||||
t.Fatal("a FAILED restore-test must not record the archive as proven")
|
||||
}
|
||||
if _, ok := h.st.LastSuccess("local"); ok {
|
||||
t.Fatal("a FAILED restore-test must not stamp the tier as proven")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO F — two tiers due at once do not run at once ────────────────────────────────────
|
||||
func TestDue_TwoDueTiersRunOneAtATime(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{
|
||||
"local": {{volid: "local:backup/a.tar.zst", landed: day0}},
|
||||
"felhom-pbs": {{volid: "felhom-pbs:backup/ct/9201/a", landed: day0}},
|
||||
}}
|
||||
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts)
|
||||
|
||||
// Both tiers are due at this instant.
|
||||
due := h.s.EvaluateDue(context.Background())
|
||||
if len(due) != 2 || !due[0].Due || !due[1].Due {
|
||||
t.Fatalf("precondition: both tiers should be due; got %v", due)
|
||||
}
|
||||
|
||||
h.s.tick(context.Background())
|
||||
if n := len(h.rr.seen()); n != 1 {
|
||||
t.Fatalf("ONE evaluation must start ONE restore-test, never two multi-GB restores over one link; got %d: %v", n, h.rr.seen())
|
||||
}
|
||||
|
||||
// The other tier was DEFERRED, not cancelled: it is still due and runs on the next evaluation.
|
||||
h.s.tick(context.Background())
|
||||
got := h.rr.seen()
|
||||
if len(got) != 2 || got[0] == got[1] {
|
||||
t.Fatalf("the deferred tier must run on the NEXT evaluation, on its own archive; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The heavy-operation gate still holds, and a tier deferred behind a backup stays DUE.
|
||||
func TestDue_DeferredBehindABackupStaysDue(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"local": {{volid: "local:backup/a.tar.zst", landed: day0}}}}
|
||||
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
||||
|
||||
gate := &InFlight{}
|
||||
h.s.inFlight = gate
|
||||
release, _, _ := gate.TryAcquire("backup:felhom-pbs")
|
||||
|
||||
h.s.tick(context.Background())
|
||||
if n := len(h.rr.seen()); n != 0 {
|
||||
t.Fatalf("the restore-test must DEFER while a backup holds the gate; got %d run(s)", n)
|
||||
}
|
||||
if due := h.s.EvaluateDue(context.Background()); !due[0].Due {
|
||||
t.Fatal("a deferred tier must remain DUE — deferral is not dismissal")
|
||||
}
|
||||
release()
|
||||
h.s.tick(context.Background())
|
||||
if n := len(h.rr.seen()); n != 1 {
|
||||
t.Fatalf("must resume once the gate frees; got %d run(s)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO H — a newborn box is UNKNOWN, not stale and not a fault ─────────────────────────
|
||||
func TestDue_NewbornTierIsNotDueAndNotAnError(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": nil}}
|
||||
h := newDueHarness(t, day0, 24*time.Hour, true, []string{"felhom-pbs"}, ts)
|
||||
|
||||
due := h.s.EvaluateDue(context.Background())
|
||||
if len(due) != 1 {
|
||||
t.Fatalf("want one verdict, got %v", due)
|
||||
}
|
||||
v := due[0]
|
||||
if v.Due || v.Err != nil || v.Archive != "" {
|
||||
t.Fatalf("a tier with no archive is UNKNOWN — not due, not an error; got %+v", v)
|
||||
}
|
||||
if v.Reason == "" {
|
||||
t.Fatal("every verdict must carry a reason — a due-check that cannot say why is a quiet path")
|
||||
}
|
||||
h.s.tick(context.Background())
|
||||
if n := len(h.rr.seen()); n != 0 {
|
||||
t.Fatalf("a newborn tier must not be restore-tested; got %d run(s)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// An archive that exists but has NOT settled yet is not a candidate — and that is not an error.
|
||||
func TestDue_UnsettledArchiveIsNotACandidate(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"local": {{volid: "local:backup/fresh.tar.zst", landed: day0}}}}
|
||||
h := newDueHarness(t, day0.Add(2*time.Hour), 24*time.Hour, true, []string{"local"}, ts)
|
||||
|
||||
if v := h.s.EvaluateDue(context.Background())[0]; v.Due || v.Archive != "" {
|
||||
t.Fatalf("an archive 2h old must not be a candidate under a 24h settle lag; got %+v", v)
|
||||
}
|
||||
h.s.tick(context.Background())
|
||||
if n := len(h.rr.seen()); n != 0 {
|
||||
t.Fatalf("nothing settled → no run; got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A tier whose archives cannot be LISTED is UNKNOWN — never silently "not due", and never silent.
|
||||
// Treating a lookup failure as "not due" would retire a tier the moment its storage stopped
|
||||
// answering, which is the same absence-is-not-evidence error this monitor family keeps making.
|
||||
func TestDue_LookupFailureIsUnknownNotNotDue(t *testing.T) {
|
||||
boom := errors.New("storage unreachable")
|
||||
ts := &tierStorage{
|
||||
archives: map[string][]archiveStub{"local": {{volid: "local:backup/a.tar.zst", landed: day0}}},
|
||||
err: map[string]error{"felhom-pbs": boom},
|
||||
}
|
||||
h := newDueHarness(t, day0.AddDate(0, 0, 1), 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts)
|
||||
|
||||
var pbs DueVerdict
|
||||
for _, v := range h.s.EvaluateDue(context.Background()) {
|
||||
if v.Target == "felhom-pbs" {
|
||||
pbs = v
|
||||
}
|
||||
}
|
||||
if pbs.Err == nil {
|
||||
t.Fatal("a lookup failure must travel in the verdict, not be swallowed")
|
||||
}
|
||||
if pbs.Due {
|
||||
t.Fatal("a tier we could not list must not be reported DUE — we have no archive to test")
|
||||
}
|
||||
if pbs.Reason == "" {
|
||||
t.Fatal("the failure must be explained, not merely flagged")
|
||||
}
|
||||
|
||||
// And the OTHER tier still runs: one tier's storage being unreadable must not cost the other
|
||||
// tier its proof.
|
||||
h.s.tick(context.Background())
|
||||
if got := h.rr.seen(); len(got) != 1 || got[0] != "local:backup/a.tar.zst" {
|
||||
t.Fatalf("the readable tier must still be proved; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ── the state's migration (§8.2) ─────────────────────────────────────────────────────────────
|
||||
|
||||
// A pre-R-86 state file carries a TIME and no archive. It must keep its time (rotation ordering
|
||||
// survives the upgrade) and yield NO proven archive, so each tier is due exactly once. Reading a
|
||||
// legacy time as proof of the CURRENT archive would mark an unproven archive proven — a guarantee
|
||||
// invented by a migration.
|
||||
func TestRestoreTestState_LegacyFileMigratesToNothingProven(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "rt.json")
|
||||
legacy := `{"local":"2026-08-01T02:00:00Z","felhom-pbs":"2026-07-30T02:00:00Z"}`
|
||||
if err := writeFileForTest(path, legacy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
st := NewRestoreTestState(path)
|
||||
if _, ok := st.ProvenArchive("local"); ok {
|
||||
t.Fatal("a legacy record names no archive — it must NOT be read as proof of the current one")
|
||||
}
|
||||
at, ok := st.LastSuccess("local")
|
||||
if !ok || !at.Equal(time.Date(2026, 8, 1, 2, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("the legacy TIME must survive (rotation ordering depends on it); got %v ok=%v", at, ok)
|
||||
}
|
||||
// Ordering still works off the legacy times.
|
||||
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" {
|
||||
t.Fatalf("oldest-first must still order legacy records; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The new shape round-trips, archive and all.
|
||||
func TestRestoreTestState_ArchiveRoundTrips(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "rt.json")
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
st := NewRestoreTestState(path)
|
||||
if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", "pbs", "boot+running", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
re := NewRestoreTestState(path)
|
||||
got, ok := re.ProvenArchive("felhom-pbs")
|
||||
if !ok || got != "felhom-pbs:backup/ct/9201/x" {
|
||||
t.Fatalf("the proven ARCHIVE must survive a restart; got %q ok=%v", got, ok)
|
||||
}
|
||||
at, ok := re.LastSuccess("felhom-pbs")
|
||||
if !ok || !at.Equal(now) {
|
||||
t.Fatalf("the proven TIME must survive too; got %v ok=%v", at, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// writeFileForTest is a tiny helper so the legacy-migration fixture reads clearly above.
|
||||
func writeFileForTest(path, content string) error {
|
||||
return os.WriteFile(path, []byte(content), 0o600)
|
||||
}
|
||||
|
||||
// Standing rule 3: an absent log line is not evidence. "Nothing is due" is now the NORMAL outcome of
|
||||
// an evaluation, so it must produce a POSITIVE observable naming each tier's verdict — otherwise a
|
||||
// quiet journal is equally consistent with a healthy loop and a dead goroutine.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-03): drop the summary back to a bare
|
||||
// `s.logger.Debug("backup: restore-test not due this evaluation")` and this fails with
|
||||
// "a not-due evaluation must name each tier's verdict; got \"\"" — i.e. nothing at INFO at all.
|
||||
func TestDue_NothingDueStillNamesEveryTiersVerdict(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{
|
||||
"local": {{volid: "local:backup/a.tar.zst", landed: day0}},
|
||||
"felhom-pbs": nil, // no archive at all
|
||||
}}
|
||||
h := newDueHarness(t, day0.AddDate(0, 0, 1), 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts)
|
||||
// Prove the local tier so NOTHING is due.
|
||||
if err := h.st.RecordSuccess("local", "local:backup/a.tar.zst", "local", "boot+running", h.clock); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Assert what the SCHEDULER emits on a real evaluation, not what a helper returns — a helper
|
||||
// test would pass against a tick that never calls it.
|
||||
var logbuf strings.Builder
|
||||
h.s.logger = slog.New(slog.NewTextHandler(&logbuf, &slog.HandlerOptions{Level: slog.LevelInfo}))
|
||||
h.s.tick(context.Background())
|
||||
got := logbuf.String()
|
||||
for _, want := range []string{"local", "felhom-pbs", "already proven", "no settled archive"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("a not-due evaluation must name each tier's verdict; got %q (missing %q)", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A tier whose storage cannot be listed must say UNKNOWN in that same line — a lookup failure that
|
||||
// reads as "nothing due" is the silence this rule exists to prevent.
|
||||
func TestDue_VerdictSummaryNamesAnUnknownTier(t *testing.T) {
|
||||
ts := &tierStorage{
|
||||
archives: map[string][]archiveStub{"local": nil},
|
||||
err: map[string]error{"felhom-pbs": errors.New("storage unreachable")},
|
||||
}
|
||||
h := newDueHarness(t, day0, 24*time.Hour, true, []string{"local", "felhom-pbs"}, ts)
|
||||
got := h.s.verdictSummary(context.Background())
|
||||
if !strings.Contains(got, "UNKNOWN") || !strings.Contains(got, "storage unreachable") {
|
||||
t.Fatalf("an unlistable tier must read as UNKNOWN with its error; got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ── R-189 — the persisted proof must be REPORTABLE, and must refuse to lie ───────────────────
|
||||
//
|
||||
// A proof held only in the in-memory store dies with the process, and under per-archive due-ness the
|
||||
// agent will not repeat the work. So the persisted record has to be able to become a host-report
|
||||
// entry — without inventing anything it does not know.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-03): drop the `reportable()` filter from
|
||||
// ProvenRestoreTests, so a pre-R-189 record (archive but no tier) is emitted →
|
||||
//
|
||||
// --- FAIL: TestProvenRestoreTests_RefusesToReportWhatItCannotDescribe
|
||||
// restoretest_due_test.go: a record with no TIER must not be reported (the hub keys its
|
||||
// per-tier proof on it); got [{... SourceTier: ...}]
|
||||
//
|
||||
// Restored.
|
||||
func TestProvenRestoreTests_RefusesToReportWhatItCannotDescribe(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "rt.json")
|
||||
// v1 (a bare time), v2 (archive, no tier) and v3 (complete) side by side — every shape this
|
||||
// file has ever had, which is what a real box carries after two upgrades.
|
||||
legacy := `{
|
||||
"old-v1": "2026-07-30T02:11:07Z",
|
||||
"old-v2": {"archive":"felhom-backup:backup/vzdump-lxc-9201-a.tar.zst","proven_at":"2026-08-01T04:41:58Z"},
|
||||
"felhom-pbs": {"archive":"felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z","tier":"pbs","verified":"boot+running","proven_at":"2026-08-03T13:25:14Z"}
|
||||
}`
|
||||
if err := writeFileForTest(path, legacy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := NewRestoreTestState(path).ProvenRestoreTests(context.Background())
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("only the record that can be described honestly may be reported; got %d: %+v", len(got), got)
|
||||
}
|
||||
e := got[0]
|
||||
if e.SourceTier != "pbs" {
|
||||
t.Fatalf("a record with no TIER must not be reported (the hub keys its per-tier proof on it); got %+v", got)
|
||||
}
|
||||
if e.SourceArchive != "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z" || !e.Pass {
|
||||
t.Fatalf("the reported entry must be the stored proof, unchanged; got %+v", e)
|
||||
}
|
||||
if e.TestedAt != "2026-08-03T13:25:14Z" {
|
||||
t.Fatalf("the entry must carry the time the run passed, not now(); got %q", e.TestedAt)
|
||||
}
|
||||
if e.Verified != "boot+running" {
|
||||
t.Fatalf("what the run verified must survive the round trip; got %q", e.Verified)
|
||||
}
|
||||
// Run mechanics are NOT invented: an absent duration is not a claim, a fabricated one would be.
|
||||
if e.DurationSeconds != 0 || e.ScratchVMID != 0 {
|
||||
t.Fatalf("the re-report must not invent run mechanics it never stored; got duration=%v scratch=%d",
|
||||
e.DurationSeconds, e.ScratchVMID)
|
||||
}
|
||||
// The legacy records still serve the DUE-check, which is a separate question from reporting.
|
||||
if _, ok := NewRestoreTestState(path).ProvenArchive("old-v2"); !ok {
|
||||
t.Fatal("a v2 record must still answer the due-check even though it cannot be reported")
|
||||
}
|
||||
}
|
||||
|
||||
// A tier proved through the SCHEDULER (not by hand) lands in the state complete enough to report —
|
||||
// the production path, not a hand-built fixture.
|
||||
func TestScheduler_ProofIsRecordedReportably(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": {{volid: "felhom-pbs:backup/ct/9201/w0", landed: day0}}}}
|
||||
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(97*time.Minute), 24*time.Hour, true, []string{"felhom-pbs"}, ts)
|
||||
// The fake runner echoes the spec's tier; give the spec a tier the way main.go does.
|
||||
h.s.spec = func(_ context.Context, archive string) reconcile.RestoreTestSpec {
|
||||
return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009, SourceTier: "pbs"}
|
||||
}
|
||||
h.s.tick(context.Background())
|
||||
|
||||
got := h.st.ProvenRestoreTests(context.Background())
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a scheduled pass must leave a REPORTABLE proof; got %d: %+v", len(got), got)
|
||||
}
|
||||
if got[0].SourceTier != "pbs" || got[0].SourceArchive != "felhom-pbs:backup/ct/9201/w0" {
|
||||
t.Fatalf("the proof must name the tier and the archive the run used; got %+v", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A FAILED run leaves nothing to report — the asymmetry of §8.1, asserted rather than assumed.
|
||||
func TestScheduler_AFailureLeavesNoPersistedProof(t *testing.T) {
|
||||
ts := &tierStorage{archives: map[string][]archiveStub{"felhom-pbs": {{volid: "felhom-pbs:backup/ct/9201/w0", landed: day0}}}}
|
||||
h := newDueHarness(t, day0.AddDate(0, 0, 1).Add(97*time.Minute), 24*time.Hour, false, []string{"felhom-pbs"}, ts)
|
||||
h.s.tick(context.Background())
|
||||
if got := h.st.ProvenRestoreTests(context.Background()); len(got) != 0 {
|
||||
t.Fatalf("a FAILED run must persist nothing — a failing tier is retried, and a stored failure "+
|
||||
"would outlive the fault; got %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// RestoreTestState persists the last SUCCESSFUL restore-test per backup tier.
|
||||
//
|
||||
// R-85 (1.4). This one genuinely needs PERSISTENCE, unlike R-84 — and the difference is worth
|
||||
// stating, because the two look like the same problem and are not:
|
||||
//
|
||||
// - R-84 (backup freshness) had a GROUND TRUTH to consult: the archive is still on the storage,
|
||||
// so the agent could ask "when did a backup last land?" and never persist anything. That is
|
||||
// strictly better, because a pruned archive correctly stops counting.
|
||||
// - A restore-test leaves NO artifact — the scratch guest is destroyed as its final act. There is
|
||||
// nothing to query. "Did we prove this tier restores?" exists only as remembered state, so it
|
||||
// must be written down or it is lost.
|
||||
//
|
||||
// Why it must survive a restart: rotation is oldest-first (the operator ruling), so an in-memory map
|
||||
// would reset every tier to "never tested" on each restart. Ordering would then depend on map
|
||||
// iteration order, and one tier could be starved indefinitely while the other is re-tested — with
|
||||
// agent deploys as routine as they are, that is not a corner case.
|
||||
//
|
||||
// Only SUCCESS is recorded. A failed run must not satisfy rotation, or a tier that fails every time
|
||||
// would look freshly proven and stop being retried — the same "a failure satisfies the cadence"
|
||||
// trap the backup due-check avoids. R-86 keeps that property unchanged and gives it a second job:
|
||||
// the due-check reads this state, so a failure that recorded proof would ALSO stop the tier from
|
||||
// ever becoming due again. The rule earns its keep twice now.
|
||||
//
|
||||
// R-86 (1.2) — WHICH ARCHIVE, not just when.
|
||||
//
|
||||
// A timestamp alone cannot answer the question the due-check asks. "This tier passed at 04:00" is
|
||||
// consistent both with "yesterday's archive is proven" and with "an archive from a week ago is
|
||||
// proven and nothing since has been looked at". Restore-testing is now per ARCHIVE GENERATION —
|
||||
// a tier is due once it holds a settled archive that has not been proven — so the identity of the
|
||||
// proven archive is the state, and the time is metadata (rotation ordering, operator reporting).
|
||||
//
|
||||
// This is the same class as the workspace rule "a timestamp records an ATTEMPT, not a RESULT":
|
||||
// here it records a result, but not WHICH result, and that is just as unable to answer the question
|
||||
// being asked of it.
|
||||
type RestoreTestState struct {
|
||||
path string
|
||||
mu sync.Mutex
|
||||
last map[string]provenTier // target id → what was last PROVEN on that tier
|
||||
}
|
||||
|
||||
// provenTier is one tier's proof: the archive that passed, which tier it was, what was verified,
|
||||
// and when.
|
||||
//
|
||||
// R-189 added `Tier` and `Verified`. Until then this record could answer the DUE-check but could not
|
||||
// be REPORTED, and being reportable is what closes R-189: a proof held only in the in-memory result
|
||||
// store vanishes on restart, and under per-archive due-ness the box will not repeat the work, so the
|
||||
// hub can stay ignorant of a real success until the next archive generation.
|
||||
//
|
||||
// `Tier` is stored rather than derived because it is known for certain at proof time (the run's own
|
||||
// spec used it to choose the restore timeout) and deriving it later would need a storage-type lookup
|
||||
// at report-building time — a network call that can fail, on a path where failing means mis-labelling
|
||||
// a proof. Store what you knew when you knew it.
|
||||
type provenTier struct {
|
||||
Archive string // volid of the archive that PASSED; "" = a legacy record with no archive
|
||||
Tier string // "local" | "pbs" — as the run reported it; "" = pre-R-189 record
|
||||
Verified string // what the run verified (e.g. "boot+running"); "" = pre-R-189 record
|
||||
At time.Time // when that run passed (UTC)
|
||||
}
|
||||
|
||||
// reportable reports whether this record can be re-reported to the hub as a restore-test result.
|
||||
//
|
||||
// It needs BOTH the archive and the tier: the hub keys its edge-triggered failure state on the
|
||||
// archive and its per-tier proof lookup on the tier, so an entry missing either is not a usable
|
||||
// proof — and emitting one anyway would be a report the hub cannot act on, dressed as evidence.
|
||||
// A pre-R-189 record is therefore silently not reported; the tier's next real proof fills it in.
|
||||
func (p provenTier) reportable() bool { return p.Archive != "" && p.Tier != "" }
|
||||
|
||||
// provenTierJSON is the on-disk shape. Two older shapes are read and neither is written:
|
||||
//
|
||||
// v1 (pre-R-86) "<target>": "<RFC3339>" — a time, no archive
|
||||
// v2 (R-86) "<target>": {archive, proven_at} — due-check usable, not reportable
|
||||
// v3 (R-189) "<target>": {archive, tier, verified, …} — both
|
||||
//
|
||||
// Fields absent in an older file unmarshal to "", which is exactly the "no usable proof" signal the
|
||||
// readers above test for — the migration needs no version number because the absence IS the answer.
|
||||
type provenTierJSON struct {
|
||||
Archive string `json:"archive"`
|
||||
Tier string `json:"tier,omitempty"`
|
||||
Verified string `json:"verified,omitempty"`
|
||||
ProvenAt string `json:"proven_at"`
|
||||
}
|
||||
|
||||
// NewRestoreTestState opens (or creates) the state at path. A missing or unreadable file is NOT an
|
||||
// error: it degrades to "nothing proven yet", which is the correct starting point and keeps a
|
||||
// corrupt file from wedging the daemon.
|
||||
//
|
||||
// MIGRATION (R-86). The pre-R-86 file is `{"<target>": "<RFC3339>"}` — a time and no archive. A
|
||||
// legacy record keeps its TIME (rotation ordering survives a deploy, which is why the file exists
|
||||
// at all) but yields NO proven archive, so every tier is due exactly once on first evaluation after
|
||||
// the upgrade. One extra restore-test per tier, once, is the safe direction: the alternative is to
|
||||
// read a legacy time as proof of whatever archive happens to be current, which would mark an
|
||||
// unproven archive proven — inventing a guarantee out of a migration.
|
||||
func NewRestoreTestState(path string) *RestoreTestState {
|
||||
s := &RestoreTestState{path: path, last: map[string]provenTier{}}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return s
|
||||
}
|
||||
var raw map[string]json.RawMessage
|
||||
if json.Unmarshal(data, &raw) != nil {
|
||||
return s
|
||||
}
|
||||
for target, msg := range raw {
|
||||
// Legacy shape: a bare RFC3339 string.
|
||||
var legacy string
|
||||
if json.Unmarshal(msg, &legacy) == nil {
|
||||
if t, perr := time.Parse(time.RFC3339, legacy); perr == nil {
|
||||
s.last[target] = provenTier{At: t.UTC()} // no archive → due once, deliberately
|
||||
}
|
||||
continue
|
||||
}
|
||||
var cur provenTierJSON
|
||||
if json.Unmarshal(msg, &cur) != nil {
|
||||
continue // one unreadable entry must not lose the others
|
||||
}
|
||||
t, perr := time.Parse(time.RFC3339, cur.ProvenAt)
|
||||
if perr != nil {
|
||||
continue
|
||||
}
|
||||
s.last[target] = provenTier{Archive: cur.Archive, Tier: cur.Tier, Verified: cur.Verified, At: t.UTC()}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// RecordSuccess stamps a tier as proven at t, naming the ARCHIVE that passed, the TIER the run
|
||||
// reported, and what it verified. Only call this for a PASSING restore-test — the archive is what
|
||||
// makes the tier not-due, so recording one for a failed run would retire the archive unproven.
|
||||
//
|
||||
// ONLY SUCCESSES ARE PERSISTED, AND THE ASYMMETRY IS DELIBERATE (R-189 §8.1). Say it here because
|
||||
// the next reader will notice failures are absent and try to "fix" it:
|
||||
//
|
||||
// a SUCCESS suppresses future work — a proven archive is never re-tested, so a lost proof leaves
|
||||
// the system quietly less tested than it believes. It must survive a restart.
|
||||
//
|
||||
// a FAILURE causes future work — a failing tier stays due and is retried at the next evaluation,
|
||||
// so a lost failure heals itself within one interval. Persisting it would do the opposite of
|
||||
// helping: a healed tier would keep reporting a failure that is no longer true.
|
||||
func (s *RestoreTestState) RecordSuccess(target, archive, tier, verified string, t time.Time) error {
|
||||
if target == "" {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.last[target] = provenTier{Archive: archive, Tier: tier, Verified: verified, At: t.UTC()}
|
||||
return s.saveLocked()
|
||||
}
|
||||
|
||||
// LastSuccess returns when this tier was last proven (ok=false = never).
|
||||
func (s *RestoreTestState) LastSuccess(target string) (time.Time, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
p, ok := s.last[target]
|
||||
return p.At, ok
|
||||
}
|
||||
|
||||
// ProvenArchive returns the archive last PROVEN on this tier (ok=false = none — either never tested,
|
||||
// or a legacy record carrying only a time). It is the due-check's whole question: an archive that is
|
||||
// not this one has not been proven.
|
||||
func (s *RestoreTestState) ProvenArchive(target string) (string, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
p, ok := s.last[target]
|
||||
if !ok || p.Archive == "" {
|
||||
return "", false
|
||||
}
|
||||
return p.Archive, true
|
||||
}
|
||||
|
||||
// Snapshot returns a copy of the last-proven TIMES.
|
||||
//
|
||||
// It carried the comment "for the host-report gauge" from the day it was written and **had no caller
|
||||
// at all** until R-189 — a seam built and never wired, and an invariant asserted in a comment with
|
||||
// nothing pinning it, in one method. The host report is now fed by ProvenRestoreTests below, which
|
||||
// carries the archive and the tier that a bare timestamp cannot. This stays for callers that want
|
||||
// only the times; if it acquires none, delete it rather than let it claim a purpose again.
|
||||
func (s *RestoreTestState) Snapshot() map[string]time.Time {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make(map[string]time.Time, len(s.last))
|
||||
for k, v := range s.last {
|
||||
out[k] = v.At
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ProvenRestoreTests renders the persisted proofs as host-report entries — the R-189 fix.
|
||||
//
|
||||
// It satisfies hub.RestoreTestReporter's shape, so the collector can merge these with the in-memory
|
||||
// results. What it emits is a RE-REPORT of a run that really happened, not a synthesis:
|
||||
//
|
||||
// - `Pass` is true because ONLY successes are stored (RecordSuccess is the sole writer);
|
||||
// - `SourceArchive`, `SourceTier`, `Verified` and `TestedAt` are the values that run reported;
|
||||
// - the run mechanics (scratch VMID, duration, warnings) are NOT re-invented. An absent duration
|
||||
// is not a claim; a fabricated one would be.
|
||||
//
|
||||
// A record that cannot be reported honestly is omitted rather than padded — see provenTier.reportable.
|
||||
// **A tier with no usable proof produces NO entry**: an unproven tier reading as proven would be a
|
||||
// worse defect than the one this fixes.
|
||||
func (s *RestoreTestState) ProvenRestoreTests(context.Context) []hub.RestoreTest {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make([]hub.RestoreTest, 0, len(s.last))
|
||||
for _, p := range s.last {
|
||||
if !p.reportable() {
|
||||
continue
|
||||
}
|
||||
out = append(out, hub.RestoreTest{
|
||||
SourceArchive: p.Archive,
|
||||
SourceTier: p.Tier,
|
||||
Pass: true,
|
||||
Verified: p.Verified,
|
||||
TestedAt: p.At.UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
// Deterministic order: the report is compared byte-wise by the contract test, and Go's map
|
||||
// iteration is randomised.
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].SourceTier < out[j].SourceTier })
|
||||
return out
|
||||
}
|
||||
|
||||
// OldestFirst orders targets by "least recently proven first"; never-proven sorts FIRST.
|
||||
//
|
||||
// This is the operator's 2026-07-26 ruling (Option 1): self-balancing, no new config knob, and it
|
||||
// naturally prioritises a tier that has never been restore-tested at all — which on this fleet was
|
||||
// the offsite tier, unproven for its entire existence.
|
||||
//
|
||||
// Ties break on target id so the order is deterministic; without that, two tiers proven in the same
|
||||
// second would rotate by map iteration order, which is randomised in Go and would make the
|
||||
// behaviour untestable and occasionally starving.
|
||||
func (s *RestoreTestState) OldestFirst(targets []string) []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := append([]string(nil), targets...)
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
pi, oki := s.last[out[i]]
|
||||
pj, okj := s.last[out[j]]
|
||||
ti, tj := pi.At, pj.At
|
||||
switch {
|
||||
case !oki && !okj:
|
||||
return out[i] < out[j] // both never proven → deterministic
|
||||
case !oki:
|
||||
return true // never proven wins
|
||||
case !okj:
|
||||
return false
|
||||
case !ti.Equal(tj):
|
||||
return ti.Before(tj)
|
||||
default:
|
||||
return out[i] < out[j]
|
||||
}
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *RestoreTestState) saveLocked() error {
|
||||
raw := make(map[string]provenTierJSON, len(s.last))
|
||||
for target, p := range s.last {
|
||||
raw[target] = provenTierJSON{
|
||||
Archive: p.Archive, Tier: p.Tier, Verified: p.Verified,
|
||||
ProvenAt: p.At.UTC().Format(time.RFC3339),
|
||||
}
|
||||
}
|
||||
data, err := json.MarshalIndent(raw, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := s.path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, s.path)
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
)
|
||||
|
||||
// R-85 Phase 2 — tier rotation, persisted state, and the one-heavy-operation gate.
|
||||
//
|
||||
// The failure this prevents is not hypothetical: demo-hp's DR tier reported `applied` with ZERO
|
||||
// snapshots for five days and nobody noticed, because the scheduler could only ever see the primary
|
||||
// tier. Rotation is what makes the offsite tier testable at all.
|
||||
|
||||
// rotRunner records which archives it was asked to restore.
|
||||
type rotRunner struct {
|
||||
mu sync.Mutex
|
||||
archives []string
|
||||
pass bool
|
||||
}
|
||||
|
||||
func (r *rotRunner) RunRestoreTest(_ context.Context, spec reconcile.RestoreTestSpec) reconcile.RestoreTestResult {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.archives = append(r.archives, spec.Archive)
|
||||
return reconcile.RestoreTestResult{
|
||||
Archive: spec.Archive, SourceTier: spec.SourceTier,
|
||||
Pass: r.pass, Verified: "boot+running",
|
||||
}
|
||||
}
|
||||
func (r *rotRunner) seen() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]string(nil), r.archives...)
|
||||
}
|
||||
|
||||
// testLanded is a landing time old enough to be settled under any cutoff these tests use. R-86
|
||||
// widened the TierPicker seam with the archive's landing time; the rotation tests below are about
|
||||
// tier ORDER and the heavy-operation gate, not about settling, so they hold it constant.
|
||||
var testLanded = time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// archiveFor is a TierPicker over a fixed map: target → archive ("" = that tier holds none).
|
||||
func archiveFor(m map[string]string) TierPicker {
|
||||
return func(_ context.Context, target string, _ time.Time) (string, time.Time, error) {
|
||||
a := m[target]
|
||||
if a == "" {
|
||||
return "", time.Time{}, nil
|
||||
}
|
||||
return a, testLanded, nil
|
||||
}
|
||||
}
|
||||
|
||||
func rotScheduler(t *testing.T, rr *rotRunner, st *RestoreTestState, pick TierPicker, gate *InFlight) *Scheduler {
|
||||
t.Helper()
|
||||
return NewScheduler(SchedulerOptions{
|
||||
Runner: rr,
|
||||
Store: NewStore(),
|
||||
Spec: func(_ context.Context, archive string) reconcile.RestoreTestSpec {
|
||||
return reconcile.RestoreTestSpec{RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009}
|
||||
},
|
||||
Cadence: time.Hour,
|
||||
Logger: quiet(),
|
||||
Tiers: []string{"local", "felhom-pbs"},
|
||||
TierPick: pick,
|
||||
State: st,
|
||||
InFlight: gate,
|
||||
})
|
||||
}
|
||||
|
||||
// ── SCENARIO A — both tiers get tested, each ONCE per archive ────────────────────────────────
|
||||
//
|
||||
// R-86 CHANGED THIS TEST'S CONTRACT, deliberately, and the old assertion is worth recording because
|
||||
// it was a faithful statement of the defect. It read:
|
||||
//
|
||||
// 4 ticks → 4 runs, and consecutive runs must hit different tiers
|
||||
//
|
||||
// i.e. every tick produced a heavy restore-test, because the ticker WAS the trigger. Under R-86 a
|
||||
// tick is an EVALUATION: both tiers are still exercised (rotation is intact), but a tier whose
|
||||
// newest settled archive is already proven is not re-tested just because time passed. So the
|
||||
// assertion is now 2 runs across 4 evaluations — one per tier, one per archive — which is a
|
||||
// STRICTLY STRONGER statement: it pins both the coverage R-85 won and the pacing R-86 adds.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): restore the single-target picker — set `Tiers`/`TierPick` to nil
|
||||
// so `pickForThisRun` falls back to `s.pick` on the primary runner — and this fails with
|
||||
// "both tiers must be exercised; got [local:…]", i.e. the offsite tier never appears. That is
|
||||
// pre-R-85 behaviour, and it is why demo-hp's DR tier went unproven for its entire existence.
|
||||
func TestRotation_BothTiersExercisedOncePerArchive(t *testing.T) {
|
||||
rr := &rotRunner{pass: true}
|
||||
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
|
||||
s := rotScheduler(t, rr, st, archiveFor(map[string]string{
|
||||
"local": "local:backup/vzdump-lxc-9201-x.tar.zst",
|
||||
"felhom-pbs": "felhom-pbs:backup/ct/9201/2026-07-26T15:42:42Z",
|
||||
}), &InFlight{})
|
||||
s.now = func() time.Time { return time.Now().UTC() }
|
||||
|
||||
for i := 0; i < 4; i++ {
|
||||
s.tick(context.Background())
|
||||
}
|
||||
|
||||
got := rr.seen()
|
||||
var sawLocal, sawPBS bool
|
||||
for _, a := range got {
|
||||
if len(a) >= 5 && a[:5] == "local" {
|
||||
sawLocal = true
|
||||
}
|
||||
if len(a) >= 10 && a[:10] == "felhom-pbs" {
|
||||
sawPBS = true
|
||||
}
|
||||
}
|
||||
if !sawLocal || !sawPBS {
|
||||
t.Fatalf("both tiers must be exercised; got %v", got)
|
||||
}
|
||||
// Exactly one run per tier: the archives never changed, so nothing became due a second time.
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 runs across 4 evaluations (one per archive generation), got %d: %v", len(got), got)
|
||||
}
|
||||
if got[0] == got[1] {
|
||||
t.Fatalf("the two runs must be different tiers — oldest-first is not ordering due tiers: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A tier with NO archive is skipped, not failed, and the other tier still runs. A brand-new offsite
|
||||
// tier legitimately has nothing to restore; turning that into a failure would make every fresh box
|
||||
// look broken for its first week.
|
||||
func TestRotation_EmptyTierSkippedNotFailed(t *testing.T) {
|
||||
rr := &rotRunner{pass: true}
|
||||
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
|
||||
s := rotScheduler(t, rr, st, archiveFor(map[string]string{
|
||||
"local": "local:backup/vzdump-lxc-9201-x.tar.zst",
|
||||
"felhom-pbs": "", // provisioned but empty
|
||||
}), &InFlight{})
|
||||
|
||||
s.tick(context.Background())
|
||||
got := rr.seen()
|
||||
if len(got) != 1 || got[0][:5] != "local" {
|
||||
t.Fatalf("an empty tier must be skipped and the testable one still run; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing testable anywhere → a clean no-op, not an error and not a run.
|
||||
func TestRotation_NoArchivesAnywhereIsANoOp(t *testing.T) {
|
||||
rr := &rotRunner{pass: true}
|
||||
s := rotScheduler(t, rr, NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")),
|
||||
archiveFor(map[string]string{}), &InFlight{})
|
||||
s.tick(context.Background())
|
||||
if got := rr.seen(); len(got) != 0 {
|
||||
t.Fatalf("no archives anywhere → no run; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A FAILED restore-test must NOT earn rotation credit, or a tier that fails every time would look
|
||||
// freshly proven and quietly stop being retried.
|
||||
func TestRotation_FailureEarnsNoCredit(t *testing.T) {
|
||||
rr := &rotRunner{pass: false}
|
||||
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
|
||||
s := rotScheduler(t, rr, st, archiveFor(map[string]string{
|
||||
"local": "local:backup/x.tar.zst",
|
||||
"felhom-pbs": "felhom-pbs:backup/ct/9201/y",
|
||||
}), &InFlight{})
|
||||
s.tick(context.Background())
|
||||
if _, ok := st.LastSuccess("local"); ok {
|
||||
t.Fatal("a FAILED restore-test must not stamp the tier as proven")
|
||||
}
|
||||
if _, ok := st.LastSuccess("felhom-pbs"); ok {
|
||||
t.Fatal("a FAILED restore-test must not stamp the tier as proven")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO E — rotation survives a restart ─────────────────────────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): make the state in-memory (construct a fresh
|
||||
// `NewRestoreTestState` on a DIFFERENT path for the second scheduler, i.e. lose the file) and this
|
||||
// fails with "after a restart the OTHER tier must be next; got felhom-pbs" — the same tier repeats
|
||||
// and the other is starved indefinitely, which with agent deploys as routine as they are is not a
|
||||
// corner case.
|
||||
func TestRotation_SurvivesRestart(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "rt.json")
|
||||
picks := archiveFor(map[string]string{
|
||||
"local": "local:backup/x.tar.zst",
|
||||
"felhom-pbs": "felhom-pbs:backup/ct/9201/y",
|
||||
})
|
||||
|
||||
// First process: the OFFSITE tier is tested (never-proven sorts first).
|
||||
rr1 := &rotRunner{pass: true}
|
||||
st1 := NewRestoreTestState(path)
|
||||
s1 := rotScheduler(t, rr1, st1, picks, &InFlight{})
|
||||
s1.tick(context.Background())
|
||||
first := rr1.seen()
|
||||
if len(first) != 1 {
|
||||
t.Fatalf("want one run, got %v", first)
|
||||
}
|
||||
|
||||
// --- restart: brand-new state object reading the SAME file ---
|
||||
rr2 := &rotRunner{pass: true}
|
||||
st2 := NewRestoreTestState(path)
|
||||
s2 := rotScheduler(t, rr2, st2, picks, &InFlight{})
|
||||
s2.tick(context.Background())
|
||||
second := rr2.seen()
|
||||
if len(second) != 1 {
|
||||
t.Fatalf("want one run after restart, got %v", second)
|
||||
}
|
||||
|
||||
if second[0] == first[0] {
|
||||
t.Fatalf("after a restart the OTHER tier must be next; got %s twice (rotation state was lost)", second[0])
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO F — no collision with a backup ──────────────────────────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): drop the TryAcquire guard from `tick` and this fails with
|
||||
// "the restore-test must DEFER while a backup holds the gate; concurrent operations = 2" — the
|
||||
// count is the assertion, since "both completed" would pass against a fully concurrent
|
||||
// implementation.
|
||||
func TestRotation_DefersWhileABackupHoldsTheGate(t *testing.T) {
|
||||
gate := &InFlight{}
|
||||
release, _, ok := gate.TryAcquire("backup:felhom-pbs")
|
||||
if !ok {
|
||||
t.Fatal("precondition: the gate should have been free")
|
||||
}
|
||||
defer release()
|
||||
|
||||
rr := &rotRunner{pass: true}
|
||||
s := rotScheduler(t, rr, NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")),
|
||||
archiveFor(map[string]string{"local": "local:backup/x.tar.zst"}), gate)
|
||||
|
||||
s.tick(context.Background())
|
||||
|
||||
concurrent := 1 + len(rr.seen()) // the backup holding the gate, plus anything the tick started
|
||||
if concurrent != 1 {
|
||||
t.Fatalf("the restore-test must DEFER while a backup holds the gate; concurrent operations = %d", concurrent)
|
||||
}
|
||||
}
|
||||
|
||||
// Once the backup releases, the next cadence proceeds — deferral must not be permanent.
|
||||
func TestRotation_ResumesAfterTheGateFrees(t *testing.T) {
|
||||
gate := &InFlight{}
|
||||
release, _, _ := gate.TryAcquire("backup:local")
|
||||
|
||||
rr := &rotRunner{pass: true}
|
||||
s := rotScheduler(t, rr, NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json")),
|
||||
archiveFor(map[string]string{"local": "local:backup/x.tar.zst"}), gate)
|
||||
|
||||
s.tick(context.Background())
|
||||
if len(rr.seen()) != 0 {
|
||||
t.Fatal("should have deferred while held")
|
||||
}
|
||||
release()
|
||||
s.tick(context.Background())
|
||||
if len(rr.seen()) != 1 {
|
||||
t.Fatalf("must resume once the gate frees; got %v", rr.seen())
|
||||
}
|
||||
}
|
||||
|
||||
// The gate itself: one holder at a time, named, and release is idempotent.
|
||||
func TestInFlight_Semantics(t *testing.T) {
|
||||
g := &InFlight{}
|
||||
rel, busy, ok := g.TryAcquire("backup:local")
|
||||
if !ok || busy != "" {
|
||||
t.Fatalf("first acquire must succeed; ok=%v busy=%q", ok, busy)
|
||||
}
|
||||
if _, busy2, ok2 := g.TryAcquire("restore-test"); ok2 || busy2 != "backup:local" {
|
||||
t.Fatalf("second acquire must fail and NAME the holder; ok=%v busy=%q", ok2, busy2)
|
||||
}
|
||||
rel()
|
||||
rel() // idempotent — a double release must not free someone else's later claim
|
||||
if g.Busy() != "" {
|
||||
t.Fatalf("gate should be idle after release; busy=%q", g.Busy())
|
||||
}
|
||||
if _, _, ok3 := g.TryAcquire("restore-test"); !ok3 {
|
||||
t.Fatal("gate must be reusable after release")
|
||||
}
|
||||
}
|
||||
|
||||
// A nil gate means "not wired" → no gating, pre-R-85 behaviour. Keeps every existing caller working.
|
||||
func TestInFlight_NilIsUngated(t *testing.T) {
|
||||
var g *InFlight
|
||||
rel, _, ok := g.TryAcquire("x")
|
||||
if !ok {
|
||||
t.Fatal("a nil gate must not block")
|
||||
}
|
||||
rel()
|
||||
if g.Busy() != "" {
|
||||
t.Fatal("a nil gate is never busy")
|
||||
}
|
||||
}
|
||||
|
||||
// ── oldest-first ordering ────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestOldestFirst_Ordering(t *testing.T) {
|
||||
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
|
||||
now := time.Now().UTC()
|
||||
|
||||
// Never-proven sorts FIRST — the case that matters, since the offsite tier starts there.
|
||||
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" {
|
||||
// both never proven → deterministic tie-break by id
|
||||
if got[0] != "felhom-pbs" && got[0] != "local" {
|
||||
t.Fatalf("unexpected: %v", got)
|
||||
}
|
||||
}
|
||||
_ = st.RecordSuccess("local", "local:backup/a.tar.zst", "local", "boot+running", now)
|
||||
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "felhom-pbs" {
|
||||
t.Fatalf("a never-proven tier must sort before a proven one; got %v", got)
|
||||
}
|
||||
_ = st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/b", "pbs", "boot+running", now.Add(time.Hour))
|
||||
if got := st.OldestFirst([]string{"local", "felhom-pbs"}); got[0] != "local" {
|
||||
t.Fatalf("the least recently proven must sort first; got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Ordering must be DETERMINISTIC for equal timestamps, or two tiers proven in the same second would
|
||||
// rotate by Go's randomised map iteration — untestable, and occasionally starving.
|
||||
func TestOldestFirst_DeterministicOnTies(t *testing.T) {
|
||||
st := NewRestoreTestState(filepath.Join(t.TempDir(), "rt.json"))
|
||||
now := time.Now().UTC()
|
||||
_ = st.RecordSuccess("b-tier", "b:archive", "local", "boot+running", now)
|
||||
_ = st.RecordSuccess("a-tier", "a:archive", "local", "boot+running", now)
|
||||
for i := 0; i < 20; i++ {
|
||||
if got := st.OldestFirst([]string{"b-tier", "a-tier"}); got[0] != "a-tier" {
|
||||
t.Fatalf("tie-break must be deterministic; iteration %d gave %v", i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The state file round-trips, and a corrupt file degrades to "nothing proven" rather than wedging.
|
||||
func TestRestoreTestState_PersistenceAndCorruption(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "rt.json")
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
|
||||
st := NewRestoreTestState(path)
|
||||
if err := st.RecordSuccess("felhom-pbs", "felhom-pbs:backup/ct/9201/x", "pbs", "boot+running", now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened := NewRestoreTestState(path)
|
||||
got, ok := reopened.LastSuccess("felhom-pbs")
|
||||
if !ok || !got.Equal(now) {
|
||||
t.Fatalf("state must round-trip; got %v ok=%v want %v", got, ok, now)
|
||||
}
|
||||
|
||||
bad := filepath.Join(dir, "corrupt.json")
|
||||
if err := os.WriteFile(bad, []byte("{{{not json"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := NewRestoreTestState(bad)
|
||||
if _, ok := c.LastSuccess("felhom-pbs"); ok {
|
||||
t.Fatal("a corrupt state file must degrade to 'nothing proven', not invent a timestamp")
|
||||
}
|
||||
}
|
||||
+218
-10
@@ -6,6 +6,7 @@ import (
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
@@ -38,21 +39,57 @@ type BackupRunner struct {
|
||||
// each successful backup, so the agent's own backups can't pile up and refill root. Empty → no prune
|
||||
// (the legacy behaviour; restore-test/selftest runners pass ""). NEVER applied to a PBS target.
|
||||
retention string
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
// waitTimeout bounds the WaitTask poll on this runner's vzdump. Per-TIER since R-82: 30m is
|
||||
// right for a local vzdump and badly wrong for an offsite PBS upload (see the 2026-07-26 live
|
||||
// failure recorded on config.BackupTargetConfig.WaitTimeoutSeconds). 0 → 30m (legacy).
|
||||
waitTimeout time.Duration
|
||||
// allowPBSPrune permits `--prune-backups` on a PBS-type target. OFF by default and ON only for
|
||||
// an ADDITIONAL tier whose keep_last was set explicitly (operator ruling 2026-07-26: keep two
|
||||
// weeks of weekly offsite backups).
|
||||
//
|
||||
// The blanket PBS refusal it replaces existed for a real reason and still applies to the
|
||||
// PRIMARY tier: BackupTarget() DEFAULTS to "felhom-pbs" and KeepLast() DEFAULTS to 3, so a box
|
||||
// with neither key set would silently prune its offsite DR to 3 restore points. An additional
|
||||
// tier cannot have that accident — its keep_last defaults to 0 (never prune), so any value
|
||||
// there is a deliberate act.
|
||||
allowPBSPrune bool
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
// rejected remembers the volids already announced by warnRejectedArchiveOnce, so an incomplete
|
||||
// archive is reported ONCE rather than on every 5-minute due-check. Bounded in practice: one
|
||||
// entry per aborted upload, and a process restart clears it. Guarded by rejectedMu because the
|
||||
// due-check is served from the local-API handler goroutines.
|
||||
rejectedMu sync.Mutex
|
||||
rejected map[string]struct{}
|
||||
}
|
||||
|
||||
// NewBackupRunner builds a runner. mode defaults to snapshot (works for a stopped guest and
|
||||
// for lvm-thin); the caller may pass ModeStop for storages without snapshot support. retention is the
|
||||
// per-run prune spec ("keep-last=N", or "" to never prune) — only the periodic local backup sets it.
|
||||
func NewBackupRunner(api BackupAPI, target string, mode proxmox.BackupMode, notes, retention string, logger *slog.Logger) *BackupRunner {
|
||||
return NewBackupRunnerWithWait(api, target, mode, notes, retention, 0, logger)
|
||||
}
|
||||
|
||||
// NewBackupRunnerWithWait is NewBackupRunner plus an explicit vzdump wait bound (0 → 30m).
|
||||
func NewBackupRunnerWithWait(api BackupAPI, target string, mode proxmox.BackupMode, notes, retention string, waitTimeout time.Duration, logger *slog.Logger) *BackupRunner {
|
||||
return NewBackupRunnerFull(api, target, mode, notes, retention, waitTimeout, false, logger)
|
||||
}
|
||||
|
||||
// NewBackupRunnerFull is the full constructor. allowPBSPrune must be true ONLY for an additional
|
||||
// tier with an explicitly configured keep_last — see BackupRunner.allowPBSPrune.
|
||||
func NewBackupRunnerFull(api BackupAPI, target string, mode proxmox.BackupMode, notes, retention string, waitTimeout time.Duration, allowPBSPrune bool, logger *slog.Logger) *BackupRunner {
|
||||
if mode == "" {
|
||||
mode = proxmox.ModeSnapshot
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &BackupRunner{api: api, target: target, mode: mode, notes: notes, retention: retention, logger: logger, now: func() time.Time { return time.Now().UTC() }}
|
||||
if waitTimeout <= 0 {
|
||||
waitTimeout = 30 * time.Minute
|
||||
}
|
||||
return &BackupRunner{api: api, target: target, mode: mode, notes: notes, retention: retention,
|
||||
waitTimeout: waitTimeout, allowPBSPrune: allowPBSPrune, logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() }}
|
||||
}
|
||||
|
||||
// localPruneSpec returns the `--prune-backups` spec to apply to THIS backup, or "" to skip pruning. It
|
||||
@@ -71,8 +108,11 @@ func (r *BackupRunner) localPruneSpec(ctx context.Context) string {
|
||||
}
|
||||
for _, s := range stores {
|
||||
if s.Storage == r.target {
|
||||
if s.Type == "pbs" {
|
||||
return "" // PBS retention is out of scope — never prune the offsite DR
|
||||
if s.Type == "pbs" && !r.allowPBSPrune {
|
||||
// Not opted in → never prune the offsite DR (the pre-R-82 rule, and still the rule
|
||||
// for the primary tier, whose target+retention both DEFAULT and could prune by
|
||||
// accident).
|
||||
return ""
|
||||
}
|
||||
return r.retention
|
||||
}
|
||||
@@ -147,7 +187,7 @@ func (r *BackupRunner) backup(ctx context.Context, vmid int, onSnapshot func())
|
||||
defer stopWatch()
|
||||
go r.watchForSnapshot(watchCtx, upid, onSnapshot)
|
||||
}
|
||||
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: 30 * time.Minute}); err != nil {
|
||||
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: r.waitTimeout}); err != nil {
|
||||
rec.Error = err.Error()
|
||||
rec.DurationSeconds = time.Since(start).Seconds()
|
||||
return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err)
|
||||
@@ -210,18 +250,69 @@ func (r *BackupRunner) watchForSnapshot(ctx context.Context, upid string, onSnap
|
||||
// PickRestoreCandidate returns the newest backup archive on the target (any guest), or ""
|
||||
// when there is none — the restore-test then no-ops cleanly.
|
||||
func (r *BackupRunner) PickRestoreCandidate(ctx context.Context) (string, error) {
|
||||
contents, err := r.api.StorageContent(ctx, r.target)
|
||||
return r.PickRestoreCandidateOn(ctx, r.target)
|
||||
}
|
||||
|
||||
// PickRestoreCandidateOn is PickRestoreCandidate for an ARBITRARY tier's storage (R-85 1.2), so the
|
||||
// scheduler can rotate across tiers instead of only ever seeing this runner's own target.
|
||||
//
|
||||
// Contract preserved: "" + nil error when the storage holds no archive. **A tier with nothing to
|
||||
// restore is not an error** — a brand-new offsite tier legitimately has nothing yet, and turning
|
||||
// that into a failure would make every fresh box look broken for its first week.
|
||||
func (r *BackupRunner) PickRestoreCandidateOn(ctx context.Context, target string) (string, error) {
|
||||
archive, _, err := r.PickSettledRestoreCandidateOn(ctx, target, time.Time{})
|
||||
return archive, err
|
||||
}
|
||||
|
||||
// PickSettledRestoreCandidateOn is the R-86 due-check's picker: the newest archive on target that
|
||||
// landed AT OR BEFORE notAfter (the settle cutoff), with the time it landed. A zero notAfter means
|
||||
// "no cutoff" — that is the pre-R-86 behaviour, which is why PickRestoreCandidateOn is now a
|
||||
// one-line call into this and its contract is untouched (one scan, one owner).
|
||||
//
|
||||
// WHY A CUTOFF AT ALL. An archive that landed minutes ago may still be settling — R-71a's
|
||||
// settle-gate exists because the offsite tier's day-0 consume raced its own floor update — and
|
||||
// restore-testing the archive a backup is still writing proves nothing about the backup that
|
||||
// finished. The due-check therefore asks about the newest SETTLED archive, and §8.1's rule is built
|
||||
// on that: the tier is due when a settled archive exists that has not been proven.
|
||||
//
|
||||
// The plausibility floor is applied here and not in the old path on purpose. Under R-86 the picked
|
||||
// archive becomes the tier's due-ness: an incomplete 1-byte phantom (F-CRIT-2's artefact — server
|
||||
// prune does NOT collect it) would be selected forever, fail its restore forever, never earn proof,
|
||||
// and so make the tier due at EVERY evaluation. Skipping it is what keeps the retry rate bounded by
|
||||
// the archive generation rather than by the evaluation interval.
|
||||
//
|
||||
// Contract preserved: ("", zero, nil) when the storage holds no eligible archive. **A tier with
|
||||
// nothing to restore is not an error** — a brand-new offsite tier legitimately has nothing yet, and
|
||||
// turning that into a failure would make every fresh box look broken for its first week.
|
||||
func (r *BackupRunner) PickSettledRestoreCandidateOn(ctx context.Context, target string, notAfter time.Time) (string, time.Time, error) {
|
||||
if target == "" {
|
||||
return "", time.Time{}, nil
|
||||
}
|
||||
contents, err := r.api.StorageContent(ctx, target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
var best string
|
||||
var bestCTime int64 = -1
|
||||
for _, e := range contents {
|
||||
if e.Content == "backup" && e.CTime > bestCTime {
|
||||
if e.Content != "backup" {
|
||||
continue
|
||||
}
|
||||
if !notAfter.IsZero() && e.CTime > notAfter.Unix() {
|
||||
continue // not settled yet — a newer archive is not a reason to re-prove an older one
|
||||
}
|
||||
if ok, why := archivePlausiblyComplete(e); !ok {
|
||||
r.warnRejectedArchiveOnce(e, why)
|
||||
continue
|
||||
}
|
||||
if e.CTime > bestCTime {
|
||||
bestCTime, best = e.CTime, e.VolID
|
||||
}
|
||||
}
|
||||
return best, nil
|
||||
if best == "" {
|
||||
return "", time.Time{}, nil
|
||||
}
|
||||
return best, time.Unix(bestCTime, 0).UTC(), nil
|
||||
}
|
||||
|
||||
// latestArchive finds the newest backup archive volid + size for vmid on the target.
|
||||
@@ -243,6 +334,121 @@ func (r *BackupRunner) latestArchive(ctx context.Context, vmid int) (string, int
|
||||
return vol, size, nil
|
||||
}
|
||||
|
||||
// NewestArchiveTime reports when this guest's newest backup archive LANDED ON THIS TARGET, from the
|
||||
// storage itself. ok=false means the target genuinely holds no archive for this guest.
|
||||
//
|
||||
// R-84: this is the cure for the redundant-backup-after-restart problem. The agent's backup Store is
|
||||
// in-memory ("lost on restart; the cadence re-populates"), so after every restart /backup/due
|
||||
// reported "no successful backup recorded yet" and the controller dutifully took another one. On the
|
||||
// local tier that is wasted minutes; on the OFFSITE tier it is a wasted multi-hour WAN upload after
|
||||
// every agent deploy — and agent deploys are routine. Three redundant local backups were observed on
|
||||
// minPlausibleArchiveBytes is the floor below which a storage entry cannot be a real whole-guest
|
||||
// backup and is therefore treated as an INCOMPLETE artefact rather than a successful one.
|
||||
//
|
||||
// MEASURED, not chosen by feel — fleet survey 2026-07-28 (Campaign 8, finding F-CRIT-2):
|
||||
//
|
||||
// smallest REAL backup anywhere on the fleet ... 612,397,450 B (~584 MiB, a guest-9100 vzdump)
|
||||
// demo-hp local / PBS ..................... 1.59 GB / 4.35-4.37 GB
|
||||
// demo-felhom local / PBS ..................... 5.82-5.84 GB / 14.47-14.51 GB
|
||||
// the phantom left by a PBS daemon killed mid-upload ....... 1 B
|
||||
//
|
||||
// 1 MiB sits 584x below the smallest real backup and 1,048,576x above the phantom. The two
|
||||
// populations are nine orders of magnitude apart, so this floor cannot plausibly clip a real
|
||||
// archive — which is the property that matters, because a floor set too HIGH does not merely lose
|
||||
// safety margin, it causes fleet-wide backup THRASH (see archivePlausiblyComplete).
|
||||
const minPlausibleArchiveBytes int64 = 1 << 20
|
||||
|
||||
// archivePlausiblyComplete reports whether a storage entry can be a COMPLETE backup, and if not,
|
||||
// why. Pure, so the contract is unit-testable without a storage.
|
||||
//
|
||||
// WHY SIZE, AND NOTHING ELSE. The richer PBS fields look like better discriminators and are all
|
||||
// traps, because this runner is TIER-AGNOSTIC — the same predicate runs against a PBS datastore and
|
||||
// against a plain `dir` storage (verified against the live PVE API, 2026-07-28):
|
||||
//
|
||||
// - `verification` is absent on the phantom, but ALSO absent on every local (dir) archive — a dir
|
||||
// storage has no verification concept — and absent on a good PBS snapshot until verify-new
|
||||
// catches up. Gating on it would reject 100% of local backups and every freshly-taken offsite
|
||||
// one: continuous re-backup across the fleet.
|
||||
// - `encrypted` fails the same way, and for the same reason.
|
||||
// - `notes` happens to be present on both good tiers today only because the agent sets it; an
|
||||
// archive written by any other path lacks it. Too fragile to gate freshness on.
|
||||
//
|
||||
// Size is the only signal that means the same thing on every tier.
|
||||
//
|
||||
// THE FAIL-SAFE DIRECTION, stated explicitly: when completeness cannot be established the entry is
|
||||
// NOT counted as a successful backup. That errs toward the tier looking LESS fresh, and its worst
|
||||
// case is one extra backup. Counting an undecidable entry is precisely the F-CRIT-2 defect — a
|
||||
// failed upload that made its tier look freshly backed up and silenced it for a full cadence.
|
||||
func archivePlausiblyComplete(e proxmox.StorageContent) (bool, string) {
|
||||
if e.Size < minPlausibleArchiveBytes {
|
||||
return false, fmt.Sprintf("size %d B is below the %d B plausibility floor — an aborted/incomplete archive, not a successful backup",
|
||||
e.Size, minPlausibleArchiveBytes)
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// warnRejectedArchiveOnce announces a rejected archive at WARN exactly once per distinct volid.
|
||||
//
|
||||
// A rejected archive must never be silent: a tier that quietly ignores the newest entry on its
|
||||
// storage is a new quiet path, and quiet paths are what F-CRIT-2 was. But the due-check runs every
|
||||
// 5 minutes and a phantom persists indefinitely — server-side prune does NOT collect it (verified
|
||||
// by dry-run 2026-07-28: with keep-last 2 it retained two real snapshots PLUS the phantom) — so
|
||||
// logging per poll would emit ~288 identical lines a day and bury the one that matters.
|
||||
func (r *BackupRunner) warnRejectedArchiveOnce(e proxmox.StorageContent, why string) {
|
||||
r.rejectedMu.Lock()
|
||||
if r.rejected == nil {
|
||||
r.rejected = map[string]struct{}{}
|
||||
}
|
||||
_, seen := r.rejected[e.VolID]
|
||||
if !seen {
|
||||
r.rejected[e.VolID] = struct{}{}
|
||||
}
|
||||
r.rejectedMu.Unlock()
|
||||
if seen {
|
||||
return
|
||||
}
|
||||
r.logger.Warn("backup: ignoring an INCOMPLETE archive when computing tier freshness — it is not a successful backup",
|
||||
"target", r.target, "vmid", e.VMID, "volid", e.VolID, "size_bytes", e.Size, "reason", why)
|
||||
}
|
||||
|
||||
// demo-felhom in a single afternoon of deploys (2026-07-26).
|
||||
//
|
||||
// Asking the STORAGE rather than persisting the store is deliberate:
|
||||
// - it is ground truth, not remembered state — if an archive was pruned or deleted it correctly
|
||||
// stops counting, whereas a persisted record would keep claiming a backup that no longer exists;
|
||||
// - it needs no new on-disk state and no migration;
|
||||
// - it is the same source `latestArchive` already trusts to build the post-backup record.
|
||||
//
|
||||
// It answers ONLY "when did a plausibly COMPLETE backup last land", which is exactly what the
|
||||
// due-check needs. Completeness is not optional here: PBS publishes an aborted upload into the same
|
||||
// listing (manifest-less, 1 byte, and NEWEST), and counting it made the tier report fresh and go
|
||||
// silent for a whole cadence — F-CRIT-2. Presence is not validity. The
|
||||
// richer fields (size, duration, uncovered volumes, error) stay with the real in-memory records — a
|
||||
// synthesized record would put invented numbers into the host-report.
|
||||
func (r *BackupRunner) NewestArchiveTime(ctx context.Context, vmid int) (time.Time, bool, error) {
|
||||
contents, err := r.api.StorageContent(ctx, r.target)
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
var best int64 = -1
|
||||
for _, e := range contents {
|
||||
if e.Content != "backup" || e.VMID != vmid {
|
||||
continue
|
||||
}
|
||||
if ok, why := archivePlausiblyComplete(e); !ok {
|
||||
r.warnRejectedArchiveOnce(e, why)
|
||||
continue
|
||||
}
|
||||
if e.CTime > best {
|
||||
best = e.CTime
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
return time.Unix(best, 0).UTC(), true, nil
|
||||
}
|
||||
|
||||
// parseBackupMode extracts the actual mode from a vzdump task log line `… backup mode: <x>`
|
||||
// (e.g. "INFO: backup mode: stop"). Returns "" if not found.
|
||||
func parseBackupMode(lines []string) string {
|
||||
@@ -308,6 +514,8 @@ func ToHubRestoreTest(res reconcile.RestoreTestResult, testedAt time.Time) hub.R
|
||||
DurationSeconds: res.Duration.Seconds(),
|
||||
Warnings: res.StartWarnings,
|
||||
WarningsRecognized: res.WarningsRecognized,
|
||||
MountParity: res.MountParity,
|
||||
MountInventory: res.MountInventory,
|
||||
}
|
||||
if res.Err != nil {
|
||||
rt.Error = res.Err.Error()
|
||||
|
||||
+206
-29
@@ -18,27 +18,77 @@ type RestoreTestRunner interface {
|
||||
// there is none yet (the tick then no-ops).
|
||||
type CandidatePicker func(ctx context.Context) (string, error)
|
||||
|
||||
// SpecBuilder yields the RestoreTestSpec for ONE run, given the archive that was picked.
|
||||
//
|
||||
// R-85 (1.1): this REPLACES a frozen spec value. It used to be built by an immediately-invoked
|
||||
// function at daemon start, so `storageTier()` and `restoreTaskTimeout()` were evaluated ONCE and
|
||||
// the resulting value reused for every run for the lifetime of the process. Two consequences:
|
||||
// - nothing tier-varying was expressible at all (the offsite tier could never be scheduled), and
|
||||
// - it was a latent staleness bug in its own right — a storage-type or config change did not take
|
||||
// effect until the daemon restarted.
|
||||
//
|
||||
// The archive is passed in because the tier MUST be derived from it (the v0.100.0 rule), never from
|
||||
// the configured target: deriving it from config is what produced the 600 s false failure when a
|
||||
// PBS archive was classified "local" and got the 10-minute local wait.
|
||||
type SpecBuilder func(ctx context.Context, archive string) reconcile.RestoreTestSpec
|
||||
|
||||
// TierPicker resolves the newest archive on a NAMED tier that landed AT OR BEFORE notAfter (the
|
||||
// settle cutoff), together with when it landed. (*BackupRunner).PickSettledRestoreCandidateOn
|
||||
// satisfies it. A zero notAfter means "no settle requirement".
|
||||
//
|
||||
// R-86 widened this seam from (target) → archive. The landing time is what makes the due-check's
|
||||
// verdict explainable — "archive X, which landed at T, has not been proven" — and the cutoff is
|
||||
// what makes the rule per-ARCHIVE-GENERATION instead of per-interval. "" must NOT be an error: a
|
||||
// brand-new offsite tier legitimately has nothing to restore yet.
|
||||
type TierPicker func(ctx context.Context, target string, notAfter time.Time) (archive string, landed time.Time, err error)
|
||||
|
||||
// Scheduler runs the self-restore-test on an agent-internal cadence. It is the fourth daemon
|
||||
// goroutine; it does real restore→boot→destroy, so it only runs when the cadence is enabled
|
||||
// AND a valid scratch band is configured (validated by the caller before construction).
|
||||
type Scheduler struct {
|
||||
runner RestoreTestRunner
|
||||
pick CandidatePicker
|
||||
store *Store
|
||||
spec reconcile.RestoreTestSpec // archive is filled per-tick
|
||||
runner RestoreTestRunner
|
||||
pick CandidatePicker
|
||||
store *Store
|
||||
spec SpecBuilder // R-85: evaluated PER RUN, never frozen at construction
|
||||
// cadence is the EVALUATION interval (R-86) — how often "is anything due?" is asked. It is no
|
||||
// longer the thing that decides a test happens; see restoretest_due.go.
|
||||
cadence time.Duration
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
// settle is how long an archive must have sat before it is a candidate (R-86).
|
||||
settle time.Duration
|
||||
logger *slog.Logger
|
||||
now func() time.Time
|
||||
|
||||
// R-85 tier rotation. All optional: without them the scheduler behaves exactly as before
|
||||
// (single tier via `pick`), which keeps every existing caller and test working untouched.
|
||||
tiers []string // configured tier target ids, primary first
|
||||
tierPick TierPicker // newest archive on a named tier
|
||||
rtState *RestoreTestState // persisted last-successful-per-tier (drives oldest-first)
|
||||
inFlight *InFlight // shared with the backup path — Scenario F
|
||||
}
|
||||
|
||||
// SchedulerOptions configures a Scheduler.
|
||||
type SchedulerOptions struct {
|
||||
Runner RestoreTestRunner
|
||||
Pick CandidatePicker
|
||||
Store *Store
|
||||
Spec reconcile.RestoreTestSpec // RestoreStorage, ScratchMin/Max, SourceTier, BootTimeout
|
||||
Cadence time.Duration // 0 → disabled
|
||||
Logger *slog.Logger
|
||||
Runner RestoreTestRunner
|
||||
Pick CandidatePicker
|
||||
Store *Store
|
||||
// Spec builds the run's spec (RestoreStorage, ScratchMin/Max, SourceTier, timeouts) from the
|
||||
// picked archive. Called ONCE PER RUN — see SpecBuilder for why it is not a value.
|
||||
Spec SpecBuilder
|
||||
// Cadence is the EVALUATION interval — how often due-ness is asked, NOT how often a test runs
|
||||
// (R-86). 0 → disabled.
|
||||
Cadence time.Duration
|
||||
// Settle is how long an archive must have sat before it is a restore-test candidate (R-86).
|
||||
// 0 → no settle requirement (any archive is a candidate).
|
||||
Settle time.Duration
|
||||
Logger *slog.Logger
|
||||
|
||||
// R-85 (all optional — omit for the pre-R-85 single-tier behaviour):
|
||||
// Tiers are the configured tier target ids (primary first); TierPick resolves an archive on a
|
||||
// named tier; State persists last-successful-per-tier; InFlight is the shared one-heavy-op gate.
|
||||
Tiers []string
|
||||
TierPick TierPicker
|
||||
State *RestoreTestState
|
||||
InFlight *InFlight
|
||||
}
|
||||
|
||||
// NewScheduler builds a Scheduler.
|
||||
@@ -48,27 +98,43 @@ func NewScheduler(opts SchedulerOptions) *Scheduler {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Scheduler{
|
||||
runner: opts.Runner,
|
||||
pick: opts.Pick,
|
||||
store: opts.Store,
|
||||
spec: opts.Spec,
|
||||
cadence: opts.Cadence,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
runner: opts.Runner,
|
||||
pick: opts.Pick,
|
||||
store: opts.Store,
|
||||
spec: opts.Spec,
|
||||
cadence: opts.Cadence,
|
||||
settle: opts.Settle,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
tiers: append([]string(nil), opts.Tiers...),
|
||||
tierPick: opts.TierPick,
|
||||
rtState: opts.State,
|
||||
inFlight: opts.InFlight,
|
||||
}
|
||||
}
|
||||
|
||||
// Run fires a restore-test on the cadence until ctx is cancelled. A 0 cadence disables it
|
||||
// (the goroutine just waits for shutdown). It does NOT fire immediately on start (a restore
|
||||
// is heavy; the first runs one interval in) — on-demand runs use the selftest harness.
|
||||
// Run EVALUATES due-ness on the interval until ctx is cancelled, and runs a restore-test only when
|
||||
// a tier is actually due (R-86). A 0 interval disables it (the goroutine just waits for shutdown).
|
||||
//
|
||||
// The ticker survives as the evaluation interval and nothing else. It is emphatically NOT the
|
||||
// trigger any more: its phase is the process's uptime, and agent deploys reset it, which is exactly
|
||||
// the defect R-86 removes. What decides that a test happens is `EvaluateDue`.
|
||||
//
|
||||
// It still does NOT evaluate immediately on start — the first evaluation is one interval in. That
|
||||
// is an EARNED restraint, kept deliberately: a restore is heavy, agent restarts are routine, and a
|
||||
// crash-loop that evaluated at start would hammer a permanently-failing tier as fast as it could
|
||||
// restart. Due-ness does not expire while we wait, so the only cost is up to one interval of
|
||||
// latency on a tier that just became due. On-demand runs use `--selftest=restore-test`.
|
||||
//
|
||||
// Returns nil on ctx cancellation.
|
||||
func (s *Scheduler) Run(ctx context.Context) error {
|
||||
if s.cadence <= 0 || s.runner == nil || s.pick == nil {
|
||||
if s.cadence <= 0 || s.runner == nil || s.spec == nil || (s.pick == nil && !s.rotating()) {
|
||||
s.logger.Info("backup: restore-test cadence disabled")
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
s.logger.Info("backup: restore-test scheduler starting", "cadence", s.cadence)
|
||||
s.logger.Info("backup: restore-test scheduler starting (per-archive due-check)",
|
||||
"eval_interval", s.cadence, "settle", s.settle)
|
||||
t := time.NewTicker(s.cadence)
|
||||
defer t.Stop()
|
||||
for {
|
||||
@@ -82,19 +148,64 @@ func (s *Scheduler) Run(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// tick runs one scheduled restore-test: pick a backup → run → record. No-ops cleanly when
|
||||
// no backup exists yet. Deterministic given s.now — tests call it directly.
|
||||
// tick is ONE EVALUATION: gate → due-check → run the first due tier → record which archive was
|
||||
// proven. No-ops cleanly when nothing is due, when no backup exists yet, or when a heavy operation
|
||||
// is already in flight. Deterministic given s.now — tests call it directly.
|
||||
//
|
||||
// One run per evaluation, by construction (Scenario F): a second due tier is left DUE and picked up
|
||||
// by the next evaluation. Deferred, never cancelled, and never two multi-GB restores over one link.
|
||||
func (s *Scheduler) tick(ctx context.Context) {
|
||||
archive, err := s.pick(ctx)
|
||||
if s.spec == nil {
|
||||
// Defensive: Run() already refuses to start without a SpecBuilder, but tick is also
|
||||
// reachable directly. Skipping loudly beats panicking the daemon goroutine — a missing
|
||||
// spec must cost a restore-test, never the agent.
|
||||
s.logger.Error("backup: restore-test has no spec builder — skipping (this is a wiring bug)")
|
||||
return
|
||||
}
|
||||
// The due-check runs BEFORE the gate is taken, and that ORDER is load-bearing under R-86.
|
||||
//
|
||||
// It used to be the other way round, and correctly so: the gate was held for one heavy run a
|
||||
// day, and the candidate lookup rode along inside it. Evaluations are now frequent, and the
|
||||
// lookup is a storage listing that for the offsite tier crosses the WAN. Holding the
|
||||
// one-heavy-operation gate for a read that answers "nothing to do" would open a small window at
|
||||
// EVERY evaluation in which a starting backup cannot acquire — and a backup that cannot acquire
|
||||
// does not merely wait, it records a failure and pages the operator (F-A1). A cheap poll must
|
||||
// not be able to manufacture that.
|
||||
//
|
||||
// Nothing is lost by checking first: due-ness does not expire, and the gate is still taken
|
||||
// before anything heavy begins.
|
||||
archive, target, err := s.pickForThisRun(ctx)
|
||||
if err != nil {
|
||||
s.logger.Warn("backup: restore-test could not pick a candidate; skipping", "err", err)
|
||||
return
|
||||
}
|
||||
if archive == "" {
|
||||
s.logger.Info("backup: restore-test skipped; no backup available yet")
|
||||
// A POSITIVE OBSERVABLE, at INFO, and this is not noise — it is standing rule 3.
|
||||
//
|
||||
// Before R-86 every tick ran a heavy restore-test, so the scheduler was audible by
|
||||
// construction. Now "nothing is due" is the NORMAL outcome, and at DEBUG it is silent: an
|
||||
// empty journal would be equally consistent with a healthy loop and with a dead goroutine,
|
||||
// which is the exact shape the R-88 watcher was retired for. One line per evaluation is four
|
||||
// lines a day at the 6h default, and it names each tier's verdict so the answer to "why did
|
||||
// nothing run last night?" is in the log rather than in a re-derivation.
|
||||
s.logger.Info("backup: restore-test evaluated — nothing due", "verdicts", s.verdictSummary(ctx))
|
||||
return
|
||||
}
|
||||
spec := s.spec
|
||||
|
||||
// Scenario F: join the one-heavy-operation-at-a-time gate. A restore-test PULLS a multi-GB
|
||||
// archive over the same tunnel an offsite backup PUSHES one; running both saturates the link and
|
||||
// drives each toward its timeout, which is how a healthy tier gets recorded as failed. DEFER —
|
||||
// never cancel what is already running: a deferred restore-test costs hours of coverage, a
|
||||
// cancelled backup costs the backup. A deferred tier stays DUE, so the next evaluation retries it.
|
||||
release, busy, ok := s.inFlight.TryAcquire("restore-test")
|
||||
if !ok {
|
||||
s.logger.Info("backup: restore-test deferred — a heavy operation is already in flight",
|
||||
"busy", busy, "target", target, "archive", archive)
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
// R-85: build the spec for THIS run, from THIS archive. Never a frozen value.
|
||||
spec := s.spec(ctx, archive)
|
||||
spec.Archive = archive
|
||||
res := s.runner.RunRestoreTest(ctx, spec)
|
||||
if res.Skipped {
|
||||
@@ -102,6 +213,19 @@ func (s *Scheduler) tick(ctx context.Context) {
|
||||
}
|
||||
rt := ToHubRestoreTest(res, s.now())
|
||||
s.store.RecordRestoreTest(rt)
|
||||
// Rotation credit is given ONLY on success. A failing tier must keep sorting first, or a tier
|
||||
// that fails every time would look freshly proven and quietly stop being retried.
|
||||
if rt.Pass && s.rtState != nil && target != "" {
|
||||
// R-86: the ARCHIVE is recorded, not merely the time — that is what makes the tier
|
||||
// not-due until a NEWER archive settles, and what makes a proof survive a restart.
|
||||
// R-189: the TIER and what was VERIFIED go with it, so the proof can be RE-REPORTED after a
|
||||
// restart. Both come from the run's own result, never re-derived — `rt.SourceTier` is what
|
||||
// this run was actually judged as, and deriving it later would need a storage lookup that
|
||||
// can fail on the one path where failing means mislabelling a proof.
|
||||
if err := s.rtState.RecordSuccess(target, archive, rt.SourceTier, rt.Verified, s.now()); err != nil {
|
||||
s.logger.Warn("backup: could not persist the restore-test proof state", "target", target, "err", err)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !rt.Pass:
|
||||
// A failing restore-test is the loudest DR signal there is.
|
||||
@@ -118,3 +242,56 @@ func (s *Scheduler) tick(ctx context.Context) {
|
||||
"archive", rt.SourceArchive, "duration_s", rt.DurationSeconds, "warnings", res.StartWarnings)
|
||||
}
|
||||
}
|
||||
|
||||
// rotating reports whether multi-tier rotation is wired.
|
||||
func (s *Scheduler) rotating() bool { return len(s.tiers) > 0 && s.tierPick != nil }
|
||||
|
||||
// pickForThisRun chooses the tier to test THIS evaluation: the first DUE tier, in oldest-proven
|
||||
// order.
|
||||
//
|
||||
// R-86 changed what this answers. It used to answer "whose turn is it?", and the answer was always
|
||||
// somebody's — the ticker had fired, so a test was going to happen. It now answers "is anything
|
||||
// due?", and "nothing" is a normal, frequent and correct answer.
|
||||
//
|
||||
// OLDEST-FIRST (operator ruling 2026-07-26, Option 1) survives as the ORDER among due tiers: the
|
||||
// tier whose last successful restore-test is oldest goes first, never-proven first of all. It is
|
||||
// self-balancing, needs no config knob, and it still cannot starve a tier — but it no longer decides
|
||||
// that a test happens at all.
|
||||
//
|
||||
// A tier with no settled archive is SKIPPED, not failed — a brand-new offsite tier has nothing to
|
||||
// restore yet, and that is normal, not broken. A tier whose archives cannot be LISTED is likewise
|
||||
// skipped, loudly, and its error is returned only when no other tier was testable: one tier's
|
||||
// storage being unreadable must not cost the other tier its proof, and must not be silent either.
|
||||
//
|
||||
// Returns ("", "", nil) when nothing anywhere is due.
|
||||
func (s *Scheduler) pickForThisRun(ctx context.Context) (archive, target string, err error) {
|
||||
if !s.rotating() {
|
||||
// Pre-R-85 single-tier path (tests and any caller that wires only `Pick`): there is no tier
|
||||
// identity and no persisted proof here, so there is nothing to compare an archive against
|
||||
// and no due-check is possible. It runs on every evaluation, exactly as it always did.
|
||||
a, perr := s.pick(ctx)
|
||||
return a, "", perr
|
||||
}
|
||||
var firstErr error
|
||||
for _, v := range s.EvaluateDue(ctx) {
|
||||
if v.Err != nil {
|
||||
s.logger.Warn("backup: restore-test candidate lookup failed for a tier; trying the next",
|
||||
"target", v.Target, "err", v.Err)
|
||||
if firstErr == nil {
|
||||
firstErr = v.Err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !v.Due {
|
||||
s.logger.Debug("backup: restore-test tier is not due", "target", v.Target, "reason", v.Reason)
|
||||
continue
|
||||
}
|
||||
s.logger.Info("backup: restore-test tier is DUE (per-archive; oldest-proven first among due tiers)",
|
||||
"target", v.Target, "archive", v.Archive, "landed", v.Landed.Format(time.RFC3339), "reason", v.Reason)
|
||||
return v.Archive, v.Target, nil
|
||||
}
|
||||
if firstErr != nil {
|
||||
return "", "", firstErr
|
||||
}
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
)
|
||||
|
||||
// R-85 (1.1) — the spec is built PER RUN, never frozen at construction.
|
||||
//
|
||||
// It used to be an immediately-invoked function at daemon start, so storageTier() and
|
||||
// restoreTaskTimeout() were evaluated ONCE and the value reused for every run for the process
|
||||
// lifetime. That is what made an offsite restore-test impossible to schedule at all, and it was a
|
||||
// latent staleness bug besides: a storage-type or config change did not take effect until restart.
|
||||
|
||||
type specSpy struct {
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
archives []string
|
||||
tiers []string // what the builder decided, per call
|
||||
}
|
||||
|
||||
func (sp *specSpy) build(_ context.Context, archive string) reconcile.RestoreTestSpec {
|
||||
sp.mu.Lock()
|
||||
defer sp.mu.Unlock()
|
||||
sp.calls++
|
||||
sp.archives = append(sp.archives, archive)
|
||||
// Decide the tier from the ARCHIVE, exactly as main.go does (the v0.100.0 rule).
|
||||
tier := "local"
|
||||
if len(archive) > 10 && archive[:10] == "felhom-pbs" {
|
||||
tier = "pbs"
|
||||
}
|
||||
sp.tiers = append(sp.tiers, tier)
|
||||
return reconcile.RestoreTestSpec{
|
||||
RestoreStorage: "local-lvm", ScratchMin: 990000, ScratchMax: 990009, SourceTier: tier,
|
||||
}
|
||||
}
|
||||
|
||||
// COMPANION RED-PROOF (observed): change Scheduler.spec back to a frozen
|
||||
// `reconcile.RestoreTestSpec` value captured at construction → this fails with
|
||||
// "the spec builder must run ONCE PER RUN, got 1 call(s) across 3 ticks", because a frozen value is
|
||||
// evaluated exactly once no matter how many ticks fire. Restored.
|
||||
func TestScheduler_SpecIsBuiltPerRun(t *testing.T) {
|
||||
sp := &specSpy{}
|
||||
rt := &fakeRTRunner{res: reconcile.RestoreTestResult{Pass: true, Verified: "boot+running"}}
|
||||
n := 0
|
||||
s := NewScheduler(SchedulerOptions{
|
||||
Runner: rt,
|
||||
Pick: func(context.Context) (string, error) {
|
||||
n++
|
||||
return fmt.Sprintf("local:backup/vzdump-lxc-9201-%d.tar.zst", n), nil
|
||||
},
|
||||
Store: NewStore(),
|
||||
Spec: sp.build,
|
||||
Cadence: time.Hour,
|
||||
Logger: quiet(),
|
||||
})
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
s.tick(context.Background())
|
||||
}
|
||||
|
||||
sp.mu.Lock()
|
||||
defer sp.mu.Unlock()
|
||||
if sp.calls != 3 {
|
||||
t.Fatalf("the spec builder must run ONCE PER RUN, got %d call(s) across 3 ticks", sp.calls)
|
||||
}
|
||||
// And it must see the archive THIS run picked — not a stale one.
|
||||
for i, a := range sp.archives {
|
||||
want := fmt.Sprintf("local:backup/vzdump-lxc-9201-%d.tar.zst", i+1)
|
||||
if a != want {
|
||||
t.Fatalf("run %d: builder saw archive %q, want %q — the spec is not tracking the picked archive", i+1, a, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The tier must follow the ARCHIVE across runs. A builder that saw only the configured target would
|
||||
// return the same tier every time — which is exactly the v0.100.0 defect that killed a 14.46 GB WAN
|
||||
// restore at the 10-minute local bound.
|
||||
func TestScheduler_SpecTierFollowsTheArchive(t *testing.T) {
|
||||
sp := &specSpy{}
|
||||
rt := &fakeRTRunner{res: reconcile.RestoreTestResult{Pass: true, Verified: "boot+running"}}
|
||||
archives := []string{
|
||||
"local:backup/vzdump-lxc-9201-x.tar.zst",
|
||||
"felhom-pbs:backup/ct/9201/2026-07-26T15:42:42Z",
|
||||
}
|
||||
i := 0
|
||||
s := NewScheduler(SchedulerOptions{
|
||||
Runner: rt,
|
||||
Pick: func(context.Context) (string, error) {
|
||||
a := archives[i%len(archives)]
|
||||
i++
|
||||
return a, nil
|
||||
},
|
||||
Store: NewStore(), Spec: sp.build, Cadence: time.Hour, Logger: quiet(),
|
||||
})
|
||||
s.tick(context.Background())
|
||||
s.tick(context.Background())
|
||||
|
||||
sp.mu.Lock()
|
||||
defer sp.mu.Unlock()
|
||||
if len(sp.tiers) != 2 || sp.tiers[0] != "local" || sp.tiers[1] != "pbs" {
|
||||
t.Fatalf("the tier must follow the archive per run; got %v", sp.tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// A nil spec builder must SKIP loudly, not panic — a wiring bug costs a restore-test, never the
|
||||
// daemon goroutine.
|
||||
func TestScheduler_NilSpecSkipsInsteadOfPanicking(t *testing.T) {
|
||||
rt := &fakeRTRunner{}
|
||||
s := NewScheduler(SchedulerOptions{
|
||||
Runner: rt,
|
||||
Pick: func(context.Context) (string, error) { return "vol", nil },
|
||||
Store: NewStore(), Cadence: time.Hour, Logger: quiet(),
|
||||
})
|
||||
s.tick(context.Background()) // must not panic
|
||||
if rt.runs != 0 {
|
||||
t.Fatalf("a nil spec must not run a restore-test; got %d run(s)", rt.runs)
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,22 @@ import (
|
||||
// Store holds the agent's LATEST backup result per target and the latest restore-test
|
||||
// result — the point-in-time state the host-report surfaces. It is updated by the backup
|
||||
// runner + the restore-test scheduler/selftest and read by the collector via the hub
|
||||
// BackupReporter / RestoreTestReporter seams. In-memory (lost on restart; the cadence
|
||||
// re-populates) and mutex-guarded for the concurrent collector vs scheduler access.
|
||||
// BackupReporter / RestoreTestReporter seams. In-memory and mutex-guarded for the concurrent
|
||||
// collector vs scheduler access.
|
||||
//
|
||||
// **"lost on restart; the cadence re-populates" — that sentence used to be here and it is now
|
||||
// FALSE for restore-tests (R-189, 2026-08-03).** It was true while a timer re-tested every tier
|
||||
// daily. Under R-86's per-archive due-check the agent will NOT re-test an archive it has already
|
||||
// proven, so a proof lost to a restart is not repeated until the next archive generation — a week on
|
||||
// the offsite tier — and the hub reports that tier unproven throughout. Observed, not predicted: a
|
||||
// real 14.5 GB offsite restore passed, the agent was restarted 2 m 43 s later for a deploy, and two
|
||||
// consecutive host-reports carried `0 restore-tests`.
|
||||
//
|
||||
// The durable half is `RestoreTestState` (on disk, per tier, with the archive) and the collector
|
||||
// merges the two — see hub.ProvenRestoreTestReporter. This store remains the ONLY place a FAILURE is
|
||||
// recorded, and that asymmetry is deliberate: a failing tier stays due and is retried, so a lost
|
||||
// failure heals itself, while a lost success leaves the system quietly less tested than it believes.
|
||||
// Backups are unaffected — their freshness has a ground truth on the storage (R-84).
|
||||
type Store struct {
|
||||
mu sync.Mutex
|
||||
byTarget map[string]hub.Backup // latest backup per target id
|
||||
|
||||
+127
-56
@@ -12,6 +12,12 @@
|
||||
// by a sudoers pattern, catching authoring gaps in CI before they ship.
|
||||
package capability
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
||||
)
|
||||
|
||||
// Capability is one privileged command the agent depends on. Name is a stable id; Feature is the
|
||||
// human-readable thing that breaks if the grant is missing (used in logs + the operator alert).
|
||||
// Binary is the absolute path the runner invokes; ReprArgs is a CONCRETE argument vector that
|
||||
@@ -24,8 +30,16 @@ type Capability struct {
|
||||
Binary string
|
||||
ReprArgs []string
|
||||
Critical bool
|
||||
// GatedBy names the config gate this capability sits behind ("" = always active). A healthy
|
||||
// probe of a gated capability whose gate is OFF reports StatusInactive instead of ok — see
|
||||
// Prober.GateActive. Broken plumbing stays degraded regardless of the gate.
|
||||
GatedBy string
|
||||
}
|
||||
|
||||
// GatePBSDR is the gate id for the PBS-DR tier: active when this box carries a live (enabled)
|
||||
// pbs_dr descriptor or a previously-converged marker (pbsdr.Manager.DRConfigured).
|
||||
const GatePBSDR = "pbs_dr"
|
||||
|
||||
// Manifest is the required set, seeded from the 2026-06-29 sudoers audit (felhom-agent/REPORT.md):
|
||||
// the OK + newly-CLOSED rows. The SURFACED/DEFERRED rows are deliberately EXCLUDED — they are not
|
||||
// required capabilities: the general `pct exec <vmid> -- *` (controller-swap; arbitrary exec, an
|
||||
@@ -35,90 +49,147 @@ type Capability struct {
|
||||
//
|
||||
// Each ReprArgs is a representative instance; the probe LISTS it (`sudo -n -l`) and never runs it,
|
||||
// so even mkfs/pct-set entries are side-effect-free to probe.
|
||||
func Manifest() []Capability { return manifest }
|
||||
//
|
||||
// Gating (v0.86.0): every `pbsdr-*` capability is behind GatePBSDR — the whole pbsdr vector IS
|
||||
// the DR-tier apply path, so the gate rides the stable name prefix (a new pbsdr-* op is gated
|
||||
// automatically; nothing else ever is). Applied here, not in the literals, to keep the manifest
|
||||
// entries positional.
|
||||
func Manifest() []Capability {
|
||||
out := make([]Capability, len(manifest))
|
||||
copy(out, manifest)
|
||||
for i := range out {
|
||||
if strings.HasPrefix(out[i].Name, "pbsdr-") {
|
||||
out[i].GatedBy = GatePBSDR
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reprFingerprint is a shape-valid all-zero SHA-256 colon fingerprint for list-mode repr vectors
|
||||
// (matches the wrapper's fingerprint validation; never executed).
|
||||
const reprFingerprint = "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"
|
||||
|
||||
var manifest = []Capability{
|
||||
// ---- Intermediary drive model (the multi-drive path — mostly Critical) ----
|
||||
{"guest-init-pid", "drive-gate guest-sees check (multi-drive concurrency)", "/usr/bin/lxc-info", []string{"-n", "9201", "-p", "-H"}, true},
|
||||
{"parent-self-bind", "intermediary shared-parent self-bind", "/usr/bin/mount", []string{"--bind", "/mnt/felhom-drives", "/mnt/felhom-drives"}, true},
|
||||
{"parent-make-shared", "intermediary shared-parent propagation", "/usr/bin/mount", []string{"--make-shared", "/mnt/felhom-drives"}, true},
|
||||
{"parent-make-private", "intermediary shared-parent peer-group isolation", "/usr/bin/mount", []string{"--make-private", "/mnt/felhom-drives"}, true},
|
||||
{"drive-bind", "drive attach (felhom-data bind under parent)", "/usr/bin/mount", []string{"--bind", "/mnt/felhom-usb/felhom-data", "/mnt/felhom-drives/felhom-usb"}, true},
|
||||
{"drive-umount", "drive detach (fail-closed unmount)", "/usr/bin/umount", []string{"/mnt/felhom-drives/felhom-usb"}, true},
|
||||
{"drives-mkdir-parent", "stable parent dir create", "/usr/bin/mkdir", []string{"-p", "/mnt/felhom-drives"}, false},
|
||||
{"drives-mkdir-sub", "per-drive stable dir create", "/usr/bin/mkdir", []string{"-p", "/mnt/felhom-drives/felhom-usb"}, false},
|
||||
{"drives-mkdir-data", "felhom-data namespace create", "/usr/bin/mkdir", []string{"-p", "/mnt/felhom-usb/felhom-data"}, false},
|
||||
{"drives-chown-data", "felhom-data guest-root chown", "/usr/bin/chown", []string{"100000:100000", "/mnt/felhom-usb/felhom-data"}, false},
|
||||
{"parent-script-install", "shared-parent boot script install", "/usr/bin/install", []string{"-m", "0755", "--", "/tmp/felhom-shared-parent-123456789.sh", "/usr/local/sbin/felhom-shared-parent.sh"}, false},
|
||||
{"parent-unit-install", "shared-parent boot unit install", "/usr/bin/install", []string{"-m", "0644", "--", "/tmp/felhom-shared-parent-123456789.service", "/etc/systemd/system/felhom-shared-parent.service"}, false},
|
||||
{"parent-unit-enable", "shared-parent boot-persistence enable", "/usr/bin/systemctl", []string{"enable", "felhom-shared-parent.service"}, false},
|
||||
{"parent-bind-mp8", "parent bind into guest at provision", "/usr/sbin/pct", []string{"set", "9201", "-mp8", "/mnt/felhom-drives"}, false},
|
||||
{"guest-init-pid", "drive-gate guest-sees check (multi-drive concurrency)", "/usr/bin/lxc-info", []string{"-n", "9201", "-p", "-H"}, true, ""},
|
||||
{"parent-self-bind", "intermediary shared-parent self-bind", "/usr/bin/mount", []string{"--bind", "/mnt/felhom-drives", "/mnt/felhom-drives"}, true, ""},
|
||||
{"parent-make-shared", "intermediary shared-parent propagation", "/usr/bin/mount", []string{"--make-shared", "/mnt/felhom-drives"}, true, ""},
|
||||
{"parent-make-private", "intermediary shared-parent peer-group isolation", "/usr/bin/mount", []string{"--make-private", "/mnt/felhom-drives"}, true, ""},
|
||||
{"drive-bind", "drive attach (felhom-data bind under parent)", "/usr/bin/mount", []string{"--bind", "/mnt/felhom-usb/felhom-data", "/mnt/felhom-drives/felhom-usb"}, true, ""},
|
||||
{"drive-umount", "drive detach (fail-closed unmount)", "/usr/bin/umount", []string{"/mnt/felhom-drives/felhom-usb"}, true, ""},
|
||||
{"drives-mkdir-parent", "stable parent dir create", "/usr/bin/mkdir", []string{"-p", "/mnt/felhom-drives"}, false, ""},
|
||||
{"drives-mkdir-sub", "per-drive stable dir create", "/usr/bin/mkdir", []string{"-p", "/mnt/felhom-drives/felhom-usb"}, false, ""},
|
||||
{"drives-mkdir-data", "felhom-data namespace create", "/usr/bin/mkdir", []string{"-p", "/mnt/felhom-usb/felhom-data"}, false, ""},
|
||||
{"drives-chown-data", "felhom-data guest-root chown", "/usr/bin/chown", []string{"100000:100000", "/mnt/felhom-usb/felhom-data"}, false, ""},
|
||||
{"parent-script-install", "shared-parent boot script install", "/usr/bin/install", []string{"-m", "0755", "--", "/tmp/felhom-shared-parent-123456789.sh", "/usr/local/sbin/felhom-shared-parent.sh"}, false, ""},
|
||||
{"parent-unit-install", "shared-parent boot unit install", "/usr/bin/install", []string{"-m", "0644", "--", "/tmp/felhom-shared-parent-123456789.service", "/etc/systemd/system/felhom-shared-parent.service"}, false, ""},
|
||||
{"parent-unit-enable", "shared-parent boot-persistence enable", "/usr/bin/systemctl", []string{"enable", "felhom-shared-parent.service"}, false, ""},
|
||||
{"parent-bind-mp8", "parent bind into guest at provision", "/usr/sbin/pct", []string{"set", "9201", "-mp8", "/mnt/felhom-drives"}, false, ""},
|
||||
|
||||
// ---- Disk inspect / format gate (Critical: the data-bearing classifier + format) ----
|
||||
{"disk-blkid", "disk data-bearing classify (format gate)", "/usr/sbin/blkid", []string{"-p", "-o", "export", "/dev/sda"}, true},
|
||||
{"disk-lsblk", "disk topology read (format gate)", "/usr/bin/lsblk", []string{"-J", "-o", "NAME,FSTYPE,PTTYPE,MOUNTPOINT", "/dev/sda"}, true},
|
||||
{"disk-mkfs-ext4", "guarded format (ext4)", "/usr/local/sbin/felhom-mkfs-guarded", []string{"/dev/sda", "ext4"}, true},
|
||||
{"disk-mkfs-xfs", "guarded format (xfs)", "/usr/local/sbin/felhom-mkfs-guarded", []string{"/dev/sda", "xfs"}, false},
|
||||
{"disk-smart", "disk SMART health read", "/usr/sbin/smartctl", []string{"-a", "-j", "/dev/sda"}, false},
|
||||
{"disk-lvs", "thin-pool usage read", "/usr/sbin/lvs", []string{"--reportformat", "json", "--units", "b", "-o", "lv_name,data_percent,metadata_percent", "--", "pve/data"}, false},
|
||||
{"disk-blkid", "disk data-bearing classify (format gate)", "/usr/sbin/blkid", []string{"-p", "-o", "export", "/dev/sda"}, true, ""},
|
||||
{"disk-lsblk", "disk topology read (format gate)", "/usr/bin/lsblk", []string{"-J", "-o", "NAME,FSTYPE,PTTYPE,MOUNTPOINT", "/dev/sda"}, true, ""},
|
||||
{"disk-mkfs-ext4", "guarded format (ext4)", "/usr/local/sbin/felhom-mkfs-guarded", []string{"/dev/sda", "ext4"}, true, ""},
|
||||
{"disk-mkfs-xfs", "guarded format (xfs)", "/usr/local/sbin/felhom-mkfs-guarded", []string{"/dev/sda", "xfs"}, false, ""},
|
||||
{"disk-smart", "disk SMART health read", "/usr/sbin/smartctl", []string{"-a", "-j", "/dev/sda"}, false, ""},
|
||||
{"disk-lvs", "thin-pool usage read", "/usr/sbin/lvs", []string{"--reportformat", "json", "--units", "b", "-o", "lv_name,data_percent,metadata_percent", "--", "pve/data"}, false, ""},
|
||||
|
||||
// ---- Storage mount units (watchdog re-mount) ----
|
||||
{"mount-unit-install", "fs-UUID mount unit install", "/usr/bin/install", []string{"-o", "root", "-g", "root", "-m", "0644", "--", "/var/lib/felhom-agent/units/felhom-x.mount", "/etc/systemd/system/felhom-x.mount"}, false},
|
||||
{"mount-daemon-reload", "systemd reload after unit write", "/usr/bin/systemctl", []string{"daemon-reload"}, false},
|
||||
{"mount-unit-enable", "mount unit enable", "/usr/bin/systemctl", []string{"enable", "--now", "--", "felhom-x.mount"}, false},
|
||||
{"mount-unit-disable", "mount unit disable", "/usr/bin/systemctl", []string{"disable", "--", "felhom-x.mount"}, false},
|
||||
{"mount-unit-stop", "mount unit stop", "/usr/bin/systemctl", []string{"stop", "--", "felhom-x.mount"}, false},
|
||||
{"mount-unit-install", "fs-UUID mount unit install", "/usr/bin/install", []string{"-o", "root", "-g", "root", "-m", "0644", "--", "/var/lib/felhom-agent/units/felhom-x.mount", "/etc/systemd/system/felhom-x.mount"}, false, ""},
|
||||
{"mount-daemon-reload", "systemd reload after unit write", "/usr/bin/systemctl", []string{"daemon-reload"}, false, ""},
|
||||
{"mount-unit-enable", "mount unit enable", "/usr/bin/systemctl", []string{"enable", "--now", "--", "felhom-x.mount"}, false, ""},
|
||||
{"mount-unit-disable", "mount unit disable", "/usr/bin/systemctl", []string{"disable", "--", "felhom-x.mount"}, false, ""},
|
||||
{"mount-unit-stop", "mount unit stop", "/usr/bin/systemctl", []string{"stop", "--", "felhom-x.mount"}, false, ""},
|
||||
|
||||
// ---- Network storage re-arm + cleanup (CAMPAIGN-3 F10/F1) ----
|
||||
{"netmount-reset-failed", "NAS automount re-arm after start-limit (F10)", "/usr/bin/systemctl", []string{"reset-failed", "--", "mnt-felhom\\x2ddrives-media.automount"}, false, ""},
|
||||
{"netmount-rmdir", "removed-share mountpoint cleanup (F1)", "/usr/bin/rmdir", []string{"/mnt/felhom-drives/media"}, false, ""},
|
||||
|
||||
// ---- Node self-heal (CAMPAIGN-3 F12-class, appliance-gated in code) ----
|
||||
{"selfheal-networking-start", "appliance networking recovery at boot (F12 defense in depth)", "/usr/bin/systemctl", []string{"start", "networking.service"}, false, ""},
|
||||
|
||||
// ---- Provisioning back-half ----
|
||||
{"provision-chown", "bootstrap mount guest-root chown", "/usr/bin/chown", []string{"-R", "100000:100000", "/var/lib/felhom-agent/guests/9201"}, false},
|
||||
{"provision-config-mount", "bootstrap config bind mount", "/usr/sbin/pct", []string{"set", "9201", "-mp0", "/var/lib/felhom-agent/guests/9201"}, false},
|
||||
{"provision-onboot", "customer guest autostart (onboot)", "/usr/sbin/pct", []string{"set", "9201", "-onboot", "1"}, false},
|
||||
{"provision-chown", "bootstrap mount guest-root chown", "/usr/bin/chown", []string{"-R", "100000:100000", "/var/lib/felhom-agent/guests/9201"}, false, ""},
|
||||
{"provision-config-mount", "bootstrap config bind mount", "/usr/sbin/pct", []string{"set", "9201", "-mp0", "/var/lib/felhom-agent/guests/9201"}, false, ""},
|
||||
{"provision-onboot", "customer guest autostart (onboot)", "/usr/sbin/pct", []string{"set", "9201", "-onboot", "1"}, false, ""},
|
||||
|
||||
// ---- Pre-start self-heal hook + guest lifecycle ----
|
||||
{"guesthook-install", "pre-start hook snippet install", "/usr/bin/install", []string{"-m", "0755", "--", "/tmp/felhom-guest-hook-123456789.sh", "/var/lib/vz/snippets/felhom-guest-hook.sh"}, false},
|
||||
{"guesthook-register", "pre-start hook register", "/usr/sbin/pct", []string{"set", "9201", "--hookscript", "local:snippets/felhom-guest-hook.sh"}, false},
|
||||
{"guesthook-delete-mp", "dead mountpoint slot delete (C1 net)", "/usr/sbin/pct", []string{"set", "9201", "--delete", "mp0"}, false},
|
||||
{"guest-reboot", "enroll activate-binds reboot", "/usr/sbin/pct", []string{"reboot", "9201"}, false},
|
||||
{"guesthook-install", "pre-start hook snippet install", "/usr/bin/install", []string{"-m", "0755", "--", "/tmp/felhom-guest-hook-123456789.sh", "/var/lib/vz/snippets/felhom-guest-hook.sh"}, false, ""},
|
||||
{"guesthook-register", "pre-start hook register", "/usr/sbin/pct", []string{"set", "9201", "--hookscript", "local:snippets/felhom-guest-hook.sh"}, false, ""},
|
||||
{"guesthook-delete-mp", "dead mountpoint slot delete (C1 net)", "/usr/sbin/pct", []string{"set", "9201", "--delete", "mp0"}, false, ""},
|
||||
{"guest-reboot", "enroll activate-binds reboot", "/usr/sbin/pct", []string{"reboot", "9201"}, false, ""},
|
||||
|
||||
// ---- LAN split-horizon resolver (dnsmasq) ----
|
||||
{"dnsmasq-install", "dnsmasq package install", "/usr/bin/apt-get", []string{"install", "-y", "-q", "dnsmasq"}, false},
|
||||
{"dnsmasq-write", "dnsmasq drop-in write", "/usr/bin/install", []string{"-m", "0644", "/tmp/felhom-resolver-x.conf", "/etc/dnsmasq.d/felhom-x.conf"}, false},
|
||||
{"dnsmasq-enable", "dnsmasq enable", "/usr/bin/systemctl", []string{"enable", "--now", "dnsmasq"}, false},
|
||||
{"dnsmasq-reload", "dnsmasq reload", "/usr/bin/systemctl", []string{"reload", "dnsmasq"}, false},
|
||||
{"dnsmasq-restart", "dnsmasq restart (LAN-DNS self-heal)", "/usr/bin/systemctl", []string{"restart", "dnsmasq"}, false},
|
||||
{"dnsmasq-rm", "dnsmasq drop-in remove (decommission)", "/usr/bin/rm", []string{"-f", "/etc/dnsmasq.d/felhom-x.conf"}, false},
|
||||
{"dnsmasq-guest-ip", "guest LAN IP discovery", "/usr/sbin/pct", []string{"exec", "9201", "--", "ip", "-4", "-o", "addr", "show", "dev", "eth0"}, false},
|
||||
{"dnsmasq-guest-domain", "guest domain discovery", "/usr/sbin/pct", []string{"exec", "9201", "--", "docker", "exec", "felhom-controller", "cat", "/opt/docker/felhom-controller/controller.yaml"}, false},
|
||||
{"dnsmasq-install", "dnsmasq package install", "/usr/bin/apt-get", []string{"install", "-y", "-q", "dnsmasq"}, false, ""},
|
||||
{"dnsmasq-write", "dnsmasq drop-in write", "/usr/bin/install", []string{"-m", "0644", "/tmp/felhom-resolver-x.conf", "/etc/dnsmasq.d/felhom-x.conf"}, false, ""},
|
||||
{"dnsmasq-enable", "dnsmasq enable", "/usr/bin/systemctl", []string{"enable", "--now", "dnsmasq"}, false, ""},
|
||||
{"dnsmasq-reload", "dnsmasq reload", "/usr/bin/systemctl", []string{"reload", "dnsmasq"}, false, ""},
|
||||
{"dnsmasq-restart", "dnsmasq restart (LAN-DNS self-heal)", "/usr/bin/systemctl", []string{"restart", "dnsmasq"}, false, ""},
|
||||
{"dnsmasq-rm", "dnsmasq drop-in remove (decommission)", "/usr/bin/rm", []string{"-f", "/etc/dnsmasq.d/felhom-x.conf"}, false, ""},
|
||||
{"dnsmasq-guest-ip", "guest LAN IP discovery", "/usr/sbin/pct", []string{"exec", "9201", "--", "ip", "-4", "-o", "addr", "show", "dev", "eth0"}, false, ""},
|
||||
{"dnsmasq-guest-domain", "guest domain discovery", "/usr/sbin/pct", []string{"exec", "9201", "--", "docker", "exec", "felhom-controller", "cat", "/opt/docker/felhom-controller/controller.yaml"}, false, ""},
|
||||
|
||||
// ---- Guest-network watchdog (FELHOM_GUESTNET, R-54, v0.92.0) ----
|
||||
// Deliberately NOT Critical: a host that has not yet taken the new sudoers file should show
|
||||
// degraded (visible, diagnosable) without paging an operator for every box on rollout day —
|
||||
// the R-50b lesson that an amber fleet is background noise. The address read is covered by
|
||||
// dnsmasq-guest-ip above and is not duplicated here (one command, one capability row).
|
||||
{"guestnet-route", "guest default-route probe", "/usr/sbin/pct", []string{"exec", "9201", "--", "ip", "route", "show", "default"}, false, ""},
|
||||
{"guestnet-ifaces", "guest interface-mode read", "/usr/sbin/pct", []string{"exec", "9201", "--", "cat", "/etc/network/interfaces"}, false, ""},
|
||||
{"guestnet-dhclient-probe", "guest DHCP-client liveness probe", "/usr/sbin/pct", []string{"exec", "9201", "--", "pgrep", "-x", "dhclient"}, false, ""},
|
||||
{"guestnet-heal", "guest DHCP-client restart (the 2026-07-20 heal)", "/usr/sbin/pct", []string{"exec", "9201", "--", "dhclient", "-pf", "/run/dhclient.eth0.pid", "-lf", "/var/lib/dhcp/dhclient.eth0.leases", "eth0"}, false, ""},
|
||||
|
||||
// ---- Controller-swap / managed auto-update (FELHOM_CONTROLLERSWAP, v0.45.0; Critical: a
|
||||
// silently-broken fleet auto-update is operator-alert-worthy) ----
|
||||
{"controllerswap-read", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "cat", "/etc/felhom-controller-image"}, true},
|
||||
{"controllerswap-image-inspect", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "docker", "image", "inspect", "gitea.dooplex.hu/admin/felhom-controller:0.0.0"}, true},
|
||||
{"controllerswap-inspect", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "docker", "inspect", "-f", "{{.State.Running}}", "felhom-controller"}, true},
|
||||
{"controllerswap-restart", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "systemctl", "restart", "felhom-controller-bootstrap.service"}, true},
|
||||
{"controllerswap-write", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "tee", "/etc/felhom-controller-image"}, true},
|
||||
{"controllerswap-read", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "cat", "/etc/felhom-controller-image"}, true, ""},
|
||||
{"controllerswap-image-inspect", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "docker", "image", "inspect", "gitea.dooplex.hu/admin/felhom-controller:0.0.0"}, true, ""},
|
||||
{"controllerswap-inspect", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "docker", "inspect", "-f", "{{.State.Running}}", "felhom-controller"}, true, ""},
|
||||
{"controllerswap-restart", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "systemctl", "restart", "felhom-controller-bootstrap.service"}, true, ""},
|
||||
{"controllerswap-write", "controller-swap / managed auto-update", "/usr/sbin/pct", []string{"exec", "9201", "--", "tee", "/etc/felhom-controller-image"}, true, ""},
|
||||
|
||||
// ---- Stale-lock recovery (FELHOM_STALELOCK, v0.49.0; Critical: a guest stuck behind a stale
|
||||
// reboot-during-backup lock can't start → the customer box stays DOWN until this clears it) ----
|
||||
{"stalelock-unlock", "reboot-during-backup stale-lock recovery", "/usr/sbin/pct", []string{"unlock", "9201"}, true},
|
||||
{"stalelock-unlock", "reboot-during-backup stale-lock recovery", "/usr/sbin/pct", []string{"unlock", "9201"}, true, ""},
|
||||
|
||||
// ---- Offsite WG tunnel (FELHOM_WG, S3/v0.64.0; Critical FLIPPED in S4/v0.66.0 — offsite
|
||||
// backups now RIDE the tunnel, so a degraded tunnel capability is operator-alert-worthy: the
|
||||
// conf install, unit enable/restart and the handshake read gate the backup path. apt-install
|
||||
// (one-time bootstrap) and disable (revocation, a deliberate teardown) stay non-critical. The
|
||||
// handshake read is the ONLY wg invocation (never `dump`). ----
|
||||
{"wg-tools-install", "wireguard-tools package install", "/usr/bin/apt-get", []string{"install", "-y", "-q", "wireguard-tools"}, false},
|
||||
{"wg-conf-install", "wg-felhom conf install", "/usr/bin/install", []string{"-o", "root", "-g", "root", "-m", "0600", "--", "/var/lib/felhom-agent/wg/wg-felhom.conf", "/etc/wireguard/wg-felhom.conf"}, true},
|
||||
{"wg-enable", "wg-quick@wg-felhom enable", "/usr/bin/systemctl", []string{"enable", "--now", "wg-quick@wg-felhom"}, true},
|
||||
{"wg-restart", "wg-quick@wg-felhom restart (conf change)", "/usr/bin/systemctl", []string{"restart", "wg-quick@wg-felhom"}, true},
|
||||
{"wg-disable", "wg-quick@wg-felhom disable (revocation)", "/usr/bin/systemctl", []string{"disable", "--now", "wg-quick@wg-felhom"}, false},
|
||||
{"wg-handshake-read", "tunnel handshake-age read", "/usr/bin/wg", []string{"show", "wg-felhom", "latest-handshakes"}, true},
|
||||
{"wg-tools-install", "wireguard-tools package install", "/usr/bin/apt-get", []string{"install", "-y", "-q", "wireguard-tools"}, false, ""},
|
||||
{"wg-conf-install", "wg-felhom conf install", "/usr/bin/install", []string{"-o", "root", "-g", "root", "-m", "0600", "--", "/var/lib/felhom-agent/wg/wg-felhom.conf", "/etc/wireguard/wg-felhom.conf"}, true, ""},
|
||||
{"wg-enable", "wg-quick@wg-felhom enable", "/usr/bin/systemctl", []string{"enable", "--now", "wg-quick@wg-felhom"}, true, ""},
|
||||
{"wg-restart", "wg-quick@wg-felhom restart (conf change)", "/usr/bin/systemctl", []string{"restart", "wg-quick@wg-felhom"}, true, ""},
|
||||
{"wg-disable", "wg-quick@wg-felhom disable (revocation)", "/usr/bin/systemctl", []string{"disable", "--now", "wg-quick@wg-felhom"}, false, ""},
|
||||
{"wg-handshake-read", "tunnel handshake-age read", "/usr/bin/wg", []string{"show", "wg-felhom", "latest-handshakes"}, true, ""},
|
||||
|
||||
// ---- PBS DR tier apply (FELHOM_PBSDR, slice 2). NON-critical (the selfupdate rationale):
|
||||
// applying the tier is an occasional hub-driven provisioning op, not a steady-state serving
|
||||
// path — a degraded grant means "can't provision/reconcile the PBS entry" (the bridge reports
|
||||
// loudly anyway), not a serving outage. The steady-state backup path is covered by the wg +
|
||||
// storage capabilities. List-mode representations only; never executed. ----
|
||||
{"pbsdr-create", "PBS DR storage-entry create (K autogen)", "/usr/local/sbin/felhom-pbs-apply", []string{"create", "felhom-pbs", "10.77.0.1", "felhom-offsite", "ns0", "felhom@pbs!ns0", reprFingerprint, "/etc/pve/priv/storage"}, false, ""},
|
||||
{"pbsdr-reconcile", "PBS DR storage-entry reconcile (set-only)", "/usr/local/sbin/felhom-pbs-apply", []string{"reconcile", "felhom-pbs", "10.77.0.1", "ns0", "felhom@pbs!ns0", reprFingerprint, "/etc/pve/priv/storage"}, false, ""},
|
||||
{"pbsdr-grant", "PBS DR storage ACL self-grant", "/usr/local/sbin/felhom-pbs-apply", []string{"grant", "felhom-pbs"}, false, ""},
|
||||
// R-39 leg (b), v0.91.0: the credential READ path. Its absence is what made the PBS verify loop
|
||||
// permanently blind to an applied-but-401 tier, so a host missing this verb is DEGRADED in a way
|
||||
// that matters — it cannot detect the failure this whole tier exists to survive.
|
||||
{"pbsdr-read", "PBS DR credential read (verify-loop auth probe)", "/usr/local/sbin/felhom-pbs-apply", []string{"read", "felhom-pbs", "/etc/pve/priv/storage"}, false, ""},
|
||||
|
||||
// ---- Escrow ceremony (FELHOM_ESCROW, controller-driven, v0.88.0). Critical: the customer
|
||||
// wizard's whole run path IS this one grant — a dropped line silently breaks every ceremony.
|
||||
// GatedBy is set EXPLICITLY (the name deliberately says "escrow", not "pbsdr-": the feature is
|
||||
// the ceremony, but it only exists behind the DR tier — no PBS key, no ceremony). ReprArgs is
|
||||
// the SHARED argv constant (internal/escrow/ceremony.go) — the exec runner uses the same one,
|
||||
// so runner ↔ manifest can't drift, and TestManifestCoveredBySudoers locks manifest ↔ sudoers.
|
||||
// List-mode probe only (`sudo -n -l`), spike-confirmed side-effect-free — never a real ceremony.
|
||||
{"escrow-ceremony", "customer recovery-code ceremony (controller-driven)", escrow.CeremonyBinary, escrow.CeremonyArgs(), true, GatePBSDR},
|
||||
|
||||
// ---- Agent self-update (FELHOM_SELFUPDATE, D1). NON-critical: self-update is an occasional
|
||||
// operator-driven op, not a steady-state serving path — a degraded grant means "can't
|
||||
// self-update" (fall back to a manual SSH deploy), not a serving outage. The apply repr uses a
|
||||
// staging-dir path + a placeholder sha (list-mode never runs it). ----
|
||||
{"selfupdate-apply", "agent self-update apply (A/B flip)", "/usr/local/sbin/felhom-selfupdate-guarded", []string{"apply", "/var/lib/felhom-agent/selfupdate/felhom-agent-0.0.0", "0000000000000000000000000000000000000000000000000000000000000000"}, false},
|
||||
{"selfupdate-commit", "agent self-update commit", "/usr/local/sbin/felhom-selfupdate-guarded", []string{"commit"}, false},
|
||||
{"selfupdate-rollback", "agent self-update rollback", "/usr/local/sbin/felhom-selfupdate-guarded", []string{"rollback"}, false},
|
||||
{"selfupdate-apply", "agent self-update apply (A/B flip)", "/usr/local/sbin/felhom-selfupdate-guarded", []string{"apply", "/var/lib/felhom-agent/selfupdate/felhom-agent-0.0.0", "0000000000000000000000000000000000000000000000000000000000000000"}, false, ""},
|
||||
{"selfupdate-commit", "agent self-update commit", "/usr/local/sbin/felhom-selfupdate-guarded", []string{"commit"}, false, ""},
|
||||
{"selfupdate-rollback", "agent self-update rollback", "/usr/local/sbin/felhom-selfupdate-guarded", []string{"rollback"}, false, ""},
|
||||
}
|
||||
|
||||
@@ -2,9 +2,12 @@ package capability
|
||||
|
||||
import (
|
||||
"os"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
||||
)
|
||||
|
||||
// sudoersPath is the in-repo allowlist, relative to this test file (internal/capability/).
|
||||
@@ -217,6 +220,50 @@ func TestRedProof_DroppedControllerSwapTeeFailsCheck(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestEscrowCeremonyArgvPinned locks the ceremony argv copies together (Scenario G, v0.88.0).
|
||||
// The exec runner and the manifest entry both consume escrow.CeremonyArgs() (one shared source),
|
||||
// and TestManifestCoveredBySudoers proves manifest ⊆ sudoers — so pinning the shared source to
|
||||
// the EXPECTED literal here transitively locks all three: runner == manifest == sudoers.
|
||||
// Red-proof: mutate one element of the argv in internal/escrow/ceremony.go and THIS test fails
|
||||
// (and so does the sudoers coverage); a sudoers-side mutation is caught by the existing
|
||||
// TestRedProof_* machinery.
|
||||
func TestEscrowCeremonyArgvPinned(t *testing.T) {
|
||||
wantBinary := "/usr/local/bin/felhom-agent"
|
||||
wantArgs := []string{"--config", "/etc/felhom-agent/agent.json", "--selftest=escrow-create", "--upload", "--output=json"}
|
||||
|
||||
if escrow.CeremonyBinary != wantBinary {
|
||||
t.Errorf("escrow.CeremonyBinary = %q, want %q", escrow.CeremonyBinary, wantBinary)
|
||||
}
|
||||
if got := escrow.CeremonyArgs(); !reflect.DeepEqual(got, wantArgs) {
|
||||
t.Errorf("escrow.CeremonyArgs() = %q, want %q (the sudoers line + manifest entry must stay byte-identical)", got, wantArgs)
|
||||
}
|
||||
|
||||
var entry Capability
|
||||
for _, c := range Manifest() {
|
||||
if c.Name == "escrow-ceremony" {
|
||||
entry = c
|
||||
}
|
||||
}
|
||||
if entry.Name == "" {
|
||||
t.Fatal("manifest missing escrow-ceremony")
|
||||
}
|
||||
if entry.Binary != escrow.CeremonyBinary || !reflect.DeepEqual(entry.ReprArgs, escrow.CeremonyArgs()) {
|
||||
t.Errorf("manifest escrow-ceremony argv diverged from the shared constant: %s %q", entry.Binary, entry.ReprArgs)
|
||||
}
|
||||
if !entry.Critical {
|
||||
t.Error("escrow-ceremony must be Critical (the wizard's whole run path is this one grant)")
|
||||
}
|
||||
if entry.GatedBy != GatePBSDR {
|
||||
t.Errorf("escrow-ceremony GatedBy = %q, want %q (no PBS key → no ceremony; inactive, never red, on a DR-off box)", entry.GatedBy, GatePBSDR)
|
||||
}
|
||||
// CeremonyArgs must return a COPY — a caller mutating its slice must not poison the source.
|
||||
mutated := escrow.CeremonyArgs()
|
||||
mutated[0] = "--poisoned"
|
||||
if got := escrow.CeremonyArgs(); !reflect.DeepEqual(got, wantArgs) {
|
||||
t.Error("escrow.CeremonyArgs() shares its backing array — callers can mutate the source")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWGCapabilityCriticality pins the exact S4 (v0.66.0) Critical set for the FELHOM_WG entries:
|
||||
// the backup path (conf install, unit enable/restart, handshake read) is operator-alert-worthy now
|
||||
// that offsite backups ride the tunnel; the one-time apt install and the deliberate disable
|
||||
|
||||
@@ -8,19 +8,30 @@ import (
|
||||
// Status is one capability's live result — the wire shape the agent attaches to its hub report
|
||||
// (HostReport.Capabilities). The hub mirrors this struct field-for-field and keys its alert on
|
||||
// Critical+degraded. Reason is empty when ok.
|
||||
//
|
||||
// "inactive" (v0.86.0, DR-tier-by-default): a config-GATED capability whose plumbing is HEALTHY
|
||||
// (binary present, sudo granted) but whose gating feature is disabled by configuration. Distinct
|
||||
// from degraded on purpose — disabled ≠ broken; the hub renders it as a neutral chip, never red.
|
||||
// Broken plumbing (binary missing / grant denied) stays DEGRADED even when the gate is off: an
|
||||
// un-migrated box must never look deliberately disabled.
|
||||
type Status struct {
|
||||
Name string `json:"name"`
|
||||
Feature string `json:"feature"`
|
||||
Critical bool `json:"critical"`
|
||||
Status string `json:"status"` // "ok" | "degraded"
|
||||
Status string `json:"status"` // "ok" | "degraded" | "inactive"
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
StatusOK = "ok"
|
||||
StatusDegraded = "degraded"
|
||||
StatusInactive = "inactive"
|
||||
)
|
||||
|
||||
// ReasonInactive is the fixed reason string for the inactive state (the hub + operator docs
|
||||
// reference it verbatim).
|
||||
const ReasonInactive = "disabled by configuration"
|
||||
|
||||
// Runner is the minimal exec seam the probe needs (satisfied by proxmox.ExecRunner). The probe
|
||||
// runs `sudo -n -l -- <binary> <args…>` LITERALLY — a sudo POLICY LIST that never executes the
|
||||
// command — so the Runner MUST be a DIRECT runner (RunnerDirect), not the sudo-prepending one
|
||||
@@ -31,9 +42,13 @@ type Runner interface {
|
||||
|
||||
// Prober checks the manifest against the live host. Exists defaults to an os.Stat check on the
|
||||
// absolute binary path (what `command -v` would resolve for an absolute path) when nil.
|
||||
// GateActive answers "is the feature behind this gate id configured on?" for GATED capabilities
|
||||
// (Capability.GatedBy). nil, or a gate it answers true for, keeps the historical behavior; false
|
||||
// downgrades a HEALTHY probe to StatusInactive (broken plumbing stays degraded regardless).
|
||||
type Prober struct {
|
||||
Runner Runner
|
||||
Exists func(path string) bool // nil → os.Stat
|
||||
Runner Runner
|
||||
Exists func(path string) bool // nil → os.Stat
|
||||
GateActive func(gate string) bool // nil → every gate treated active
|
||||
}
|
||||
|
||||
// Probe lists every manifest capability against the sudo policy and checks its binary exists,
|
||||
@@ -72,6 +87,11 @@ func (p Prober) Probe(ctx context.Context) []Status {
|
||||
case p.Runner != nil && !p.granted(ctx, c):
|
||||
s.Status, s.Reason = StatusDegraded, "sudo policy denied"
|
||||
}
|
||||
// Config gate (v0.86.0): only a HEALTHY probe is downgraded to inactive — a degraded one
|
||||
// stays degraded (missing binary/grant = un-migrated or mis-installed box, never "off").
|
||||
if s.Status == StatusOK && c.GatedBy != "" && p.GateActive != nil && !p.GateActive(c.GatedBy) {
|
||||
s.Status, s.Reason = StatusInactive, ReasonInactive
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
@@ -85,13 +105,16 @@ func (p Prober) granted(ctx context.Context, c Capability) bool {
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Summarize returns (okCount, total, degraded) for logging. degraded lists every non-ok status.
|
||||
// Summarize returns (okCount, total, degraded) for logging. degraded lists DEGRADED statuses
|
||||
// only — inactive is a deliberate, healthy state and must not land in the error log (it is
|
||||
// counted via len(statuses)-ok-len(degraded) by callers that want it).
|
||||
func Summarize(statuses []Status) (ok, total int, degraded []Status) {
|
||||
total = len(statuses)
|
||||
for _, s := range statuses {
|
||||
if s.Status == StatusOK {
|
||||
switch s.Status {
|
||||
case StatusOK:
|
||||
ok++
|
||||
} else {
|
||||
case StatusDegraded:
|
||||
degraded = append(degraded, s)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,3 +115,88 @@ func TestProbe_NilRunnerNoPanic(t *testing.T) {
|
||||
t.Fatalf("nil-runner probe returned %d statuses, want %d", got, len(Manifest()))
|
||||
}
|
||||
}
|
||||
|
||||
// ── DR-tier gate (v0.86.0) ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
// Gate OFF + healthy plumbing → the gated pbsdr-* capabilities report INACTIVE (the neutral
|
||||
// "disabled by configuration" state), NOT ok and NOT degraded — Scenario B of the DR-by-default
|
||||
// spec. Ungated siblings are untouched. Red-proof partner: collapse inactive into ok (drop the
|
||||
// gate branch in Probe) → this fails while TestProbe_GateOffBinaryMissingStaysDegraded passes.
|
||||
func TestProbe_GateOffHealthyIsInactive(t *testing.T) {
|
||||
r := &fakeRunner{denyBinary: map[string]bool{}}
|
||||
p := Prober{
|
||||
Runner: r,
|
||||
Exists: func(string) bool { return true },
|
||||
GateActive: func(gate string) bool { return gate != GatePBSDR }, // DR tier OFF
|
||||
}
|
||||
statuses := p.Probe(context.Background())
|
||||
// v0.88.0: escrow-ceremony joins the gate EXPLICITLY (non-pbsdr name, GatedBy literal) —
|
||||
// the ceremony only exists behind the DR tier (no PBS key, no ceremony).
|
||||
// v0.91.0: pbsdr-read (the R-39 credential-read verb) rides the same `pbsdr-` prefix gate — a new
|
||||
// pbsdr-* op is gated by construction, which is exactly the property this list is here to hold.
|
||||
for _, name := range []string{"pbsdr-create", "pbsdr-reconcile", "pbsdr-grant", "pbsdr-read", "escrow-ceremony"} {
|
||||
s := find(statuses, name)
|
||||
if s.Status != StatusInactive || s.Reason != ReasonInactive {
|
||||
t.Fatalf("%s = %+v, want inactive/%q", name, s, ReasonInactive)
|
||||
}
|
||||
}
|
||||
// An ungated sibling stays plain ok.
|
||||
if s := find(statuses, "drive-bind"); s.Status != StatusOK {
|
||||
t.Fatalf("drive-bind = %+v, want ok (ungated)", s)
|
||||
}
|
||||
// Summarize must NOT count inactive as degraded (it is not error-log-worthy).
|
||||
ok, total, degraded := Summarize(statuses)
|
||||
if len(degraded) != 0 {
|
||||
t.Fatalf("inactive leaked into degraded: %+v", degraded)
|
||||
}
|
||||
if ok != total-5 {
|
||||
t.Fatalf("ok=%d total=%d, want exactly the 5 gated ones non-ok", ok, total)
|
||||
}
|
||||
}
|
||||
|
||||
// Gate OFF + BROKEN plumbing (binary missing) → DEGRADED stays degraded. An un-migrated
|
||||
// pre-v1.15.0 box must never masquerade as deliberately disabled ("never silently pretend").
|
||||
func TestProbe_GateOffBinaryMissingStaysDegraded(t *testing.T) {
|
||||
r := &fakeRunner{denyBinary: map[string]bool{}}
|
||||
p := Prober{
|
||||
Runner: r,
|
||||
Exists: func(path string) bool { return path != "/usr/local/sbin/felhom-pbs-apply" },
|
||||
GateActive: func(gate string) bool { return gate != GatePBSDR }, // DR tier OFF
|
||||
}
|
||||
statuses := p.Probe(context.Background())
|
||||
for _, name := range []string{"pbsdr-create", "pbsdr-reconcile", "pbsdr-grant"} {
|
||||
s := find(statuses, name)
|
||||
if s.Status != StatusDegraded || s.Reason != "binary not found" {
|
||||
t.Fatalf("%s = %+v, want degraded/binary not found even with the gate off", name, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Gate ON (DR configured) + healthy plumbing → plain ok, exactly the pre-v0.86.0 behavior.
|
||||
// A nil GateActive must behave the same (fails ACTIVE).
|
||||
func TestProbe_GateOnOrNilIsOK(t *testing.T) {
|
||||
for _, gate := range []func(string) bool{nil, func(string) bool { return true }} {
|
||||
r := &fakeRunner{denyBinary: map[string]bool{}}
|
||||
p := Prober{Runner: r, Exists: func(string) bool { return true }, GateActive: gate}
|
||||
statuses := p.Probe(context.Background())
|
||||
if s := find(statuses, "pbsdr-create"); s.Status != StatusOK {
|
||||
t.Fatalf("pbsdr-create = %+v, want ok (gate active/nil)", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The gate covers exactly the pbsdr-* entries (name-prefix mechanism) PLUS escrow-ceremony (an
|
||||
// explicit GatedBy literal — v0.88.0: the ceremony only exists behind the DR tier, but its name
|
||||
// says what the feature is). Nothing else may be gated (a regression here would silently un-gate
|
||||
// the tier or gate an unrelated capability).
|
||||
func TestManifest_ExactlyPBSDRGated(t *testing.T) {
|
||||
for _, c := range Manifest() {
|
||||
wantGated := strings.HasPrefix(c.Name, "pbsdr-") || c.Name == "escrow-ceremony"
|
||||
if gated := c.GatedBy == GatePBSDR; gated != wantGated {
|
||||
t.Fatalf("%s: GatedBy=%q, want gated=%v", c.Name, c.GatedBy, wantGated)
|
||||
}
|
||||
if c.GatedBy != "" && c.GatedBy != GatePBSDR {
|
||||
t.Fatalf("%s: unknown gate id %q", c.Name, c.GatedBy)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// R-82 Slice A.1 — per-target cadence + retention resolution.
|
||||
//
|
||||
// The load-bearing property is ADDITIVITY: every config that exists on a live box today must
|
||||
// resolve to exactly one tier that behaves as it does now. The second property is that a
|
||||
// mis-configured tier is REJECTED LOUDLY rather than defaulted — a weekly DR tier silently running
|
||||
// daily would fill the datastore, and a silently dropped tier is the "applied and empty" fault
|
||||
// R-82 exists to fix.
|
||||
|
||||
func TestBackupTiers_LegacyConfigIsUnchanged(t *testing.T) {
|
||||
// Exactly the shape live on demo-felhom today.
|
||||
var b BackupConfig
|
||||
raw := `{"local_backup_target":"local","local_backup_retention":3,"backup_cadence_seconds":0}`
|
||||
if err := json.Unmarshal([]byte(raw), &b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tiers, warnings := b.BackupTiers()
|
||||
if len(warnings) != 0 {
|
||||
t.Fatalf("a legacy config must produce NO warnings; got %v", warnings)
|
||||
}
|
||||
if len(tiers) != 1 {
|
||||
t.Fatalf("a config with no backup_targets must resolve to exactly ONE tier; got %+v", tiers)
|
||||
}
|
||||
got := tiers[0]
|
||||
if got.TargetID != "local" || got.Cadence != 24*time.Hour || got.KeepLast != 3 || !got.Primary {
|
||||
t.Fatalf("legacy tier changed: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// An empty BackupConfig still resolves — to the felhom-pbs default target, 24h, keep-last 3.
|
||||
// (Unchanged pre-R-82 behaviour; pinned so the default target can't drift unnoticed.)
|
||||
func TestBackupTiers_ZeroConfigKeepsDefaults(t *testing.T) {
|
||||
tiers, warnings := BackupConfig{}.BackupTiers()
|
||||
if len(warnings) != 0 || len(tiers) != 1 {
|
||||
t.Fatalf("zero config: tiers=%+v warnings=%v", tiers, warnings)
|
||||
}
|
||||
if tiers[0].TargetID != defaultBackupTarget || tiers[0].Cadence != 24*time.Hour || tiers[0].KeepLast != 3 {
|
||||
t.Fatalf("zero-config defaults changed: %+v", tiers[0])
|
||||
}
|
||||
}
|
||||
|
||||
// The whole point: local daily + PBS weekly, each with its OWN retention.
|
||||
func TestBackupTiers_LocalDailyPlusPBSWeekly(t *testing.T) {
|
||||
var b BackupConfig
|
||||
raw := `{
|
||||
"local_backup_target":"local",
|
||||
"local_backup_retention":3,
|
||||
"backup_cadence_seconds":86400,
|
||||
"backup_targets":[{"target_id":"felhom-pbs","cadence_seconds":604800,"keep_last":2}]
|
||||
}`
|
||||
if err := json.Unmarshal([]byte(raw), &b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tiers, warnings := b.BackupTiers()
|
||||
if len(warnings) != 0 {
|
||||
t.Fatalf("unexpected warnings: %v", warnings)
|
||||
}
|
||||
if len(tiers) != 2 {
|
||||
t.Fatalf("want 2 tiers, got %+v", tiers)
|
||||
}
|
||||
if !tiers[0].Primary || tiers[0].TargetID != "local" || tiers[0].Cadence != 24*time.Hour || tiers[0].KeepLast != 3 {
|
||||
t.Fatalf("primary tier wrong: %+v", tiers[0])
|
||||
}
|
||||
if tiers[1].Primary || tiers[1].TargetID != "felhom-pbs" || tiers[1].Cadence != 7*24*time.Hour || tiers[1].KeepLast != 2 {
|
||||
t.Fatalf("PBS tier wrong: %+v", tiers[1])
|
||||
}
|
||||
// THE knob-sharing check: the two retentions are independent values, not one shared number.
|
||||
if tiers[0].KeepLast == tiers[1].KeepLast {
|
||||
t.Fatalf("this fixture sets 3 and 2 deliberately — equal values mean the knob is shared: %+v", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// A tier with no cadence is REJECTED, not defaulted. Defaulting would turn a weekly DR tier into a
|
||||
// daily one and fill the 37.2 GB datastore (R-82 Phase 0, P0.3).
|
||||
func TestBackupTiers_MissingCadenceIsRejectedLoudly(t *testing.T) {
|
||||
b := BackupConfig{
|
||||
LocalBackupTarget: "local",
|
||||
ExtraTargets: []BackupTargetConfig{{TargetID: "felhom-pbs", KeepLast: 2}},
|
||||
}
|
||||
tiers, warnings := b.BackupTiers()
|
||||
if len(tiers) != 1 {
|
||||
t.Fatalf("a cadence-less tier must NOT be armed; got %+v", tiers)
|
||||
}
|
||||
if len(warnings) != 1 || !strings.Contains(warnings[0], "cadence_seconds must be > 0") {
|
||||
t.Fatalf("rejection must be reported so the caller can log it loudly; got %v", warnings)
|
||||
}
|
||||
if !strings.Contains(warnings[0], "felhom-pbs") {
|
||||
t.Fatalf("the warning must name the tier it dropped; got %q", warnings[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupTiers_RejectsEmptyAndDuplicateTargets(t *testing.T) {
|
||||
b := BackupConfig{
|
||||
LocalBackupTarget: "local",
|
||||
ExtraTargets: []BackupTargetConfig{
|
||||
{TargetID: "", CadenceSeconds: 3600},
|
||||
{TargetID: "local", CadenceSeconds: 3600}, // repeats the primary
|
||||
{TargetID: "felhom-pbs", CadenceSeconds: 604800}, // good
|
||||
{TargetID: "felhom-pbs", CadenceSeconds: 99}, // duplicate
|
||||
},
|
||||
}
|
||||
tiers, warnings := b.BackupTiers()
|
||||
if len(tiers) != 2 || tiers[1].TargetID != "felhom-pbs" || tiers[1].Cadence != 7*24*time.Hour {
|
||||
t.Fatalf("want primary + one PBS tier at the FIRST definition; got %+v", tiers)
|
||||
}
|
||||
if len(warnings) != 3 {
|
||||
t.Fatalf("want 3 rejections (empty, duplicate-of-primary, duplicate); got %v", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
// keep_last unset means DO NOT PRUNE. That is the fail-safe: a DR tier must never start pruning
|
||||
// itself because someone forgot a field.
|
||||
func TestBackupTiers_UnsetKeepLastMeansNoPrune(t *testing.T) {
|
||||
b := BackupConfig{
|
||||
LocalBackupTarget: "local",
|
||||
ExtraTargets: []BackupTargetConfig{{TargetID: "felhom-pbs", CadenceSeconds: 604800}},
|
||||
}
|
||||
tiers, _ := b.BackupTiers()
|
||||
if len(tiers) != 2 {
|
||||
t.Fatalf("got %+v", tiers)
|
||||
}
|
||||
if tiers[1].KeepLast != 0 {
|
||||
t.Fatalf("an unset keep_last must resolve to 0 = never prune; got %d", tiers[1].KeepLast)
|
||||
}
|
||||
// And a negative is clamped to the same fail-safe rather than becoming a prune spec.
|
||||
b.ExtraTargets[0].KeepLast = -5
|
||||
tiers, _ = b.BackupTiers()
|
||||
if tiers[1].KeepLast != 0 {
|
||||
t.Fatalf("a negative keep_last must clamp to 0 (never prune); got %d", tiers[1].KeepLast)
|
||||
}
|
||||
}
|
||||
|
||||
// The primary's retention still comes from the legacy knob with its legacy clamp — untouched.
|
||||
func TestBackupTiers_PrimaryRetentionClampUnchanged(t *testing.T) {
|
||||
for _, tc := range []struct{ in, want int }{{0, 3}, {-1, 3}, {1, 1}, {7, 7}} {
|
||||
b := BackupConfig{LocalBackupTarget: "local", LocalBackupRetention: tc.in}
|
||||
tiers, _ := b.BackupTiers()
|
||||
if tiers[0].KeepLast != tc.want {
|
||||
t.Fatalf("LocalBackupRetention=%d → KeepLast=%d, want %d", tc.in, tiers[0].KeepLast, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// R-82 live-failure regression (2026-07-26): the runner hard-coded a 30-minute vzdump wait, which
|
||||
// is right for a local vzdump and wrong for an offsite PBS upload. The first full ~10 GB PBS
|
||||
// snapshot on demo-felhom ran past 30 min; the agent gave up waiting and recorded success=false
|
||||
// WHILE THE BACKUP WAS STILL RUNNING — a false failure that leaves the tier permanently "due" and
|
||||
// makes the next attempt collide with the guest lock vzdump still holds.
|
||||
func TestBackupTiers_WaitTimeoutIsPerTier(t *testing.T) {
|
||||
b := BackupConfig{
|
||||
LocalBackupTarget: "local",
|
||||
ExtraTargets: []BackupTargetConfig{{TargetID: "felhom-pbs", CadenceSeconds: 604800}},
|
||||
}
|
||||
tiers, _ := b.BackupTiers()
|
||||
if len(tiers) != 2 {
|
||||
t.Fatalf("got %+v", tiers)
|
||||
}
|
||||
if tiers[0].WaitTimeout != 30*time.Minute {
|
||||
t.Fatalf("the PRIMARY must keep the historical 30m wait (unchanged behaviour); got %s", tiers[0].WaitTimeout)
|
||||
}
|
||||
if tiers[1].WaitTimeout != 12*time.Hour {
|
||||
t.Fatalf("an offsite tier must default to a GENEROUS wait (operator ruling: let the first backup run as long as needed) — a false timeout is worse than a slow pass; got %s", tiers[1].WaitTimeout)
|
||||
}
|
||||
// And it must be overridable per tier.
|
||||
b.ExtraTargets[0].WaitTimeoutSeconds = 3600
|
||||
tiers, _ = b.BackupTiers()
|
||||
if tiers[1].WaitTimeout != time.Hour {
|
||||
t.Fatalf("wait_timeout_seconds must override; got %s", tiers[1].WaitTimeout)
|
||||
}
|
||||
// The two tiers must NOT share one bound.
|
||||
if tiers[0].WaitTimeout == tiers[1].WaitTimeout {
|
||||
t.Fatalf("wait bounds are shared between tiers — the whole point is that they differ: %+v", tiers)
|
||||
}
|
||||
}
|
||||
+334
-13
@@ -32,8 +32,50 @@ type Config struct {
|
||||
LocalAPI LocalAPIConfig `json:"local_api"`
|
||||
LANResolver LANResolverConfig `json:"lan_resolver"`
|
||||
WGTunnel WGTunnelConfig `json:"wg_tunnel"`
|
||||
GuestNet GuestNetConfig `json:"guest_net"`
|
||||
OOB OOBConfig `json:"oob"`
|
||||
SelfUpdate SelfUpdateConfig `json:"selfupdate"`
|
||||
LogLevel string `json:"log_level"` // debug|info|warn|error (default info)
|
||||
|
||||
// DeploymentMode gates host-service self-heal (CAMPAIGN-3 Part 6). "appliance" = a Felhom-managed
|
||||
// node the agent may remediate (e.g. start networking at boot — F12-class defense in depth). Any
|
||||
// other value, including absent/unknown, is treated as "byo" (a customer's own host): the self-heal
|
||||
// CHECK still runs and WARNs, but the REMEDY is structurally unreachable. Fail-safe to byo — never
|
||||
// touch a host we do not own. Distinct from Privileged.Mode (sudo vs direct exec) — do NOT overload.
|
||||
DeploymentMode string `json:"deployment_mode,omitempty"`
|
||||
|
||||
// SourcePath is the file this config was loaded from ("" = all-env). Set by Load, never
|
||||
// serialized — the pbsdr bridge's escrow.pbs_storage_id seed writes back to it.
|
||||
SourcePath string `json:"-"`
|
||||
}
|
||||
|
||||
// DeploymentModeAppliance is the ONLY value that unlocks host-service self-heal. Everything else,
|
||||
// including "" and any typo, is byo (fail-safe — a host we do not own is never remediated).
|
||||
const DeploymentModeAppliance = "appliance"
|
||||
|
||||
// IsAppliance reports whether this node is a Felhom-managed appliance (self-heal remedies allowed).
|
||||
// Fail-safe: absent/unknown → false (byo).
|
||||
func (c *Config) IsAppliance() bool { return c.DeploymentMode == DeploymentModeAppliance }
|
||||
|
||||
// OOBConfig configures the dedicated felhom-sshd OOB access instance + belt (TASK H1). **Enabled
|
||||
// DEFAULTS TO FALSE** — a rollout to a box without explicit oob.enabled=true is a no-op (no port
|
||||
// claim, no config render, no belt mutation, no oob report stanza) until the operator endpoint +
|
||||
// static belt table exist.
|
||||
type OOBConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
IntervalSeconds int `json:"interval_seconds"` // reconcile cadence; default 60
|
||||
StateDir string `json:"state_dir"` // staged config/authkeys under <StateDir>/felhom-sshd/; default /var/lib/felhom-agent
|
||||
}
|
||||
|
||||
// WithDefaults fills the OOB reconcile cadence + state dir.
|
||||
func (o OOBConfig) WithDefaults() OOBConfig {
|
||||
if o.IntervalSeconds == 0 {
|
||||
o.IntervalSeconds = 60
|
||||
}
|
||||
if o.StateDir == "" {
|
||||
o.StateDir = "/var/lib/felhom-agent"
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
// SelfUpdateConfig configures the operator-signed agent self-update (TASK D1). The artifact HOST
|
||||
@@ -97,6 +139,46 @@ func (w WGTunnelConfig) WithDefaults() WGTunnelConfig {
|
||||
return w
|
||||
}
|
||||
|
||||
// GuestNetConfig configures the R-54 guest-network watchdog (internal/guestnet).
|
||||
//
|
||||
// **This is the repo's first DEFAULT-ON feature gate, and the inversion is deliberate.** Every other
|
||||
// gate here is `Enabled bool` defaulting to false, because those features reach outward (an offsite
|
||||
// endpoint, an OOB tunnel) and enrolling a box into one by an update would be wrong. This one only
|
||||
// looks INWARD at guests the agent already owns, and the failure it prevents — an unsupervised DHCP
|
||||
// client dying and taking the box off the internet 1-2 hours later, invisibly
|
||||
// (INCIDENT-guest-dhclient-killed-2026-07-20) — is one every box has today. A watchdog that must be
|
||||
// remembered per box is a watchdog that is missing on the box that needed it. Opting out is
|
||||
// therefore the explicit act: `"guest_net": {"disable": true}`.
|
||||
type GuestNetConfig struct {
|
||||
Disable bool `json:"disable"` // explicit opt-OUT; default is enabled
|
||||
IntervalSeconds int `json:"interval_seconds"` // probe cadence; default 60
|
||||
MinHealIntervalSeconds int `json:"min_heal_interval_seconds"` // per-guest cool-off; default 600
|
||||
MaxHealsPerHour int `json:"max_heals_per_hour"` // per-guest hourly cap; default 3
|
||||
SettleSeconds int `json:"settle_seconds"` // boot-race guard (guest AND agent uptime); default 180
|
||||
}
|
||||
|
||||
// Enabled reports whether the guest-network watchdog should run.
|
||||
func (g GuestNetConfig) Enabled() bool { return !g.Disable }
|
||||
|
||||
// WithDefaults fills the cadence and the three dampers. A NEGATIVE value is honoured as-is by the
|
||||
// watchdog constructor's own guards, so an operator can set 0 to mean "package default" without
|
||||
// having to know the number.
|
||||
func (g GuestNetConfig) WithDefaults() GuestNetConfig {
|
||||
if g.IntervalSeconds == 0 {
|
||||
g.IntervalSeconds = 60
|
||||
}
|
||||
if g.MinHealIntervalSeconds == 0 {
|
||||
g.MinHealIntervalSeconds = 600
|
||||
}
|
||||
if g.MaxHealsPerHour == 0 {
|
||||
g.MaxHealsPerHour = 3
|
||||
}
|
||||
if g.SettleSeconds == 0 {
|
||||
g.SettleSeconds = 180
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// LANResolverConfig configures the host-level split-horizon DNS resolver (internal/lanresolver): a
|
||||
// dnsmasq the agent manages so LAN clients reach their guest DIRECTLY at the same hostname + real cert.
|
||||
// Disabled unless Enable is set. HostIP defaults to the local-API bridge IP (the host LAN anchor);
|
||||
@@ -153,6 +235,16 @@ type LocalAPIConfig struct {
|
||||
// TokenStore is the durable, hashed token→guest map (only a HASH of each token is
|
||||
// persisted; the plaintext exists transiently at mint→write-to-mount, then is discarded).
|
||||
TokenStore string `json:"token_store"` // default /var/lib/felhom-agent/local-tokens.log
|
||||
// IslandBridge + IslandGuestAddr configure the R-50 host-internal control-plane bridge. When
|
||||
// BOTH are set, the provisioner attaches each guest a static net1 on IslandBridge with
|
||||
// IslandGuestAddr, so the controller reaches the agent over a fixed private address that no
|
||||
// LAN/DHCP/site move can invalidate (the F1 fix — AUDIT-vacation-remote-ops-2026-07-20). Empty
|
||||
// (the default) = LAN-only, byte-for-byte the pre-R-50 behaviour. On an island install ListenAddr
|
||||
// is the host side (169.254.253.1:8443); IslandGuestAddr is the guest side (169.254.253.2/30 — a
|
||||
// /30 is exactly host + one guest). Additive-only: it never removes a NIC, so a guest restored on
|
||||
// a non-island host (both empty) is unaffected.
|
||||
IslandBridge string `json:"island_bridge"` // e.g. "vmbr9" (portless host-internal bridge)
|
||||
IslandGuestAddr string `json:"island_guest_addr"` // guest net1 CIDR, e.g. "169.254.253.2/30"
|
||||
}
|
||||
|
||||
// Default local-API file locations (under the agent's state dir).
|
||||
@@ -167,6 +259,12 @@ func (l LocalAPIConfig) Enabled() bool {
|
||||
return l.Enable && strings.TrimSpace(l.ListenAddr) != ""
|
||||
}
|
||||
|
||||
// IslandEnabled reports whether the provisioner should attach a guest island NIC (net1). True only
|
||||
// when BOTH the bridge and the guest CIDR are set (R-50); empty = pre-R-50 LAN-only behaviour.
|
||||
func (l LocalAPIConfig) IslandEnabled() bool {
|
||||
return strings.TrimSpace(l.IslandBridge) != "" && strings.TrimSpace(l.IslandGuestAddr) != ""
|
||||
}
|
||||
|
||||
// TokenStorePath returns the configured token-store path (default applied).
|
||||
func (l LocalAPIConfig) TokenStorePath() string {
|
||||
if l.TokenStore != "" {
|
||||
@@ -201,6 +299,17 @@ func (l LocalAPIConfig) Validate() error {
|
||||
if _, _, err := net.SplitHostPort(l.ListenAddr); err != nil {
|
||||
return fmt.Errorf("config: local_api.listen_addr %q is not host:port: %w", l.ListenAddr, err)
|
||||
}
|
||||
// R-50: island fields are all-or-nothing, and the guest addr must be a CIDR (the net1 ip= value).
|
||||
// A half-set island (bridge without guest addr, or vice versa) is a provisioning mistake, not a
|
||||
// silent LAN fallback — fail loudly so a botched install config is caught at load, not at day-0.
|
||||
if (strings.TrimSpace(l.IslandBridge) != "") != (strings.TrimSpace(l.IslandGuestAddr) != "") {
|
||||
return fmt.Errorf("config: local_api.island_bridge and local_api.island_guest_addr must be set together (got bridge=%q guest_addr=%q)", l.IslandBridge, l.IslandGuestAddr)
|
||||
}
|
||||
if l.IslandEnabled() {
|
||||
if _, _, err := net.ParseCIDR(strings.TrimSpace(l.IslandGuestAddr)); err != nil {
|
||||
return fmt.Errorf("config: local_api.island_guest_addr %q is not a CIDR (want e.g. 169.254.253.2/30): %w", l.IslandGuestAddr, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -224,9 +333,23 @@ type BackupConfig struct {
|
||||
LocalBackupTarget string `json:"local_backup_target"`
|
||||
// RestoreStorage is where a restore-test's restored rootfs lands, e.g. "local-lvm".
|
||||
RestoreStorage string `json:"restore_storage"`
|
||||
// RestoreTestCadenceSeconds is the self-restore-test interval; 0 → default (24h).
|
||||
// Set negative to DISABLE the automatic cadence (on-demand selftest still works).
|
||||
// RestoreTestCadenceSeconds is the LEGACY restore-test knob, retained for one meaning only:
|
||||
// NEGATIVE still DISABLES the automatic restore-test entirely (on-demand selftest still works),
|
||||
// and 0 still means "use the default". It no longer sets how often a test runs — R-86 replaced
|
||||
// the interval trigger with a per-archive due-check — so a positive value now seeds
|
||||
// RestoreTestSettleSeconds instead (see RestoreTestSettle). Prefer the two explicit keys below.
|
||||
RestoreTestCadenceSeconds int `json:"restore_test_cadence_seconds"`
|
||||
// RestoreTestEvalIntervalSeconds is how often the scheduler ASKS whether any tier is due
|
||||
// (R-86); 0 → default. It is not how often a test runs: a tier is tested once per archive
|
||||
// generation no matter how often it is asked. This interval sets two things — the latency
|
||||
// between an archive settling and its proof, and the retry rate of a tier whose restore-test
|
||||
// keeps failing. See defaultRestoreTestEvalInterval for the measurement it was chosen from.
|
||||
RestoreTestEvalIntervalSeconds int `json:"restore_test_eval_interval_seconds"`
|
||||
// RestoreTestSettleSeconds is how long an archive must have sat on its tier before it is a
|
||||
// restore-test candidate (R-86); 0 → default (24h), negative → 0 (no settle requirement).
|
||||
// Restore-testing an archive a backup is still writing proves nothing about the backup that
|
||||
// finished — this is the same settle discipline R-71a's gate applies to the offsite consume.
|
||||
RestoreTestSettleSeconds int `json:"restore_test_settle_seconds"`
|
||||
// ScratchVMIDMin/Max bound the throwaway restore-test scratch-guest VMID band. The
|
||||
// restore-test refuses to run unless this is a valid band (min>0, max>=min); 9999 is
|
||||
// always excluded. Defaults to 990000–990009.
|
||||
@@ -257,6 +380,132 @@ type BackupConfig struct {
|
||||
// default 3; ALWAYS clamped to ≥1 by KeepLast() so a mis-config can never prune the fresh backup.
|
||||
// NEVER applied to a PBS target (offsite retention is a separate lifecycle).
|
||||
LocalBackupRetention int `json:"local_backup_retention"`
|
||||
|
||||
// ExtraTargets (R-82) are ADDITIONAL backup tiers beyond the primary one above — the shape that
|
||||
// makes "local daily + PBS weekly" expressible at all. Each carries its OWN cadence and its OWN
|
||||
// retention, because those are semantically different per tier: keep-last=3 on a daily tier is
|
||||
// three DAYS of restore points; on a weekly tier it is three WEEKS. Sharing one knob between
|
||||
// tiers silently means one of them is wrong.
|
||||
//
|
||||
// ADDITIVE BY CONSTRUCTION: an existing config with no `backup_targets` key resolves to exactly
|
||||
// one tier — the primary — and behaves byte-identically to pre-R-82. Nothing here changes the
|
||||
// local tier.
|
||||
ExtraTargets []BackupTargetConfig `json:"backup_targets"`
|
||||
}
|
||||
|
||||
// BackupTargetConfig is ONE additional backup tier: a vzdump storage plus its own cadence and
|
||||
// retention. A tier with no cadence is not a tier — see BackupTiers for why that is rejected loudly
|
||||
// rather than defaulted.
|
||||
type BackupTargetConfig struct {
|
||||
// TargetID is the Proxmox storage id (content=backup), e.g. "felhom-pbs".
|
||||
TargetID string `json:"target_id"`
|
||||
// CadenceSeconds is THIS tier's /backup/due window. REQUIRED (>0) — see BackupTiers.
|
||||
CadenceSeconds int `json:"cadence_seconds"`
|
||||
// KeepLast is THIS tier's per-run `--prune-backups` keep-last. 0/unset → NEVER prune this tier
|
||||
// (the fail-safe default, and the current behaviour for every PBS target). A PBS tier is never
|
||||
// pruned by the per-run flag regardless — see BackupRunner.localPruneSpec.
|
||||
KeepLast int `json:"keep_last"`
|
||||
// WaitTimeoutSeconds bounds how long the agent WAITS for this tier's vzdump task. 0/unset →
|
||||
// defaultExtraTierWaitTimeout.
|
||||
//
|
||||
// THIS FIELD EXISTS BECAUSE OF A LIVE FAILURE (2026-07-26, R-82 Slice A validation). The runner
|
||||
// hard-coded a 30-minute wait, which is right for a local vzdump (minutes) and badly wrong for
|
||||
// an offsite PBS backup over a home uplink: the first full ~10 GB snapshot ran past 30 min, the
|
||||
// agent gave up waiting and recorded success=false — WHILE THE BACKUP WAS STILL RUNNING. That
|
||||
// false failure is worse than a slow pass: the tier stays "due", a retry collides with the
|
||||
// guest lock vzdump still holds, and the hub sees a DR tier that never succeeds.
|
||||
//
|
||||
// Same reasoning as RestoreTestPBSRestoreTimeoutSeconds on the restore side, and the same
|
||||
// direction: when in doubt wait LONGER. A slow backup is a slow backup; a false timeout is a
|
||||
// corrupt status plus lock contention.
|
||||
WaitTimeoutSeconds int `json:"wait_timeout_seconds"`
|
||||
}
|
||||
|
||||
// Per-tier vzdump wait bounds.
|
||||
//
|
||||
// The PRIMARY keeps the historical 30 minutes: it is the local tier, a local vzdump takes minutes,
|
||||
// and one hanging 30 minutes is a genuine fault worth surfacing. Unchanged behaviour.
|
||||
//
|
||||
// An ADDITIONAL tier is by construction the offsite/WAN one in this design, where the binding
|
||||
// constraint is uplink speed, not health. Measured on demo-felhom: ~33 MB/min over the wg link to
|
||||
// Hetzner, so a first FULL ~10 GB snapshot projects to ~5h. Operator ruling 2026-07-26: "let the
|
||||
// first backup run as long as needed" — 12h gives that real margin on a slower link while still
|
||||
// being BOUNDED, so a genuinely hung task eventually surfaces instead of hanging forever.
|
||||
const (
|
||||
defaultPrimaryTierWaitTimeout = 30 * time.Minute
|
||||
defaultExtraTierWaitTimeout = 12 * time.Hour
|
||||
)
|
||||
|
||||
// BackupTier is a RESOLVED backup tier: one target, its own cadence, its own retention. The agent
|
||||
// builds one runner per tier from these.
|
||||
type BackupTier struct {
|
||||
TargetID string
|
||||
Cadence time.Duration
|
||||
// WaitTimeout bounds the wait on this tier's vzdump task (see WaitTimeoutSeconds).
|
||||
WaitTimeout time.Duration
|
||||
// KeepLast is the per-run prune keep-last; 0 means DO NOT PRUNE this tier.
|
||||
KeepLast int
|
||||
// Primary marks the tier that the UNTARGETED local-API endpoints act on — the pre-R-82 tier.
|
||||
// Exactly one tier is primary, and it is always first.
|
||||
Primary bool
|
||||
}
|
||||
|
||||
// BackupTiers resolves the effective tier list, primary first, plus any warnings the caller MUST
|
||||
// log (they describe tiers that were REJECTED, and a silently-dropped backup tier is precisely the
|
||||
// "applied and empty" fault R-82 exists to fix).
|
||||
//
|
||||
// Rules:
|
||||
// - Tier 0 is always the primary, built from BackupTarget()/BackupCadence()/KeepLast() — so a
|
||||
// config with no `backup_targets` is byte-identical to pre-R-82.
|
||||
// - An extra with an empty target_id is rejected.
|
||||
// - An extra with cadence_seconds <= 0 is REJECTED, not defaulted. Defaulting a PBS tier to the
|
||||
// 24h local default would quietly turn a weekly tier into a daily one and fill the DR datastore;
|
||||
// a tier whose cadence you did not state is not a tier.
|
||||
// - An extra repeating the primary's target is rejected (one policy per target, or the two
|
||||
// cadences race and neither is the truth).
|
||||
// - Duplicate extras are rejected after the first.
|
||||
func (b BackupConfig) BackupTiers() ([]BackupTier, []string) {
|
||||
primary := BackupTier{
|
||||
TargetID: b.BackupTarget(),
|
||||
Cadence: b.BackupCadence(),
|
||||
KeepLast: b.KeepLast(),
|
||||
WaitTimeout: defaultPrimaryTierWaitTimeout,
|
||||
Primary: true,
|
||||
}
|
||||
tiers := []BackupTier{primary}
|
||||
var warnings []string
|
||||
|
||||
seen := map[string]bool{primary.TargetID: true}
|
||||
for i, t := range b.ExtraTargets {
|
||||
id := strings.TrimSpace(t.TargetID)
|
||||
switch {
|
||||
case id == "":
|
||||
warnings = append(warnings, fmt.Sprintf("backup_targets[%d]: empty target_id — tier ignored", i))
|
||||
continue
|
||||
case seen[id]:
|
||||
warnings = append(warnings, fmt.Sprintf("backup_targets[%d]: target %q already configured — duplicate tier ignored", i, id))
|
||||
continue
|
||||
case t.CadenceSeconds <= 0:
|
||||
warnings = append(warnings, fmt.Sprintf("backup_targets[%d] (%s): cadence_seconds must be > 0 — tier ignored (a cadence is NOT defaulted: a weekly tier silently running daily would fill the DR datastore)", i, id))
|
||||
continue
|
||||
}
|
||||
seen[id] = true
|
||||
keep := t.KeepLast
|
||||
if keep < 0 {
|
||||
keep = 0
|
||||
}
|
||||
wait := defaultExtraTierWaitTimeout
|
||||
if t.WaitTimeoutSeconds > 0 {
|
||||
wait = time.Duration(t.WaitTimeoutSeconds) * time.Second
|
||||
}
|
||||
tiers = append(tiers, BackupTier{
|
||||
TargetID: id,
|
||||
Cadence: time.Duration(t.CadenceSeconds) * time.Second,
|
||||
KeepLast: keep,
|
||||
WaitTimeout: wait,
|
||||
})
|
||||
}
|
||||
return tiers, warnings
|
||||
}
|
||||
|
||||
// defaultLocalBackupKeepLast is the local vzdump retention default (newest N restore points kept).
|
||||
@@ -310,26 +559,92 @@ func (b BackupConfig) BackupTarget() string {
|
||||
return defaultBackupTarget
|
||||
}
|
||||
|
||||
// Default scratch VMID band + restore-test cadence.
|
||||
// Default scratch VMID band + the two R-86 restore-test knobs.
|
||||
const (
|
||||
defaultScratchVMIDMin = 990000
|
||||
defaultScratchVMIDMax = 990009
|
||||
defaultRestoreTestCadence = 24 * time.Hour
|
||||
defaultScratchVMIDMin = 990000
|
||||
defaultScratchVMIDMax = 990009
|
||||
|
||||
// defaultRestoreTestEvalInterval is how often due-ness is ASKED. It is bounded from BOTH sides,
|
||||
// and neither bound alone would have picked it:
|
||||
//
|
||||
// FLOOR — what one evaluation costs. MEASURED on demo-felhom, 2026-08-03 (R-86 Part 1.4), via
|
||||
// --selftest=restore-test-due and by timing the underlying API call directly. One evaluation
|
||||
// is one storage-content listing per tier:
|
||||
//
|
||||
// local dir storage (3 archives) ....... 18 ms (18.7 / 18.3 / 18.5)
|
||||
// PBS tier, WAN to ep0 (2 snapshots) ... 392 ms (375 / 378 / 424)
|
||||
// both tiers together .................. 430 ms
|
||||
//
|
||||
// So cost does NOT set this: even at one evaluation a minute the offsite leg would be ~0.7 %
|
||||
// of a WAN link's time and ~9 minutes of ep0's day. Worth writing down anyway, because the
|
||||
// number that would have forbidden a frequent poll is the one nobody measures.
|
||||
//
|
||||
// CEILING — the retry rate of a FAILING tier. Under a per-archive due-check a tier whose
|
||||
// restore-test keeps failing stays due, so the evaluation interval IS its retry interval, and
|
||||
// a retry is a multi-GB restore. Every few minutes would be an incident of its own; the old
|
||||
// timer retried a broken tier once a day.
|
||||
//
|
||||
// 6h sits between them: four heavy retries a day at the very worst, latency from settle to
|
||||
// proof of at most 6h against a 24h settle lag (so a daily tier is still proved daily), and no
|
||||
// second rate limiter anywhere — the pacing remains one test per archive generation.
|
||||
defaultRestoreTestEvalInterval = 6 * time.Hour
|
||||
|
||||
// defaultRestoreTestSettle is how long an archive must sit before it may be restore-tested.
|
||||
// 24h is R-86's own figure ("~24 h after its own newest archive") and it is what makes the
|
||||
// candidate on a daily tier YESTERDAY's archive rather than the one still being written.
|
||||
defaultRestoreTestSettle = 24 * time.Hour
|
||||
)
|
||||
|
||||
// RestoreTestCadence returns the configured restore-test interval: a positive value as-is,
|
||||
// 0 → 24h default, negative → 0 (disabled).
|
||||
func (b BackupConfig) RestoreTestCadence() time.Duration {
|
||||
// RestoreTestEvalInterval returns how often the scheduler evaluates due-ness (R-86): a positive
|
||||
// value as-is, 0 → the measured default, negative → 0 (disabled).
|
||||
//
|
||||
// The LEGACY `restore_test_cadence_seconds` keeps exactly one power here, the one a box may be
|
||||
// relying on: a NEGATIVE value still disables the automatic restore-test outright. It no longer
|
||||
// sets the interval, because the interval no longer decides that a test happens.
|
||||
func (b BackupConfig) RestoreTestEvalInterval() time.Duration {
|
||||
if b.RestoreTestCadenceSeconds < 0 {
|
||||
return 0 // legacy DISABLE — preserved verbatim
|
||||
}
|
||||
switch {
|
||||
case b.RestoreTestCadenceSeconds > 0:
|
||||
return time.Duration(b.RestoreTestCadenceSeconds) * time.Second
|
||||
case b.RestoreTestCadenceSeconds < 0:
|
||||
case b.RestoreTestEvalIntervalSeconds > 0:
|
||||
return time.Duration(b.RestoreTestEvalIntervalSeconds) * time.Second
|
||||
case b.RestoreTestEvalIntervalSeconds < 0:
|
||||
return 0 // disabled
|
||||
default:
|
||||
return defaultRestoreTestCadence
|
||||
return defaultRestoreTestEvalInterval
|
||||
}
|
||||
}
|
||||
|
||||
// RestoreTestSettle returns how long an archive must have sat before it is a restore-test
|
||||
// candidate (R-86): a positive value as-is, negative → 0 (no settle requirement), 0 → the default.
|
||||
//
|
||||
// WHAT HAPPENED TO THE OLD KEY. A box that set `restore_test_cadence_seconds` to a positive value
|
||||
// was expressing "how long may pass between a backup and the confidence that it restores". That
|
||||
// quantity survives R-86 as the SETTLE LAG, so a positive legacy value seeds this rather than being
|
||||
// dropped or silently repurposed as the evaluation interval — and the daemon says so at start-up
|
||||
// (see RestoreTestLegacyCadenceInUse). It is deliberately not carried into the evaluation interval:
|
||||
// a box that set 72h to spare a weak endpoint would otherwise get a 72h-latency due-check, whereas
|
||||
// what it actually wanted — fewer heavy restores — is what per-archive due-ness already gives it.
|
||||
func (b BackupConfig) RestoreTestSettle() time.Duration {
|
||||
switch {
|
||||
case b.RestoreTestSettleSeconds > 0:
|
||||
return time.Duration(b.RestoreTestSettleSeconds) * time.Second
|
||||
case b.RestoreTestSettleSeconds < 0:
|
||||
return 0 // explicitly no settle requirement
|
||||
case b.RestoreTestCadenceSeconds > 0:
|
||||
return time.Duration(b.RestoreTestCadenceSeconds) * time.Second // legacy seeding
|
||||
default:
|
||||
return defaultRestoreTestSettle
|
||||
}
|
||||
}
|
||||
|
||||
// RestoreTestLegacyCadenceInUse reports whether the deprecated key is what is deciding the settle
|
||||
// lag, so the daemon can name both replacements ONCE at start-up. A config key that changed meaning
|
||||
// without saying so is exactly the silent repurposing §8.3 forbids.
|
||||
func (b BackupConfig) RestoreTestLegacyCadenceInUse() bool {
|
||||
return b.RestoreTestCadenceSeconds > 0 && b.RestoreTestSettleSeconds == 0
|
||||
}
|
||||
|
||||
// PBSVerifyCadence returns the verify-loop interval: positive as-is, 0 → 6h default,
|
||||
// negative → 0 (disabled).
|
||||
func (b BackupConfig) PBSVerifyCadence() time.Duration {
|
||||
@@ -518,6 +833,7 @@ func Load(path string) (Config, error) {
|
||||
return cfg, fmt.Errorf("config: parsing %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
cfg.SourcePath = path // where this config came from (pbsdr's escrow seed writes back here)
|
||||
applyEnv(&cfg)
|
||||
return cfg, nil
|
||||
}
|
||||
@@ -548,6 +864,9 @@ func applyEnv(cfg *Config) {
|
||||
if v := os.Getenv("FELHOM_AGENT_LOG_LEVEL"); v != "" {
|
||||
cfg.LogLevel = v
|
||||
}
|
||||
if v := os.Getenv("FELHOM_AGENT_DEPLOYMENT_MODE"); v != "" {
|
||||
cfg.DeploymentMode = v
|
||||
}
|
||||
// hub
|
||||
if v := os.Getenv("FELHOM_AGENT_HUB_URL"); v != "" {
|
||||
cfg.Hub.URL = v
|
||||
@@ -571,6 +890,8 @@ func applyEnv(cfg *Config) {
|
||||
cfg.Backup.RestoreStorage = v
|
||||
}
|
||||
cfg.Backup.RestoreTestCadenceSeconds = envInt("FELHOM_AGENT_BACKUP_RESTORE_TEST_CADENCE_SECONDS", cfg.Backup.RestoreTestCadenceSeconds)
|
||||
cfg.Backup.RestoreTestEvalIntervalSeconds = envInt("FELHOM_AGENT_BACKUP_RESTORE_TEST_EVAL_INTERVAL_SECONDS", cfg.Backup.RestoreTestEvalIntervalSeconds)
|
||||
cfg.Backup.RestoreTestSettleSeconds = envInt("FELHOM_AGENT_BACKUP_RESTORE_TEST_SETTLE_SECONDS", cfg.Backup.RestoreTestSettleSeconds)
|
||||
}
|
||||
|
||||
// envInt overlays an int env var, keeping cur (with a stderr warning) on parse
|
||||
|
||||
@@ -133,3 +133,84 @@ func TestLoadFileThenEnvOverride(t *testing.T) {
|
||||
t.Errorf("default endpoint lost: %q", cfg.Proxmox.Endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
// CAMPAIGN-3 Part 6: deployment_mode gates node self-heal, and it is FAIL-SAFE to byo — absent or any
|
||||
// unknown value is byo, ONLY the exact "appliance" unlocks the remedy.
|
||||
func TestIsAppliance_FailSafeToByo(t *testing.T) {
|
||||
cases := []struct {
|
||||
mode string
|
||||
want bool
|
||||
}{
|
||||
{"appliance", true},
|
||||
{"byo", false},
|
||||
{"", false}, // absent field → byo (fail-safe)
|
||||
{"Appliance", false}, // case-sensitive — a typo must not unlock the remedy
|
||||
{"garbage", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
cfg := &Config{DeploymentMode: c.mode}
|
||||
if got := cfg.IsAppliance(); got != c.want {
|
||||
t.Errorf("IsAppliance(mode=%q) = %t, want %t", c.mode, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The env overlay can set deployment_mode (FELHOM_AGENT_DEPLOYMENT_MODE).
|
||||
func TestDeploymentModeEnvOverlay(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "agent.json")
|
||||
if err := os.WriteFile(path, []byte(`{"proxmox":{"node":"n","token":"u@pve!t=s"},"deployment_mode":"byo"}`), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("FELHOM_AGENT_DEPLOYMENT_MODE", "appliance")
|
||||
cfg, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if !cfg.IsAppliance() {
|
||||
t.Errorf("env overlay did not set deployment_mode: %q", cfg.DeploymentMode)
|
||||
}
|
||||
}
|
||||
|
||||
// R-50: the island NIC fields are all-or-nothing and the guest addr must be a CIDR. A half-set or
|
||||
// malformed island must fail at config load (a botched install) rather than silently fall back to
|
||||
// LAN-only, which would leave a guest with an island bind and no island NIC — the exact silent break
|
||||
// R-50 exists to kill. Covers LocalAPIConfig.Validate + IslandEnabled.
|
||||
func TestLocalAPIConfig_IslandValidation(t *testing.T) {
|
||||
base := LocalAPIConfig{Enable: true, ListenAddr: "169.254.253.1:8443"}
|
||||
|
||||
// both empty → fine (pre-R-50 default), IslandEnabled false
|
||||
if err := base.Validate(); err != nil {
|
||||
t.Errorf("no island config must validate: %v", err)
|
||||
}
|
||||
if base.IslandEnabled() {
|
||||
t.Errorf("IslandEnabled must be false when unset")
|
||||
}
|
||||
// both set, valid CIDR → fine, IslandEnabled true
|
||||
ok := base
|
||||
ok.IslandBridge, ok.IslandGuestAddr = "vmbr9", "169.254.253.2/30"
|
||||
if err := ok.Validate(); err != nil {
|
||||
t.Errorf("valid island config must validate: %v", err)
|
||||
}
|
||||
if !ok.IslandEnabled() {
|
||||
t.Errorf("IslandEnabled must be true when both set")
|
||||
}
|
||||
// bridge only → rejected (all-or-nothing)
|
||||
half := base
|
||||
half.IslandBridge = "vmbr9"
|
||||
if err := half.Validate(); err == nil {
|
||||
t.Errorf("half-set island (bridge only) must be rejected")
|
||||
}
|
||||
// guest addr only → rejected
|
||||
half2 := base
|
||||
half2.IslandGuestAddr = "169.254.253.2/30"
|
||||
if err := half2.Validate(); err == nil {
|
||||
t.Errorf("half-set island (guest addr only) must be rejected")
|
||||
}
|
||||
// both set but guest addr is not a CIDR → rejected
|
||||
bad := base
|
||||
bad.IslandBridge, bad.IslandGuestAddr = "vmbr9", "169.254.253.2" // missing /30
|
||||
if err := bad.Validate(); err == nil {
|
||||
t.Errorf("island guest addr without a CIDR mask must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ package desired
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
@@ -66,12 +67,17 @@ func (s *Syncer) OnEnvelope(ctx context.Context, env *hub.ControlEnvelope) {
|
||||
if env.DesiredGeneration <= have {
|
||||
return // cached: the heavy desired-state moves only on a generation advance
|
||||
}
|
||||
s.logger.Debug("desired: generation advanced — fetching desired-state",
|
||||
"have_generation", have, "envelope_generation", env.DesiredGeneration)
|
||||
start := time.Now()
|
||||
resp, err := s.fetcher.FetchDesiredState(ctx)
|
||||
if err != nil {
|
||||
s.logger.Warn("desired: fetch failed; keeping cached desired-state",
|
||||
"have_generation", have, "envelope_generation", env.DesiredGeneration, "err", err)
|
||||
return
|
||||
}
|
||||
s.logger.Debug("desired: fetched", "generation", resp.Generation,
|
||||
"duration_ms", time.Since(start).Milliseconds())
|
||||
state := mapWire(resp.DesiredState, s.logger)
|
||||
// Cache against the FETCHED generation (not the envelope's) — robust to a generation that
|
||||
// advanced again between the heartbeat and this fetch (we won't re-fetch the same state).
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package escrow
|
||||
|
||||
// Controller-driven ceremony contract (v0.88.0, TASK 2026-07-13; mechanics validated by
|
||||
// felhom.eu/documentation/audits/SPIKE-controller-escrow-2026-07-13.md). The agent's local API
|
||||
// re-invokes the agent binary as root via `sudo -n` with ONE fixed argument vector; sudoers
|
||||
// matches that vector byte-for-byte (spike §2.2: any alteration — value, extra flag, order,
|
||||
// config path — is refused), so the argv below is the SINGLE SOURCE OF TRUTH shared by the
|
||||
// exec (localapi), the capability manifest entry, and (byte-identically) the FELHOM_ESCROW
|
||||
// sudoers line. Never build it with flag helpers and never normalize `--` to `-` — Go's flag
|
||||
// package would accept either spelling, sudoers only the literal one.
|
||||
|
||||
// CeremonyBinary is the installed agent binary path the sudoers line pins.
|
||||
const CeremonyBinary = "/usr/local/bin/felhom-agent"
|
||||
|
||||
// ceremonyArgv is the fixed vector. --config is pinned explicitly: `sudo -n` env_reset strips
|
||||
// FELHOM_AGENT_CONFIG, and the pin closes env-injection of an alternate config (spike probe (e)).
|
||||
var ceremonyArgv = []string{
|
||||
"--config", "/etc/felhom-agent/agent.json",
|
||||
"--selftest=escrow-create",
|
||||
"--upload",
|
||||
"--output=json",
|
||||
}
|
||||
|
||||
// CeremonyArgs returns a fresh copy of the fixed argv (callers must not be able to mutate the
|
||||
// shared source).
|
||||
func CeremonyArgs() []string {
|
||||
out := make([]string, len(ceremonyArgv))
|
||||
copy(out, ceremonyArgv)
|
||||
return out
|
||||
}
|
||||
|
||||
// CeremonyOutput is the --output=json wire contract (version 1): the ONE JSON object json mode
|
||||
// emits on stdout — nothing else lands there; every human/info line goes to stderr. RecoveryCode
|
||||
// is the only secret field: the consumer must extract it, hand it to the one-shot claim holder,
|
||||
// and zero both the parsed struct and the raw stdout buffer. (Best-effort — Go's GC may hold
|
||||
// stale copies; the discipline still shrinks the exposure window.)
|
||||
type CeremonyOutput struct {
|
||||
Version int `json:"version"`
|
||||
RecoveryCode string `json:"recovery_code"`
|
||||
KeyFingerprint string `json:"key_fingerprint"`
|
||||
EntropyBits float64 `json:"entropy_bits"`
|
||||
BlobBytes int `json:"blob_bytes"`
|
||||
IdentityBlobBytes int `json:"identity_blob_bytes"`
|
||||
ResticPwSealed bool `json:"restic_pw_sealed"`
|
||||
Uploaded bool `json:"uploaded"`
|
||||
}
|
||||
|
||||
// CeremonyOutputVersion is the current CeremonyOutput.Version value.
|
||||
const CeremonyOutputVersion = 1
|
||||
@@ -11,8 +11,16 @@ import (
|
||||
)
|
||||
|
||||
func TestWordlistLoaded(t *testing.T) {
|
||||
if WordlistSize() != 7776 {
|
||||
t.Fatalf("EFF large wordlist should be 7776 words, got %d", WordlistSize())
|
||||
// The EFF large list is 7776 entries; joinSafe removes the 4 that contain RecoveryCodeSep
|
||||
// (drop-down, felt-tip, t-shirt, yo-yo), leaving 7772 as the effective draw space.
|
||||
if got := WordlistSize(); got != 7772 {
|
||||
t.Fatalf("effective wordlist should be 7772 words (7776 EFF - 4 hyphenated), got %d", got)
|
||||
}
|
||||
if got := WordlistFilteredOut(); got != 4 {
|
||||
t.Fatalf("joinSafe should have removed exactly 4 hyphenated entries, removed %d", got)
|
||||
}
|
||||
if got := WordlistSize() + WordlistFilteredOut(); got != 7776 {
|
||||
t.Fatalf("filtered + removed should reconstitute the 7776-word EFF list, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,19 +33,27 @@ func TestGenerateRecoveryCode_EntropyAndFormat(t *testing.T) {
|
||||
inList[w] = true
|
||||
}
|
||||
for i := 0; i < 50; i++ {
|
||||
r, err := GenerateRecoveryCode()
|
||||
// Count words by GENERATION count, not by re-splitting the joined string: the two agree
|
||||
// only because joinSafe holds, and conflating them is what made this test flake ~1/5.
|
||||
words, err := generateWords(wordlist)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateRecoveryCode: %v", err)
|
||||
t.Fatalf("generateWords: %v", err)
|
||||
}
|
||||
words := strings.Split(r, "-")
|
||||
if len(words) != RecoveryCodeWords {
|
||||
t.Fatalf("recovery code must be %d words, got %d (%q)", RecoveryCodeWords, len(words), r)
|
||||
t.Fatalf("generator must draw %d words, drew %d", RecoveryCodeWords, len(words))
|
||||
}
|
||||
for _, w := range words {
|
||||
if !inList[w] {
|
||||
t.Errorf("recovery-code word %q is not from the EFF wordlist", w)
|
||||
}
|
||||
}
|
||||
// Separately assert the property joinSafe buys: the joined code segments back to the same
|
||||
// count. Never print r — it is a live-shaped secret.
|
||||
r := strings.Join(words, RecoveryCodeSep)
|
||||
if got := len(strings.Split(r, RecoveryCodeSep)); got != RecoveryCodeWords {
|
||||
t.Fatalf("joined code must segment into %d words, got %d (a drawn word contained %q)",
|
||||
RecoveryCodeWords, got, RecoveryCodeSep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,9 @@ package escrow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -30,6 +32,58 @@ type IdentityBundle struct {
|
||||
// blobs created before S3 lack it and CANNOT be retro-fitted (R is never retained) — S5 DR
|
||||
// falls back to fresh-key re-registration, which keeps the box's /32 (hub S2 re-key-in-place).
|
||||
WGPrivateKey string `json:"wg_private_key,omitempty"`
|
||||
// ResticRepoPassword is the offsite restic repo password (fork-4). OPTIONAL: escrow blobs created
|
||||
// before fork-4 lack it and CANNOT be retro-fitted (R is never retained). It is the DATA key for the
|
||||
// offsite tier — irreplaceable (unlike the SFTP access key, which is regenerable at DR). The
|
||||
// controller's atomicity gate ensures no offsite ciphertext exists until this is escrowed.
|
||||
ResticRepoPassword string `json:"restic_repo_password,omitempty"`
|
||||
}
|
||||
|
||||
// StagedResticPasswordPath is the well-known 0600 file where the controller-pushed restic repo password
|
||||
// is transiently staged (by the local API) for the escrow-create ceremony to pick up, then wiped. A fixed
|
||||
// path so the local-API writer and the CLI ceremony reader agree without threading config through.
|
||||
func StagedResticPasswordPath() string {
|
||||
return filepath.Join("/var/lib/felhom-agent", "escrow-stage", "restic_repo_password")
|
||||
}
|
||||
|
||||
// WipeStagedResticPassword removes the staged restic password (called by the ceremony after a successful
|
||||
// escrow-create — the secret now lives only inside the R-wrapped blob). A missing file is a clean no-op.
|
||||
func WipeStagedResticPassword() error {
|
||||
if err := os.Remove(StagedResticPasswordPath()); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("escrow: wipe staged restic password: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HashResticPassword is the CANONICAL hasher for the offsite restic repo password (SLICE 3 hub-verified
|
||||
// escrow auto-confirm): sha256 hex of the TRIMMED password string — exactly the value AttachResticPassword
|
||||
// seals into the blob and the value the controller uses (both sides TrimSpace their file reads, so the
|
||||
// trimmed string is the drift-free convention; pinned by the SAME test vector in felhom-agent and
|
||||
// felhom-controller). The hash of a 256-bit random secret is non-reversible and non-brute-forceable —
|
||||
// safe to store on the hub and serve in report ACKs; the PASSWORD itself is never logged or served.
|
||||
func HashResticPassword(pw string) string {
|
||||
sum := sha256.Sum256([]byte(strings.TrimSpace(pw)))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// AttachResticPassword injects the offsite restic repo password from the staged 0600 file into the bundle
|
||||
// when it exists (fork-4 escrow-create auto-inject). Returns whether it attached. The VALUE is validated
|
||||
// (non-empty) but NEVER logged by callers — log the field NAME only (mirrors AttachWGKey). A missing file
|
||||
// is a clean no-attach (pre-fork-4 behavior, byte-compatible bundle).
|
||||
func AttachResticPassword(b *IdentityBundle, stagePath string) (bool, error) {
|
||||
raw, err := os.ReadFile(stagePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("escrow: reading staged restic password: %w", err)
|
||||
}
|
||||
pw := strings.TrimSpace(string(raw))
|
||||
if pw == "" {
|
||||
return false, fmt.Errorf("escrow: staged restic password file %s is empty", stagePath)
|
||||
}
|
||||
b.ResticRepoPassword = pw
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// AttachWGKey injects the offsite WG private key into the bundle when the key file exists (S3
|
||||
|
||||
@@ -3,7 +3,9 @@ package escrow
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
@@ -65,6 +67,76 @@ func TestIdentity_RoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// fork-4: the IdentityBundle carries the offsite restic repo password under R, byte-exact and encrypted;
|
||||
// a wrong R fails closed. (The spike proved a recovered value opens the real repo; this guards the field.)
|
||||
func TestIdentity_RoundTrip_CarriesResticPassword(t *testing.T) {
|
||||
ensureAge(t)
|
||||
ctx := context.Background()
|
||||
const R = "throwaway-correct-horse-battery-staple-fork4"
|
||||
const pw = "deadbeefcafef00d0123456789abcdef0123456789abcdef0123456789abcdef" // 64 hex, synthetic
|
||||
bundle := IdentityBundle{TunnelToken: "tt", PBSToken: "pt", ResticRepoPassword: pw}
|
||||
blob, err := WrapIdentityBundle(ctx, bundle, R)
|
||||
if err != nil {
|
||||
t.Fatalf("WrapIdentityBundle: %v", err)
|
||||
}
|
||||
if bytes.Contains(blob, []byte(pw)) {
|
||||
t.Fatal("the blob leaks the restic password plaintext — not encrypted")
|
||||
}
|
||||
got, err := UnwrapIdentityBundle(ctx, blob, R)
|
||||
if err != nil {
|
||||
t.Fatalf("UnwrapIdentityBundle: %v", err)
|
||||
}
|
||||
if got.ResticRepoPassword != pw {
|
||||
t.Fatalf("recovered restic password not byte-exact: got %q", got.ResticRepoPassword)
|
||||
}
|
||||
if got != bundle {
|
||||
t.Fatalf("recovered bundle = %+v, want %+v", got, bundle)
|
||||
}
|
||||
if _, err := UnwrapIdentityBundle(ctx, blob, R+"-WRONG"); err == nil {
|
||||
t.Fatal("a wrong recovery code must fail closed (no bundle, no restic password)")
|
||||
}
|
||||
}
|
||||
|
||||
// PINNED CROSS-REPO TEST VECTOR (SLICE 3): the same vector is asserted in felhom-controller — if either
|
||||
// side drifts (trailing newline, encoding, trim behavior), its half of this test fails and auto-confirm
|
||||
// can never silently mismatch. Convention: sha256 hex over the TRIMMED password string.
|
||||
func TestHashResticPassword_PinnedVector(t *testing.T) {
|
||||
const vector = "cafef00ddeadbeef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
const want = "dbfc02f987e1ac0c91911d5761267089b1144628745a3343e4d96194e43c08e4"
|
||||
if got := HashResticPassword(vector); got != want {
|
||||
t.Fatalf("pinned vector drift: got %s want %s", got, want)
|
||||
}
|
||||
// trim convention: surrounding whitespace/newlines do not change the hash (both sides trim)
|
||||
if got := HashResticPassword(" " + vector + "\n"); got != want {
|
||||
t.Fatalf("whitespace must not change the hash (trim convention), got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// AttachResticPassword: missing file → clean no-attach; staged file → trimmed value attached; empty → error.
|
||||
func TestAttachResticPassword(t *testing.T) {
|
||||
b := &IdentityBundle{}
|
||||
if ok, err := AttachResticPassword(b, filepath.Join(t.TempDir(), "absent")); ok || err != nil {
|
||||
t.Fatalf("missing staged file must be a clean no-attach, got ok=%v err=%v", ok, err)
|
||||
}
|
||||
f := filepath.Join(t.TempDir(), "pw")
|
||||
if err := os.WriteFile(f, []byte(" abc123def \n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ok, err := AttachResticPassword(b, f)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("attach from staged file: ok=%v err=%v", ok, err)
|
||||
}
|
||||
if b.ResticRepoPassword != "abc123def" {
|
||||
t.Fatalf("want trimmed value, got %q", b.ResticRepoPassword)
|
||||
}
|
||||
if err := os.WriteFile(f, []byte(" \n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := AttachResticPassword(&IdentityBundle{}, f); err == nil {
|
||||
t.Fatal("an empty staged file must error (an operator would want to know)")
|
||||
}
|
||||
}
|
||||
|
||||
// Wrong R fails CLOSED — no bundle emitted.
|
||||
func TestIdentity_WrongRFailsClosed(t *testing.T) {
|
||||
ensureAge(t)
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package escrow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// R-199 links 6→8 — fetch this host's own sealed identity blob, open it with the customer's recovery
|
||||
// code R, and hand back EXACTLY ONE field: the offsite restic repository password.
|
||||
//
|
||||
// WHY ONLY ONE FIELD. The bundle also carries the Cloudflare tunnel token, the PBS access token and
|
||||
// the WG private key (see IdentityBundle). The caller in this flow — the in-guest controller, one
|
||||
// trust tier down — needs none of them, and returning them would widen the blast radius of a
|
||||
// controller compromise for no gain. Narrowing costs nothing here and is not recoverable later.
|
||||
//
|
||||
// WHY R NEVER TOUCHES DISK. `UnwrapIdentity` stages the BLOB and the recovered plaintext in a
|
||||
// `MkdirTemp` that it removes, and feeds R through the pty; R itself is never written. This wrapper
|
||||
// keeps that property: it takes R as an argument, passes it straight through, and holds no copy.
|
||||
// Callers must clear their own reference (the `R = ""` discipline in cmd/felhom-agent).
|
||||
//
|
||||
// The errors below are DISTINCT on purpose. "could not fetch", "no blob", "wrong code" and "the blob
|
||||
// predates the field" are FOUR different situations for the operator and only one of them is a fault.
|
||||
//
|
||||
// ⚠ THERE WERE THREE, AND THE FOURTH WAS THE DEFECT (R-224, 2026-08-06). This comment said "three"
|
||||
// and named "no blob", "wrong code" and "predates the field" — while a FAILED FETCH was wrapped as an
|
||||
// anonymous error and fell through the caller's `default` branch into the wrong-code message. So a
|
||||
// hub that could not be reached was reported to the customer as a bad recovery code.
|
||||
//
|
||||
// Measured live on 2026-08-05 (CAMPAIGN-11 F3): with the hub REJECTed at the appliance's firewall and
|
||||
// a CORRECT current recovery code, the customer was told the code did not open their package — in
|
||||
// 0.0556 s, when a real unseal costs ~1 s of scrypt. The agent's own log carried the truth the whole
|
||||
// time (`escrow: fetching the sealed bundle: hub: transport error: … no route to host`) and the HTTP
|
||||
// boundary threw it away.
|
||||
//
|
||||
// The discriminator therefore has to be a VALUE, not a log line — that is what ErrBundleFetch is.
|
||||
|
||||
var (
|
||||
// ErrBundleFetch — the sealed bundle could not be FETCHED (the hub refused, was unreachable, or
|
||||
// the transport failed). **The recovery code was never used**, so nothing about it is known and
|
||||
// nothing may be said about it. Wraps the underlying cause for the operator log; carries no secret.
|
||||
ErrBundleFetch = errors.New("escrow: the sealed bundle could not be fetched")
|
||||
// ErrNoEscrowBlob — the hub holds no sealed bundle for this host. Not a fault: no ceremony has run.
|
||||
ErrNoEscrowBlob = errors.New("escrow: the hub holds no sealed identity bundle for this host (no ceremony has run)")
|
||||
// ErrNoResticPassword — the bundle opened, but carries no repository password. Real and expected
|
||||
// for a pre-fork-4 blob (agent < v0.77.0, 2026-07-09): the field did not exist and CANNOT be
|
||||
// retro-fitted, because R is never retained. Distinguished from a wrong code so the operator is
|
||||
// not sent hunting for a mistyped recovery code that was typed correctly.
|
||||
ErrNoResticPassword = errors.New("escrow: the recovered bundle carries NO offsite repository password (a pre-fork-4 blob — the field did not exist when it was sealed and cannot be retro-fitted)")
|
||||
)
|
||||
|
||||
// BlobFetcher yields this host's own opaque identity-escrow blob. present=false is a clean "none".
|
||||
// An interface-free func field keeps this package free of any dependency on the hub client.
|
||||
type BlobFetcher func(ctx context.Context) (blob []byte, present bool, err error)
|
||||
|
||||
// OffsiteKeyRecoverer is the assembled links 6→8. Construct it with a fetcher; call it with R.
|
||||
type OffsiteKeyRecoverer struct {
|
||||
Fetch BlobFetcher
|
||||
}
|
||||
|
||||
// RecoverOffsiteRepoPassword fetches, unseals and extracts. It returns ONLY the repository password.
|
||||
//
|
||||
// A WRONG RECOVERY CODE FAILS CLOSED at the scrypt KDF inside UnwrapIdentity — `age -d` exits
|
||||
// non-zero and emits no plaintext, so there is no partial result and nothing is written anywhere.
|
||||
// That property is the crypto's, not a check here, which is why this function has no "validate R"
|
||||
// step to get wrong.
|
||||
//
|
||||
// NOTHING IS LOGGED BY THIS FUNCTION and no error it returns contains R, the password, or blob bytes.
|
||||
func (r OffsiteKeyRecoverer) RecoverOffsiteRepoPassword(ctx context.Context, recoveryCode string) (string, error) {
|
||||
if r.Fetch == nil {
|
||||
return "", fmt.Errorf("escrow: recoverer has no blob fetcher configured")
|
||||
}
|
||||
if recoveryCode == "" {
|
||||
return "", fmt.Errorf("escrow: the recovery code is required")
|
||||
}
|
||||
blob, present, err := r.Fetch(ctx)
|
||||
if err != nil {
|
||||
// R-224: joined with ErrBundleFetch so the caller can classify by VALUE. The cause stays
|
||||
// wrapped for the operator log; neither carries a secret. Before this, the fetch failure was
|
||||
// an anonymous error and the local-api handler's `default` branch reported it to the customer
|
||||
// as a wrong recovery code.
|
||||
return "", fmt.Errorf("%w: %w", ErrBundleFetch, err)
|
||||
}
|
||||
if !present || len(blob) == 0 {
|
||||
return "", ErrNoEscrowBlob
|
||||
}
|
||||
bundle, err := UnwrapIdentityBundle(ctx, blob, recoveryCode)
|
||||
if err != nil {
|
||||
return "", err // already the fail-closed "the recovery code did not unwrap…" message; no secret in it
|
||||
}
|
||||
if bundle.ResticRepoPassword == "" {
|
||||
return "", ErrNoResticPassword
|
||||
}
|
||||
return bundle.ResticRepoPassword, nil
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
package escrow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-199 links 6→8, with REAL crypto (age is present on the build/demo host; ensureAge skips
|
||||
// elsewhere). These are the unit half of the session's question — "is the repository password
|
||||
// actually recoverable from the sealed bundle" — and the live half is the same equality on hardware.
|
||||
|
||||
const testR = "correct horse battery staple sedative anaconda wobbly kingdom placard yodel"
|
||||
|
||||
func sealBundle(t *testing.T, b IdentityBundle, r string) []byte {
|
||||
t.Helper()
|
||||
blob, err := WrapIdentityBundle(context.Background(), b, r)
|
||||
if err != nil {
|
||||
t.Fatalf("WrapIdentityBundle: %v", err)
|
||||
}
|
||||
return blob
|
||||
}
|
||||
|
||||
func fetcherFor(blob []byte) BlobFetcher {
|
||||
return func(context.Context) ([]byte, bool, error) { return blob, true, nil }
|
||||
}
|
||||
|
||||
// Scenario A (unit) — the recovered repository password is BYTE-IDENTICAL to the sealed one, and it
|
||||
// is the REPOSITORY password rather than some other field of a bundle that also parses.
|
||||
//
|
||||
// RED-PROOF: return bundle.PBSToken (or TunnelToken, or WGPrivateKey) instead of
|
||||
// bundle.ResticRepoPassword → a plausible-looking bundle yields a non-matching key → this FAILS.
|
||||
// That mutation is the shape of the bug that would otherwise ship silently, because every one of
|
||||
// those fields is a non-empty string that looks like a secret.
|
||||
func TestRecoverOffsiteRepoPassword_ReturnsTheRepositoryPassword(t *testing.T) {
|
||||
ensureAge(t)
|
||||
const repoPW = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
blob := sealBundle(t, IdentityBundle{
|
||||
TunnelToken: "TUNNEL-TOKEN-NOT-THE-ANSWER",
|
||||
PBSToken: "PBS-TOKEN-NOT-THE-ANSWER",
|
||||
WGPrivateKey: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=",
|
||||
ResticRepoPassword: repoPW,
|
||||
}, testR)
|
||||
|
||||
got, err := (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), testR)
|
||||
if err != nil {
|
||||
t.Fatalf("recover: %v", err)
|
||||
}
|
||||
if got != repoPW {
|
||||
t.Fatalf("the recovered key is not the sealed repository password (len %d vs %d) — a different "+
|
||||
"field of the bundle was returned", len(got), len(repoPW))
|
||||
}
|
||||
// Belt: it must not be any of the OTHER fields, so a future refactor cannot satisfy the check
|
||||
// above by coincidence.
|
||||
for _, other := range []string{"TUNNEL-TOKEN-NOT-THE-ANSWER", "PBS-TOKEN-NOT-THE-ANSWER", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="} {
|
||||
if got == other {
|
||||
t.Fatalf("the recoverer returned the wrong bundle field")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario B — a WRONG recovery code fails closed, the failure names no secret, and nothing is
|
||||
// written. The fail-closed property is the crypto's (age's scrypt KDF), which is why there is no
|
||||
// validation step here to get wrong — the test pins that it stays that way.
|
||||
func TestRecoverOffsiteRepoPassword_WrongCodeFailsClosed(t *testing.T) {
|
||||
ensureAge(t)
|
||||
const repoPW = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
|
||||
blob := sealBundle(t, IdentityBundle{TunnelToken: "t", PBSToken: "p", ResticRepoPassword: repoPW}, testR)
|
||||
|
||||
got, err := (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), "not the recovery code at all")
|
||||
if err == nil {
|
||||
t.Fatal("a wrong recovery code MUST fail — a plausible-but-wrong bundle is the one outcome the design forbids")
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("a failed unseal returned %d bytes — there must be no partial result", len(got))
|
||||
}
|
||||
// The error may name the step; it may never name a secret.
|
||||
for _, secret := range []string{repoPW, testR, "not the recovery code at all"} {
|
||||
if strings.Contains(err.Error(), secret) {
|
||||
t.Fatalf("the failure message leaked a secret: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A bundle with no repository password is its OWN answer, not a wrong-code error. Sealed before
|
||||
// fork-4 (agent < v0.77.0) the field did not exist; sending the operator to re-check a correctly
|
||||
// typed recovery code would be the wrong instruction.
|
||||
func TestRecoverOffsiteRepoPassword_PreForkFourBundle(t *testing.T) {
|
||||
ensureAge(t)
|
||||
blob := sealBundle(t, IdentityBundle{TunnelToken: "t", PBSToken: "p"}, testR)
|
||||
|
||||
_, err := (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), testR)
|
||||
if !errors.Is(err, ErrNoResticPassword) {
|
||||
t.Fatalf("a pre-fork-4 bundle must report its own error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario D at this layer — no blob is a clean, distinguishable answer.
|
||||
func TestRecoverOffsiteRepoPassword_NoBlob(t *testing.T) {
|
||||
rec := OffsiteKeyRecoverer{Fetch: func(context.Context) ([]byte, bool, error) { return nil, false, nil }}
|
||||
_, err := rec.RecoverOffsiteRepoPassword(context.Background(), testR)
|
||||
if !errors.Is(err, ErrNoEscrowBlob) {
|
||||
t.Fatalf("absent blob must yield ErrNoEscrowBlob, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario F — R persists NOWHERE. TMPDIR is redirected into the test's own directory, the unseal is
|
||||
// run for real, and the whole tree is then walked: no file may contain R (or the recovered password),
|
||||
// and the staging directory the unseal creates must be gone.
|
||||
//
|
||||
// RED-PROOF: write R to a temp file anywhere in the flow (e.g. add
|
||||
// `os.WriteFile(filepath.Join(work,"r"), []byte(recoveryCode), 0o600)` inside UnwrapIdentity before
|
||||
// its defer removes the dir — or simply drop that defer and let the plaintext staging survive) → the
|
||||
// walk finds it → this FAILS.
|
||||
func TestRecoverOffsiteRepoPassword_RLeavesNoTrace(t *testing.T) {
|
||||
ensureAge(t)
|
||||
const repoPW = "1111111111111111111111111111111111111111111111111111111111111111"
|
||||
tmp := t.TempDir()
|
||||
t.Setenv("TMPDIR", tmp) // os.MkdirTemp honours this — every staging dir lands under the walk
|
||||
|
||||
const wrongR = "wrong code entirely"
|
||||
blob := sealBundle(t, IdentityBundle{TunnelToken: "t", PBSToken: "p", ResticRepoPassword: repoPW}, testR)
|
||||
if _, err := (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), testR); err != nil {
|
||||
t.Fatalf("recover: %v", err)
|
||||
}
|
||||
// A failed unseal must leave nothing either — exercise both paths before walking.
|
||||
_, _ = (OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}).RecoverOffsiteRepoPassword(context.Background(), wrongR)
|
||||
|
||||
// THE PRIMARY ASSERTION IS EMPTINESS, not content. A content scan alone is defeatable by a later
|
||||
// call OVERWRITING the leaked file with a different secret — which is exactly how the first
|
||||
// version of this test passed its own red-proof while R sat on disk. Nothing in this test writes
|
||||
// under TMPDIR, so after both calls the tree must contain no files at all.
|
||||
var survivors []string
|
||||
err := filepath.Walk(tmp, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info == nil || info.IsDir() || path == tmp {
|
||||
return nil
|
||||
}
|
||||
survivors = append(survivors, strings.TrimPrefix(path, tmp))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(survivors) > 0 {
|
||||
t.Fatalf("the unseal left %d file(s) behind under TMPDIR: %v — R, the sealed blob and the "+
|
||||
"recovered plaintext all pass through there and none of them may outlive the call", len(survivors), survivors)
|
||||
}
|
||||
// Defence in depth: any secret that DOES appear anywhere is named, for every code used.
|
||||
_ = filepath.Walk(tmp, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info == nil || info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
body, rerr := os.ReadFile(path)
|
||||
if rerr != nil {
|
||||
return nil
|
||||
}
|
||||
for label, secret := range map[string]string{"R": testR, "a wrong R": wrongR, "the repository password": repoPW} {
|
||||
if strings.Contains(string(body), secret) {
|
||||
t.Errorf("%s survived on disk at %s", label, path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// And the staging directories are gone, not merely free of secrets.
|
||||
entries, _ := os.ReadDir(tmp)
|
||||
for _, e := range entries {
|
||||
if e.IsDir() && strings.HasPrefix(e.Name(), "felhom-idesc-") {
|
||||
t.Fatalf("an unseal staging directory survived: %s", e.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A fetch failure surfaces as a fetch failure, not as a wrong-code error — the operator must not be
|
||||
// sent to re-read their recovery code because the hub was unreachable.
|
||||
//
|
||||
// ⚠ THIS TEST WAS GREEN THROUGHOUT THE DEFECT IT DESCRIBES (R-224, 2026-08-06). Its sentence is
|
||||
// exactly right and it did not prevent anything, for two reasons worth keeping:
|
||||
//
|
||||
// 1. **It asserted the MECHANISM, one layer below the consequence.** It checked this package's error
|
||||
// STRING. The merge happened one layer up, in the local-api handler's `default` branch, which
|
||||
// answered a fetch failure with "the recovery code did not open the sealed bundle". The customer
|
||||
// never sees this string; they see that one. The project's own rule — prefer the test that asserts
|
||||
// the CONSEQUENCE (does the customer get blamed?) over the one that asserts the MECHANISM (is the
|
||||
// error distinct here?) — names this case precisely.
|
||||
// 2. **It asserted on TEXT.** `strings.Contains(err.Error(), …)` cannot be consumed by a caller, so
|
||||
// it pinned something no production code could branch on. The distinction it checked was real and
|
||||
// unusable.
|
||||
//
|
||||
// It now asserts the SENTINEL, which is what the handler branches on, and its consequence-level twin
|
||||
// lives in `internal/localapi/escrow_recover_class_test.go` where the status is asserted.
|
||||
func TestRecoverOffsiteRepoPassword_FetchErrorIsDistinct(t *testing.T) {
|
||||
rec := OffsiteKeyRecoverer{Fetch: func(context.Context) ([]byte, bool, error) {
|
||||
return nil, false, errors.New("hub: connection refused")
|
||||
}}
|
||||
_, err := rec.RecoverOffsiteRepoPassword(context.Background(), testR)
|
||||
if err == nil || !errors.Is(err, ErrBundleFetch) {
|
||||
t.Fatalf("a fetch failure must classify as ErrBundleFetch, got %v", err)
|
||||
}
|
||||
if errors.Is(err, ErrNoEscrowBlob) || errors.Is(err, ErrNoResticPassword) {
|
||||
t.Fatal("a transport failure must not masquerade as a content verdict")
|
||||
}
|
||||
}
|
||||
|
||||
// ── R-224 — A FAILED FETCH IS NOT A WRONG CODE ──────────────────────────────────────────────────
|
||||
//
|
||||
// CAMPAIGN-11 F3 measured the consequence of these two being indistinguishable: with the hub
|
||||
// firewalled off and a CORRECT current recovery code, the customer was told the code did not open
|
||||
// their package, in 0.0556 s — no unseal was attempted at all.
|
||||
//
|
||||
// The pair below is the whole point. Asserting only the first would pass with a `return ErrBundleFetch`
|
||||
// stuck on every error path, which is the same defect pointing the other way.
|
||||
func TestRecoverOffsiteRepoPassword_FetchFailureIsClassifiedAsFetch(t *testing.T) {
|
||||
boom := errors.New("hub: transport error: dial tcp 37.191.56.193:443: connect: no route to host")
|
||||
r := OffsiteKeyRecoverer{Fetch: func(context.Context) ([]byte, bool, error) { return nil, false, boom }}
|
||||
|
||||
_, err := r.RecoverOffsiteRepoPassword(context.Background(), testR)
|
||||
if err == nil {
|
||||
t.Fatal("a failing fetch must return an error")
|
||||
}
|
||||
// RED-PROOF: drop the `%w: %w` join in RecoverOffsiteRepoPassword (return the bare wrapped cause,
|
||||
// as it was before R-224) → this FAILS, and the local-api handler falls back to the wrong-code
|
||||
// message exactly as it did on 2026-08-05.
|
||||
if !errors.Is(err, ErrBundleFetch) {
|
||||
t.Fatalf("a failed fetch must classify as ErrBundleFetch, got %v", err)
|
||||
}
|
||||
// The underlying cause survives for the operator log.
|
||||
if !errors.Is(err, boom) {
|
||||
t.Fatalf("the fetch cause must stay wrapped for the operator, got %v", err)
|
||||
}
|
||||
// And it must NOT be mistaken for either of the bundle-content situations.
|
||||
if errors.Is(err, ErrNoEscrowBlob) || errors.Is(err, ErrNoResticPassword) {
|
||||
t.Fatalf("a transport failure is neither of the bundle-content errors: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The other half: a genuinely wrong code must NOT classify as a fetch failure, or the fix trades one
|
||||
// misattribution for its mirror image and the customer is told the hub is down when they mistyped.
|
||||
func TestRecoverOffsiteRepoPassword_WrongCodeIsNotAFetchFailure(t *testing.T) {
|
||||
ensureAge(t)
|
||||
blob := sealBundle(t, IdentityBundle{ResticRepoPassword: "0123456789abcdef"}, testR)
|
||||
r := OffsiteKeyRecoverer{Fetch: fetcherFor(blob)}
|
||||
|
||||
_, err := r.RecoverOffsiteRepoPassword(context.Background(),
|
||||
"wrong horse battery staple sedative anaconda wobbly kingdom placard yodel")
|
||||
if err == nil {
|
||||
t.Fatal("a wrong recovery code must fail closed")
|
||||
}
|
||||
if errors.Is(err, ErrBundleFetch) {
|
||||
t.Fatalf("a wrong code must NOT classify as a fetch failure, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A clean "the hub holds nothing" keeps its own identity too — it is not a fetch failure, and the
|
||||
// customer must not be told the hub was unreachable when it answered perfectly well.
|
||||
func TestRecoverOffsiteRepoPassword_AbsentBlobIsNotAFetchFailure(t *testing.T) {
|
||||
r := OffsiteKeyRecoverer{Fetch: func(context.Context) ([]byte, bool, error) { return nil, false, nil }}
|
||||
_, err := r.RecoverOffsiteRepoPassword(context.Background(), testR)
|
||||
if !errors.Is(err, ErrNoEscrowBlob) {
|
||||
t.Fatalf("an absent blob must stay ErrNoEscrowBlob, got %v", err)
|
||||
}
|
||||
if errors.Is(err, ErrBundleFetch) {
|
||||
t.Fatalf("an absent blob is not a fetch FAILURE, got %v", err)
|
||||
}
|
||||
}
|
||||
+58
-17
@@ -29,9 +29,20 @@ import (
|
||||
//go:embed eff_large_wordlist.txt
|
||||
var wordlistRaw []byte
|
||||
|
||||
// wordlist is the EFF large wordlist (7776 words, 12.92 bits/word) — the diceware standard for
|
||||
// human-transcribed passphrases. Parsed once at init.
|
||||
var wordlist = parseWordlist(wordlistRaw)
|
||||
// RecoveryCodeSep joins the words of a recovery code R. It is ALSO the reason for the
|
||||
// joinSafe filter below: a word that itself contains the separator makes the joined code
|
||||
// ambiguous to segment by eye, which is unaffordable in the one situation R exists for — a
|
||||
// customer transcribing it during a disaster. Do not change it: R is consumed as a whole
|
||||
// passphrase (see Wrap/Unwrap), so the separator is a transcription aid, not a parsed delimiter.
|
||||
const RecoveryCodeSep = "-"
|
||||
|
||||
// wordlist is the EFF large wordlist (the diceware standard for human-transcribed passphrases),
|
||||
// minus the handful of entries that contain RecoveryCodeSep. Parsed and filtered once at init.
|
||||
// Sizes are asserted in wordlist_test.go so a wordlist swap cannot silently move the entropy floor.
|
||||
var wordlist = joinSafe(parseWordlist(wordlistRaw))
|
||||
|
||||
// wordlistRawSize is the unfiltered parse length, kept for audit (see WordlistFilteredOut).
|
||||
var wordlistRawSize = len(parseWordlist(wordlistRaw))
|
||||
|
||||
func parseWordlist(raw []byte) []string {
|
||||
var w []string
|
||||
@@ -44,28 +55,55 @@ func parseWordlist(raw []byte) []string {
|
||||
return w
|
||||
}
|
||||
|
||||
// RecoveryCodeWords is the number of words in a recovery code R. 10 words from the 7776-word EFF
|
||||
// list ≈ 129.2 bits (≥128) — the chosen entropy floor (slice7-escrow-spike-findings.md §4).
|
||||
// joinSafe drops every word containing RecoveryCodeSep, so that a generated code always segments
|
||||
// back into exactly RecoveryCodeWords words. In the EFF large list this removes exactly 4 entries
|
||||
// (drop-down, felt-tip, t-shirt, yo-yo) of 7776, costing ~0.0007 bits/word — the floor still holds
|
||||
// (asserted in the tests). Generation-time only: codes already issued remain valid, because R is
|
||||
// verified as a whole passphrase and is never re-split.
|
||||
func joinSafe(words []string) []string {
|
||||
out := make([]string, 0, len(words))
|
||||
for _, w := range words {
|
||||
if strings.Contains(w, RecoveryCodeSep) {
|
||||
continue
|
||||
}
|
||||
out = append(out, w)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RecoveryCodeWords is the number of words in a recovery code R. 10 words from the filtered EFF
|
||||
// list (7772 words) ≈ 129.2 bits (≥128) — the chosen entropy floor (slice7-escrow-spike-findings.md §4).
|
||||
const RecoveryCodeWords = 10
|
||||
|
||||
// GenerateRecoveryCode returns a fresh recovery code R: RecoveryCodeWords words chosen uniformly
|
||||
// (crypto/rand via big.Int — no modulo bias) from the EFF large wordlist, hyphen-joined.
|
||||
//
|
||||
// SECRET: the returned string is R. Surface it to the customer exactly once; never log or persist it.
|
||||
func GenerateRecoveryCode() (string, error) {
|
||||
if len(wordlist) < 2 {
|
||||
return "", fmt.Errorf("escrow: wordlist not loaded (%d words)", len(wordlist))
|
||||
// generateWords draws RecoveryCodeWords words uniformly (crypto/rand via big.Int — no modulo bias)
|
||||
// from list. Split out from GenerateRecoveryCode so tests can drive an unfiltered list and prove
|
||||
// the filter is what keeps a code segmentable.
|
||||
func generateWords(list []string) ([]string, error) {
|
||||
if len(list) < 2 {
|
||||
return nil, fmt.Errorf("escrow: wordlist not loaded (%d words)", len(list))
|
||||
}
|
||||
n := big.NewInt(int64(len(wordlist)))
|
||||
n := big.NewInt(int64(len(list)))
|
||||
words := make([]string, RecoveryCodeWords)
|
||||
for i := range words {
|
||||
idx, err := rand.Int(rand.Reader, n)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("escrow: recovery-code rng: %w", err)
|
||||
return nil, fmt.Errorf("escrow: recovery-code rng: %w", err)
|
||||
}
|
||||
words[i] = wordlist[idx.Int64()]
|
||||
words[i] = list[idx.Int64()]
|
||||
}
|
||||
return strings.Join(words, "-"), nil
|
||||
return words, nil
|
||||
}
|
||||
|
||||
// GenerateRecoveryCode returns a fresh recovery code R: RecoveryCodeWords words chosen uniformly
|
||||
// from the filtered EFF large wordlist, joined with RecoveryCodeSep.
|
||||
//
|
||||
// SECRET: the returned string is R. Surface it to the customer exactly once; never log or persist it.
|
||||
func GenerateRecoveryCode() (string, error) {
|
||||
words, err := generateWords(wordlist)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.Join(words, RecoveryCodeSep), nil
|
||||
}
|
||||
|
||||
// RecoveryCodeEntropyBits is the approximate entropy of a generated code, for display/audit only
|
||||
@@ -77,5 +115,8 @@ func RecoveryCodeEntropyBits() float64 {
|
||||
return float64(RecoveryCodeWords) * math.Log2(float64(len(wordlist)))
|
||||
}
|
||||
|
||||
// WordlistSize is the loaded wordlist length (for audit/tests).
|
||||
// WordlistSize is the effective (filtered) wordlist length — the draw space. For audit/tests.
|
||||
func WordlistSize() int { return len(wordlist) }
|
||||
|
||||
// WordlistFilteredOut is how many parsed entries joinSafe removed. For audit/tests.
|
||||
func WordlistFilteredOut() int { return wordlistRawSize - len(wordlist) }
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package escrow
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The four EFF large-list entries that contain RecoveryCodeSep. Named here so a wordlist swap that
|
||||
// changes the set fails loudly rather than silently re-opening the ambiguity.
|
||||
var hyphenatedEFFWords = []string{"drop-down", "felt-tip", "t-shirt", "yo-yo"}
|
||||
|
||||
func TestJoinSafe_RemovesExactlyTheHyphenatedEFFWords(t *testing.T) {
|
||||
raw := parseWordlist(wordlistRaw)
|
||||
rawSet := make(map[string]bool, len(raw))
|
||||
for _, w := range raw {
|
||||
rawSet[w] = true
|
||||
}
|
||||
for _, w := range hyphenatedEFFWords {
|
||||
if !rawSet[w] {
|
||||
t.Fatalf("fixture drift: %q is no longer in the embedded EFF list", w)
|
||||
}
|
||||
}
|
||||
|
||||
filtered := joinSafe(raw)
|
||||
if len(raw)-len(filtered) != len(hyphenatedEFFWords) {
|
||||
t.Fatalf("joinSafe removed %d entries, expected exactly %d",
|
||||
len(raw)-len(filtered), len(hyphenatedEFFWords))
|
||||
}
|
||||
got := make(map[string]bool, len(filtered))
|
||||
for _, w := range filtered {
|
||||
if strings.Contains(w, RecoveryCodeSep) {
|
||||
t.Errorf("filtered wordlist still contains a separator-bearing word %q", w)
|
||||
}
|
||||
got[w] = true
|
||||
}
|
||||
for _, w := range hyphenatedEFFWords {
|
||||
if got[w] {
|
||||
t.Errorf("joinSafe kept %q, which contains %q", w, RecoveryCodeSep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEntropyFloorSurvivesFiltering states the numbers explicitly: dropping 4 of 7776 words costs
|
||||
// ~0.0007 bits/word, so the 10-word code stays above the 128-bit floor with room to spare.
|
||||
func TestEntropyFloorSurvivesFiltering(t *testing.T) {
|
||||
const floorBits = 128.0
|
||||
before := float64(RecoveryCodeWords) * math.Log2(7776)
|
||||
after := RecoveryCodeEntropyBits()
|
||||
|
||||
if after < floorBits {
|
||||
t.Fatalf("filtered entropy %.3f bits is below the %.0f-bit floor", after, floorBits)
|
||||
}
|
||||
if want := float64(RecoveryCodeWords) * math.Log2(float64(WordlistSize())); math.Abs(after-want) > 1e-9 {
|
||||
t.Fatalf("RecoveryCodeEntropyBits() = %.6f, want %.6f (10 * log2(%d))", after, want, WordlistSize())
|
||||
}
|
||||
// Concrete expectations, so a wordlist change that quietly erodes the margin is visible:
|
||||
// 10*log2(7776) = 129.248 bits before, 10*log2(7772) = 129.241 bits after — a 0.007-bit cost.
|
||||
if math.Abs(before-129.248) > 0.001 {
|
||||
t.Fatalf("unfiltered entropy baseline moved: %.3f, expected 129.248", before)
|
||||
}
|
||||
if math.Abs(after-129.241) > 0.001 {
|
||||
t.Fatalf("filtered entropy moved: %.3f, expected 129.241", after)
|
||||
}
|
||||
if cost := before - after; cost > 0.01 {
|
||||
t.Fatalf("filtering cost %.4f bits, expected well under 0.01", cost)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeneratedCodeSegments_FilteredVsUnfiltered is the deterministic red-proof companion.
|
||||
//
|
||||
// Against a list where EVERY word contains the separator, a 10-word draw MUST segment into more
|
||||
// than 10 parts — that is the pre-fix behaviour, reproduced with probability 1 instead of the ~1/5
|
||||
// flake the real list produced. Against the same list run through joinSafe, generation must refuse
|
||||
// (nothing is left to draw from), proving joinSafe — not luck — is what makes a code segmentable.
|
||||
func TestGeneratedCodeSegments_FilteredVsUnfiltered(t *testing.T) {
|
||||
unfiltered := hyphenatedEFFWords
|
||||
|
||||
words, err := generateWords(unfiltered)
|
||||
if err != nil {
|
||||
t.Fatalf("generateWords(unfiltered): %v", err)
|
||||
}
|
||||
if len(words) != RecoveryCodeWords {
|
||||
t.Fatalf("generator drew %d words, want %d", len(words), RecoveryCodeWords)
|
||||
}
|
||||
joined := strings.Join(words, RecoveryCodeSep)
|
||||
segs := len(strings.Split(joined, RecoveryCodeSep))
|
||||
if segs <= RecoveryCodeWords {
|
||||
t.Fatalf("unfiltered draw segmented into %d parts; the pre-fix defect should yield more than %d",
|
||||
segs, RecoveryCodeWords)
|
||||
}
|
||||
if segs != 2*RecoveryCodeWords {
|
||||
t.Fatalf("every fixture word has exactly one separator, so 10 words must segment into 20 parts, got %d", segs)
|
||||
}
|
||||
|
||||
// Same fixture, filtered: the draw space is empty, so generation must error rather than
|
||||
// silently fall back to something ambiguous.
|
||||
if _, err := generateWords(joinSafe(unfiltered)); err == nil {
|
||||
t.Fatal("generateWords on a fully-filtered list must fail, not return a code")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGenerateRecoveryCode_NeverContainsAmbiguousWord is the production-wiring test: it asserts the
|
||||
// exported entry point (not just the helper) draws from the filtered list.
|
||||
func TestGenerateRecoveryCode_NeverContainsAmbiguousWord(t *testing.T) {
|
||||
inFiltered := make(map[string]bool, len(wordlist))
|
||||
for _, w := range wordlist {
|
||||
inFiltered[w] = true
|
||||
}
|
||||
for i := 0; i < 500; i++ {
|
||||
r, err := GenerateRecoveryCode()
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateRecoveryCode: %v", err)
|
||||
}
|
||||
parts := strings.Split(r, RecoveryCodeSep)
|
||||
if len(parts) != RecoveryCodeWords {
|
||||
// Do not print r: it is a live-shaped secret.
|
||||
t.Fatalf("code %d segmented into %d parts, want %d", i, len(parts), RecoveryCodeWords)
|
||||
}
|
||||
for _, p := range parts {
|
||||
if !inFiltered[p] {
|
||||
t.Fatalf("segment %q is not a filtered-wordlist word", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
// Package fasttick is the agent-plane immediacy SECONDARY (v0.90.0, R-28). While ANY desired-state
|
||||
// item is still unapplied — most importantly the pre-tunnel WG-registration window where a hub poke
|
||||
// is undeliverable by construction — it pulses the hub control loop's out-of-band report trigger on
|
||||
// a fast (30 s) cadence, and self-disarms EMERGENTLY the instant everything converges. It is the
|
||||
// state-based complement to the poke: the poke handles hub→box changes once the tunnel exists; the
|
||||
// fast-tick handles the window before that (and any lingering unapplied drift) from the box side.
|
||||
//
|
||||
// By ruling it is STATE-BASED, not a fixed burst and not a timer: there is nothing to journal
|
||||
// (stateless across restarts) and nothing to leak. A perma-unconverged box fast-ticks at ~2 small
|
||||
// reports/min, bounded and visible; the LOUD pbsdr states (consumed_failed/verify_failed) are
|
||||
// deliberately EXCLUDED from the sources so a stuck-loud box does not hammer (§8).
|
||||
//
|
||||
// It pulses the SAME cap-1 channel the storage watchdog and the poke listener use, so a pulse
|
||||
// coalesces with a poke/watchdog nudge for free — no extra debounce here.
|
||||
package fasttick
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultInterval is the ruled fast cadence while unconverged.
|
||||
const DefaultInterval = 30 * time.Second
|
||||
|
||||
// Source reports whether one subsystem still has unapplied desired-state. Implementations MUST be a
|
||||
// cheap, CACHED read — no exec, no network per call (the fast-tick calls every source each tick).
|
||||
type Source interface {
|
||||
Unconverged() (unconverged bool, reason string)
|
||||
}
|
||||
|
||||
// SourceFunc adapts a plain func to a Source (main.go closes over each subsystem).
|
||||
type SourceFunc func() (bool, string)
|
||||
|
||||
// Unconverged implements Source.
|
||||
func (f SourceFunc) Unconverged() (bool, string) { return f() }
|
||||
|
||||
// Loop evaluates the sources on a ticker and pulses the out-of-band channel while any is unconverged.
|
||||
type Loop struct {
|
||||
sources []Source
|
||||
out chan<- struct{}
|
||||
interval time.Duration
|
||||
logger *slog.Logger
|
||||
armed bool // for armed↔disarmed transition logging (avoids 30 s reason spam)
|
||||
}
|
||||
|
||||
// New builds a fast-tick loop. out is the hub loop's out-of-band trigger channel (cap-1). A
|
||||
// non-positive interval falls back to DefaultInterval.
|
||||
func New(out chan<- struct{}, interval time.Duration, logger *slog.Logger, sources ...Source) *Loop {
|
||||
if interval <= 0 {
|
||||
interval = DefaultInterval
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Loop{sources: sources, out: out, interval: interval, logger: logger}
|
||||
}
|
||||
|
||||
// Run evaluates the sources every interval until ctx is cancelled. Stateless — nothing to recover.
|
||||
func (l *Loop) Run(ctx context.Context) error {
|
||||
l.logger.Info("fast-tick armed: "+l.interval.String()+" out-of-band cadence while desired-state is unapplied", "interval", l.interval)
|
||||
ticker := time.NewTicker(l.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
l.logger.Info("fast-tick: shutting down", "reason", ctx.Err())
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
l.step()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// step evaluates the sources once. If any is unconverged it pulses the channel (non-blocking: a full
|
||||
// channel means an out-of-band report is already pending, so the pulse coalesces harmlessly) and, on
|
||||
// the disarmed→armed edge, logs the reason. When all converge it logs the armed→disarmed edge once.
|
||||
// Returns whether this tick found the box unconverged (test hook). No per-tick logging when steady.
|
||||
func (l *Loop) step() bool {
|
||||
unconverged, reason := l.evaluate()
|
||||
if unconverged {
|
||||
select {
|
||||
case l.out <- struct{}{}:
|
||||
default: // an out-of-band report is already queued — coalesce, never block
|
||||
}
|
||||
if !l.armed {
|
||||
l.logger.Info("fast-tick: desired-state unapplied — pulsing out-of-band reports", "reason", reason, "cadence", l.interval)
|
||||
l.armed = true
|
||||
}
|
||||
} else if l.armed {
|
||||
l.logger.Info("fast-tick: desired-state converged — back to the normal cadence")
|
||||
l.armed = false
|
||||
}
|
||||
return unconverged
|
||||
}
|
||||
|
||||
// evaluate returns the first unconverged source's reason (order = priority for the log line).
|
||||
func (l *Loop) evaluate() (bool, string) {
|
||||
for _, s := range l.sources {
|
||||
if u, reason := s.Unconverged(); u {
|
||||
return true, reason
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package fasttick
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func quiet() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }
|
||||
|
||||
// flagSource is a fake Source whose convergence the test flips at will.
|
||||
type flagSource struct {
|
||||
unconverged bool
|
||||
reason string
|
||||
}
|
||||
|
||||
func (f *flagSource) Unconverged() (bool, string) { return f.unconverged, f.reason }
|
||||
|
||||
// drain reports how many pulses are queued (channel is cap-1 in production; tests may use larger).
|
||||
func drain(ch chan struct{}) int {
|
||||
n := 0
|
||||
for {
|
||||
select {
|
||||
case <-ch:
|
||||
n++
|
||||
default:
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// D1 — any source unconverged → one pulse per tick (drained between ticks).
|
||||
func TestFastTick_UnconvergedPulses(t *testing.T) {
|
||||
out := make(chan struct{}, 1)
|
||||
src := &flagSource{unconverged: true, reason: "test drift"}
|
||||
l := New(out, time.Hour, quiet(), src)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
if !l.step() {
|
||||
t.Fatalf("tick %d: step reported converged, want unconverged", i)
|
||||
}
|
||||
if got := drain(out); got != 1 {
|
||||
t.Fatalf("tick %d: pulses = %d, want 1", i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// D2 — ALL sources converged → zero pulses across N ticks (the emergent disarm).
|
||||
func TestFastTick_ConvergedSilent(t *testing.T) {
|
||||
out := make(chan struct{}, 4)
|
||||
l := New(out, time.Hour, quiet(), &flagSource{unconverged: false}, &flagSource{unconverged: false})
|
||||
for i := 0; i < 5; i++ {
|
||||
if l.step() {
|
||||
t.Fatalf("tick %d: step reported unconverged with all sources converged", i)
|
||||
}
|
||||
}
|
||||
if got := drain(out); got != 0 {
|
||||
t.Fatalf("pulses on a fully-converged box = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
// D3 (the ruled red-proof) — flip unconverged→converged mid-run → pulses STOP from the next tick.
|
||||
func TestFastTick_ConvergenceDisarms(t *testing.T) {
|
||||
out := make(chan struct{}, 8)
|
||||
src := &flagSource{unconverged: true, reason: "drift"}
|
||||
l := New(out, time.Hour, quiet(), src)
|
||||
|
||||
// Two unconverged ticks pulse.
|
||||
l.step()
|
||||
l.step()
|
||||
if got := drain(out); got != 2 {
|
||||
t.Fatalf("pre-convergence pulses = %d, want 2", got)
|
||||
}
|
||||
// Converge.
|
||||
src.unconverged = false
|
||||
// Every subsequent tick is silent — the cadence returns to normal.
|
||||
for i := 0; i < 4; i++ {
|
||||
if l.step() {
|
||||
t.Fatalf("post-convergence tick %d still unconverged", i)
|
||||
}
|
||||
}
|
||||
if got := drain(out); got != 0 {
|
||||
t.Fatalf("pulses fired after convergence = %d, want 0 (disarm failed)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// D4 — the out channel is already full (a poke just landed): the non-blocking send drops, no block,
|
||||
// no goroutine leak, no queue growth.
|
||||
func TestFastTick_ChannelFullDrops(t *testing.T) {
|
||||
out := make(chan struct{}, 1)
|
||||
out <- struct{}{} // pre-fill: an out-of-band report is already pending
|
||||
l := New(out, time.Hour, quiet(), &flagSource{unconverged: true, reason: "drift"})
|
||||
|
||||
done := make(chan bool, 1)
|
||||
go func() {
|
||||
l.step() // must NOT block on the full channel
|
||||
l.step()
|
||||
done <- true
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("step blocked on a full channel (non-blocking send violated)")
|
||||
}
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("channel depth = %d, want 1 (coalesced, no queue growth)", len(out))
|
||||
}
|
||||
}
|
||||
|
||||
// Priority ordering: the first unconverged source supplies the reason.
|
||||
func TestFastTick_FirstReasonWins(t *testing.T) {
|
||||
out := make(chan struct{}, 1)
|
||||
l := New(out, time.Hour, quiet(),
|
||||
&flagSource{unconverged: false},
|
||||
&flagSource{unconverged: true, reason: "second"},
|
||||
&flagSource{unconverged: true, reason: "third"})
|
||||
if u, r := l.evaluate(); !u || r != "second" {
|
||||
t.Fatalf("evaluate = (%v, %q), want (true, second)", u, r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package felhomsshd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// Belt keeps the STATIC nft table `inet felhom_oob` (installed by host-install) converged on the
|
||||
// current OOB state by mutating only its SETS — @operator_ips (the operator /32 allowed to reach
|
||||
// felhom-sshd over wg-felhom) and @ssh_port (felhom-sshd's claimed port). The agent NEVER adds/removes
|
||||
// RULES [trap 4]: set-element mutation can't change rule semantics, so the narrow sudoers grant stays
|
||||
// safe. Idempotent: reads the current elements and mutates only on a difference (no churn).
|
||||
type Belt struct {
|
||||
runner proxmox.Runner
|
||||
logger *slog.Logger
|
||||
|
||||
table string // "felhom_oob"
|
||||
}
|
||||
|
||||
// NewBelt builds a Belt over the narrow runner. table defaults to "felhom_oob".
|
||||
func NewBelt(runner proxmox.Runner, logger *slog.Logger) *Belt {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Belt{runner: runner, logger: logger, table: "felhom_oob"}
|
||||
}
|
||||
|
||||
// SyncPort converges @ssh_port on the claimed port (0 empties it). Always safe to call — the port
|
||||
// comes from the agent's own claim, not the hub desired-state.
|
||||
func (b *Belt) SyncPort(ctx context.Context, port int) {
|
||||
wantPorts := []string{}
|
||||
if port > 0 && port <= 65535 {
|
||||
wantPorts = []string{strconv.Itoa(port)}
|
||||
}
|
||||
b.syncSet(ctx, "ssh_port", wantPorts)
|
||||
}
|
||||
|
||||
// SyncOperator converges @operator_ips on the operator /32 (operatorIP "" = OOB explicitly off →
|
||||
// empty the set). Call ONLY when the desired-state has actually been FETCHED — a nil/unfetched block
|
||||
// must NOT empty the set (that would lock the operator out until the next fetch, the wgtunnel
|
||||
// fetched=false-is-never-a-teardown rule). Every value is netip-validated before it reaches nft.
|
||||
func (b *Belt) SyncOperator(ctx context.Context, operatorIP string) {
|
||||
wantOps := []string{}
|
||||
if operatorIP != "" {
|
||||
ip, err := netip.ParseAddr(operatorIP)
|
||||
if err != nil || !ip.Is4() {
|
||||
b.logger.Error("felhomsshd belt: operator ip invalid — refusing", "ip", operatorIP)
|
||||
return
|
||||
}
|
||||
wantOps = []string{ip.String()}
|
||||
}
|
||||
b.syncSet(ctx, "operator_ips", wantOps)
|
||||
}
|
||||
|
||||
var nftElemRe = regexp.MustCompile(`elements\s*=\s*\{([^}]*)\}`)
|
||||
|
||||
// syncSet converges one named set on `want` (sorted, deduped). Reads current elements; if they match,
|
||||
// ZERO nft mutations (the idempotency the scenario asserts). Otherwise flush + add the desired
|
||||
// elements. A read failure (table/set absent) → one warning, no mutation.
|
||||
func (b *Belt) syncSet(ctx context.Context, setName string, want []string) {
|
||||
want = sortedUnique(want)
|
||||
cur, ok := b.readSet(ctx, setName)
|
||||
if !ok {
|
||||
b.logger.Warn("felhomsshd belt: set unreadable (table not installed?) — skipping", "set", setName)
|
||||
return
|
||||
}
|
||||
if equalStringSlices(cur, want) {
|
||||
return // steady state: no mutation
|
||||
}
|
||||
if _, errOut, err := b.runner.Run(ctx, "nft", "flush", "set", "inet", b.table, setName); err != nil {
|
||||
b.logger.Error("felhomsshd belt: flush failed", "set", setName, "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||
return
|
||||
}
|
||||
for _, e := range want {
|
||||
if _, errOut, err := b.runner.Run(ctx, "nft", "add", "element", "inet", b.table, setName, "{ "+e+" }"); err != nil {
|
||||
b.logger.Error("felhomsshd belt: add element failed", "set", setName, "elem", e, "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||
return
|
||||
}
|
||||
}
|
||||
b.logger.Info("felhomsshd belt: set synced", "set", setName, "elements", strings.Join(want, ","))
|
||||
}
|
||||
|
||||
// readSet returns the current elements of a named set (sorted), or ok=false if the set can't be read.
|
||||
func (b *Belt) readSet(ctx context.Context, setName string) ([]string, bool) {
|
||||
out, _, err := b.runner.Run(ctx, "nft", "list", "set", "inet", b.table, setName)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
m := nftElemRe.FindSubmatch(out)
|
||||
if m == nil {
|
||||
return []string{}, true // set exists but empty (no "elements = {}" block)
|
||||
}
|
||||
var elems []string
|
||||
for _, f := range strings.Split(string(m[1]), ",") {
|
||||
f = strings.TrimSpace(f)
|
||||
if f != "" {
|
||||
elems = append(elems, f)
|
||||
}
|
||||
}
|
||||
return sortedUnique(elems), true
|
||||
}
|
||||
|
||||
func sortedUnique(in []string) []string {
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(in))
|
||||
for _, s := range in {
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func equalStringSlices(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package felhomsshd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// scriptRunner records every exec and returns scripted stdout per "name+first-arg" key; scripted
|
||||
// errors per the same key. Everything else returns empty success.
|
||||
type scriptRunner struct {
|
||||
mu sync.Mutex
|
||||
calls [][]string
|
||||
stdout map[string]string
|
||||
errs map[string]error
|
||||
}
|
||||
|
||||
func newScriptRunner() *scriptRunner {
|
||||
return &scriptRunner{stdout: map[string]string{}, errs: map[string]error{}}
|
||||
}
|
||||
|
||||
func key(name string, args ...string) string {
|
||||
// `nft list set inet felhom_oob <setName>` → key by the set name (last arg), so the two set reads
|
||||
// are distinguishable. Everything else keys by first arg.
|
||||
if name == "nft" && len(args) > 0 && args[0] == "list" {
|
||||
return "nft list " + args[len(args)-1]
|
||||
}
|
||||
if len(args) > 0 {
|
||||
return name + " " + args[0]
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (r *scriptRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
r.mu.Lock()
|
||||
r.calls = append(r.calls, append([]string{name}, args...))
|
||||
k := key(name, args...)
|
||||
out := r.stdout[k]
|
||||
err := r.errs[k]
|
||||
r.mu.Unlock()
|
||||
if err != nil {
|
||||
return nil, []byte("scripted failure"), err
|
||||
}
|
||||
return []byte(out), nil, nil
|
||||
}
|
||||
|
||||
func (r *scriptRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
|
||||
return r.Run(ctx, name, args...)
|
||||
}
|
||||
|
||||
func (r *scriptRunner) count(name, firstArg string) int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
n := 0
|
||||
for _, c := range r.calls {
|
||||
if c[0] == name && len(c) > 1 && c[1] == firstArg {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (r *scriptRunner) sawRuleMutation() bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, c := range r.calls {
|
||||
// trap 4: the agent must NEVER run `nft add rule` / `nft -f` / `nft flush ruleset|table`.
|
||||
if c[0] == "nft" && len(c) > 1 {
|
||||
if c[1] == "-f" || (len(c) > 2 && c[1] == "add" && c[2] == "rule") {
|
||||
return true
|
||||
}
|
||||
if c[1] == "flush" && len(c) > 2 && (c[2] == "ruleset" || c[2] == "table") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestBelt_SyncMutatesThenIdempotent(t *testing.T) {
|
||||
r := newScriptRunner()
|
||||
// first read: both sets empty (no "elements" block)
|
||||
r.stdout["nft list operator_ips"] = "set operator_ips {\n\ttype ipv4_addr\n}"
|
||||
r.stdout["nft list ssh_port"] = "set ssh_port {\n\ttype inet_service\n}"
|
||||
b := NewBelt(r, nil)
|
||||
b.SyncPort(context.Background(), 8822)
|
||||
b.SyncOperator(context.Background(), "10.77.0.250")
|
||||
|
||||
// mutations happened: flush + add for BOTH sets
|
||||
if r.count("nft", "flush") < 2 || r.count("nft", "add") < 2 {
|
||||
t.Fatalf("first sync must flush+add both sets; flushes=%d adds=%d", r.count("nft", "flush"), r.count("nft", "add"))
|
||||
}
|
||||
if r.sawRuleMutation() {
|
||||
t.Fatal("belt must NEVER mutate rules — only set elements (trap 4)")
|
||||
}
|
||||
|
||||
// second sync: the reads now return the desired elements → ZERO mutations (idempotent)
|
||||
r2 := newScriptRunner()
|
||||
r2.stdout["nft list operator_ips"] = "elements = { 10.77.0.250 }"
|
||||
r2.stdout["nft list ssh_port"] = "elements = { 8822 }"
|
||||
b2 := NewBelt(r2, nil)
|
||||
b2.SyncPort(context.Background(), 8822)
|
||||
b2.SyncOperator(context.Background(), "10.77.0.250")
|
||||
if r2.count("nft", "flush") != 0 || r2.count("nft", "add") != 0 {
|
||||
t.Fatalf("idempotent sync must do ZERO mutations; flushes=%d adds=%d", r2.count("nft", "flush"), r2.count("nft", "add"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBelt_OperatorEmptyEmptiesSet(t *testing.T) {
|
||||
r := newScriptRunner()
|
||||
r.stdout["nft list operator_ips"] = "elements = { 10.77.0.250 }" // currently has an operator IP
|
||||
r.stdout["nft list ssh_port"] = "elements = { 8822 }"
|
||||
b := NewBelt(r, nil)
|
||||
b.SyncOperator(context.Background(), "") // OOB explicitly off → operator set must be emptied
|
||||
if r.count("nft", "flush") < 1 {
|
||||
t.Fatal("emptying the operator set must flush it")
|
||||
}
|
||||
}
|
||||
|
||||
// health: inactive + INVALID config → NO restart (the red-proof); inactive + valid → restart.
|
||||
func TestHeal_NoRestartOnInvalidConfig(t *testing.T) {
|
||||
r := newScriptRunner()
|
||||
r.errs["sshd -t"] = context.DeadlineExceeded // sshd -t FAILS (config invalid)
|
||||
m := newTestManager(r, false) // inactive
|
||||
m.HealAndCheck(context.Background(), 8822)
|
||||
if r.count("systemctl", "restart") != 0 {
|
||||
t.Fatal("a broken config must NOT trigger a restart (never degraded→dead)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeal_RestartsWhenDownWithValidConfigThenCooldown(t *testing.T) {
|
||||
r := newScriptRunner() // sshd -t passes (no error)
|
||||
m := newTestManager(r, false)
|
||||
m.HealAndCheck(context.Background(), 8822)
|
||||
if r.count("systemctl", "restart") != 1 {
|
||||
t.Fatalf("down + valid config → exactly one restart, got %d", r.count("systemctl", "restart"))
|
||||
}
|
||||
// cooldown: a second call inside the window → still one restart
|
||||
m.HealAndCheck(context.Background(), 8822)
|
||||
if r.count("systemctl", "restart") != 1 {
|
||||
t.Fatalf("restart cooldown violated: %d restarts", r.count("systemctl", "restart"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatus_ReflectsBlockAndPort(t *testing.T) {
|
||||
r := newScriptRunner()
|
||||
r.stdout["sshd -T"] = "port 8822\nsomethingelse yes\n"
|
||||
m := newTestManager(r, true) // active
|
||||
m.port = 8822
|
||||
st := m.Status(context.Background(), &hub.WireWireguard{OOBPeerIP: "10.77.0.250", OOBOperatorSSHKey: "ssh-ed25519 AAAA op"})
|
||||
if !st.FelhomSshdActive || st.FelhomSshdPort != 8822 {
|
||||
t.Fatalf("status: %+v", st)
|
||||
}
|
||||
if !st.OperatorPeerConfigured || !st.OperatorKeyConfigured {
|
||||
t.Fatalf("status must reflect operator peer + key configured: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
// newTestManager builds a Manager with injected active-state + a now clock; sshd -T/-t go through the
|
||||
// runner. isFree unused here.
|
||||
func newTestManager(r *scriptRunner, active bool) *Manager {
|
||||
m := NewManager(r, "/tmp/felhomsshd-test", nil)
|
||||
m.isActive = func(context.Context) bool { return active }
|
||||
m.isFailed = func(context.Context) bool { return false }
|
||||
m.now = func() time.Time { return time.Unix(1783270000, 0) }
|
||||
return m
|
||||
}
|
||||
|
||||
func TestConfig_NoRuntimeDirectoryString(t *testing.T) {
|
||||
c, _ := renderConfig(2222)
|
||||
if strings.Contains(c, "RuntimeDirectory") {
|
||||
t.Fatal("SF-1: config must never contain RuntimeDirectory")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package felhomsshd
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Candidates is the ordered OOB-port candidate list (spike §2). First free wins; NEVER :22 or a
|
||||
// random port. Package-var (not const) so tests can shrink it.
|
||||
var Candidates = []int{8822, 2222, 8022, 62222}
|
||||
|
||||
// ErrPortsExhausted is returned when every candidate is busy — a LOUD failure [SF-4/trap 6], never a
|
||||
// silent fallback to :22 or a random high port.
|
||||
var ErrPortsExhausted = fmt.Errorf("felhomsshd: all candidate OOB ports are busy — refusing to fall back to :22 or a random port")
|
||||
|
||||
// portProbe reports whether a TCP port is free (nothing listening AND a real bind succeeds). Injected
|
||||
// for tests; production impl = probeFree (ss + net.Listen).
|
||||
type portProbe func(port int) bool
|
||||
|
||||
// claimPort returns the OOB port:
|
||||
// - if a port is already PERSISTED (and != 22) → keep it unconditionally. It is OUR port; on a
|
||||
// continuous reconcile felhom-sshd is itself LISTENING on it, so re-probing with isFree would
|
||||
// (wrongly) find it "busy" by our own daemon and thrash to another candidate every tick. Once
|
||||
// claimed, the port is stable (the belt @ssh_port and the operator's known port depend on it).
|
||||
// - else the FIRST free candidate → persist + return (isFree = ss-empty AND a real bind succeeds).
|
||||
// - else ErrPortsExhausted (LOUD — never :22 or a random port).
|
||||
//
|
||||
// persist writes the port file; readPersisted reads it. isFree is the probe. All injected for tests.
|
||||
func claimPort(candidates []int, isFree portProbe, readPersisted func() (int, bool), persist func(int) error) (int, error) {
|
||||
if cur, ok := readPersisted(); ok && cur != 22 {
|
||||
return cur, nil // persisted = ours; keep it (no thrash — felhom-sshd holds it)
|
||||
}
|
||||
for _, p := range candidates {
|
||||
if p == 22 {
|
||||
continue // defensive: never :22
|
||||
}
|
||||
if isFree(p) {
|
||||
if err := persist(p); err != nil {
|
||||
return 0, fmt.Errorf("felhomsshd: persist claimed port %d: %w", p, err)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
}
|
||||
return 0, ErrPortsExhausted
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// Package felhomsshd manages the dedicated OOB sshd instance (TASK H1). It is a SECOND sshd —
|
||||
// separate port, config, host keys, AuthorizedKeysFile, and systemd unit — that COEXISTS with the
|
||||
// customer's/stock sshd on :22 (never touched). Design + safety earned by
|
||||
// SPIKE-felhom-sshd-2026-07-05 (§2 claim, §3 SAFE unit, §5 reload-not-restart, §7 AuthorizedKeysFile
|
||||
// isolation) and SPIKE-oob-wg-operator-peer-2026-07-05 (the tunnel-only belt).
|
||||
//
|
||||
// The agent RENDERS the config (Port from the claim) and reloads on change — the wg-felhom pattern.
|
||||
// It NEVER declares RuntimeDirectory= (G1 [SF-1]) and NEVER restarts on a config change [SF-2].
|
||||
package felhomsshd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
// ConfDir is the dedicated config tree (host-install creates it; the agent renders the config).
|
||||
ConfDir = "/etc/felhom-sshd"
|
||||
// ConfPath is the rendered sshd config (referenced by the static unit's ExecStart/ExecReload).
|
||||
ConfPath = ConfDir + "/sshd_config"
|
||||
// HostKeyPath is the dedicated ed25519 host key (host-install generates it; stable across reloads).
|
||||
HostKeyPath = ConfDir + "/ssh_host_ed25519_key"
|
||||
// AuthKeysDir holds per-user authorized_keys OUTSIDE ~/.ssh, so the customer's sshd (which reads
|
||||
// ~/.ssh/authorized_keys) structurally cannot honour the operator key [SF-3/§7].
|
||||
AuthKeysDir = ConfDir + "/authorized_keys"
|
||||
// PortFile persists the claimed port (idempotent re-pick).
|
||||
PortFile = ConfDir + "/port"
|
||||
// PidFile is the instance pidfile (NOT a RuntimeDirectory — that is the G1 incident cause).
|
||||
PidFile = "/run/felhom-sshd.pid"
|
||||
// Unit is the systemd unit name.
|
||||
Unit = "felhom-sshd"
|
||||
// OperatorUser is the default operator login (scoped sudo; key in AuthKeysDir only).
|
||||
OperatorUser = "felhom-op"
|
||||
)
|
||||
|
||||
// renderConfig builds the felhom-sshd config for a claimed port. Pure + deterministic (byte-stable
|
||||
// for a given port → a stable conf-hash, no reload churn). The security posture is the SAFE template
|
||||
// from the spike §3: key-only, dedicated host key + AuthorizedKeysFile, AllowUsers scoped to
|
||||
// root+felhom-op, binds 0.0.0.0 (+ ::) so it never waits on a late interface, no RuntimeDirectory.
|
||||
func renderConfig(port int) (string, error) {
|
||||
if port < 1 || port > 65535 {
|
||||
return "", fmt.Errorf("felhomsshd: port %d out of range", port)
|
||||
}
|
||||
if port == 22 {
|
||||
// The whole point is coexistence — the dedicated instance must NEVER claim :22 [SF-4/trap 5].
|
||||
return "", fmt.Errorf("felhomsshd: refusing to render on :22 (the stock/customer sshd port)")
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("# felhom OOB sshd — agent-managed (H1); DO NOT EDIT\n")
|
||||
fmt.Fprintf(&b, "Port %d\n", port)
|
||||
b.WriteString("ListenAddress 0.0.0.0\n")
|
||||
b.WriteString("ListenAddress ::\n")
|
||||
fmt.Fprintf(&b, "HostKey %s\n", HostKeyPath)
|
||||
fmt.Fprintf(&b, "PidFile %s\n", PidFile)
|
||||
fmt.Fprintf(&b, "AuthorizedKeysFile %s/%%u\n", AuthKeysDir)
|
||||
b.WriteString("PasswordAuthentication no\n")
|
||||
b.WriteString("PermitRootLogin prohibit-password\n")
|
||||
b.WriteString("PubkeyAuthentication yes\n")
|
||||
b.WriteString("KbdInteractiveAuthentication no\n")
|
||||
b.WriteString("UsePAM yes\n")
|
||||
fmt.Fprintf(&b, "AllowUsers root %s\n", OperatorUser)
|
||||
b.WriteString("X11Forwarding no\n")
|
||||
b.WriteString("Subsystem sftp internal-sftp\n")
|
||||
return b.String(), nil
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package felhomsshd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRenderConfig_SafeTemplateAndByteStable(t *testing.T) {
|
||||
c, err := renderConfig(8822)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, must := range []string{
|
||||
"Port 8822\n",
|
||||
"AuthorizedKeysFile /etc/felhom-sshd/authorized_keys/%u\n",
|
||||
"HostKey /etc/felhom-sshd/ssh_host_ed25519_key\n",
|
||||
"PasswordAuthentication no\n",
|
||||
"PermitRootLogin prohibit-password\n",
|
||||
"AllowUsers root felhom-op\n",
|
||||
"PidFile /run/felhom-sshd.pid\n",
|
||||
} {
|
||||
if !strings.Contains(c, must) {
|
||||
t.Errorf("config missing %q:\n%s", must, c)
|
||||
}
|
||||
}
|
||||
// [SF-1] the incident cause must NEVER appear.
|
||||
if strings.Contains(c, "RuntimeDirectory") {
|
||||
t.Fatal("config/unit must never mention RuntimeDirectory")
|
||||
}
|
||||
// byte-stable for a given port (conf-hash stability → no reload churn).
|
||||
c2, _ := renderConfig(8822)
|
||||
if c != c2 {
|
||||
t.Fatal("renderConfig not deterministic")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderConfig_RefusesPort22AndOutOfRange(t *testing.T) {
|
||||
if _, err := renderConfig(22); err == nil {
|
||||
t.Fatal("renderConfig(22) must be refused — never claim the stock/customer port")
|
||||
}
|
||||
if _, err := renderConfig(0); err == nil {
|
||||
t.Fatal("port 0 accepted")
|
||||
}
|
||||
if _, err := renderConfig(70000); err == nil {
|
||||
t.Fatal("out-of-range port accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// claim harness: a fake free-set + an in-memory persisted port.
|
||||
func claimHarness(free map[int]bool, persisted int) (portProbe, func() (int, bool), func(int) error, *int) {
|
||||
stored := persisted
|
||||
isFree := func(p int) bool { return free[p] }
|
||||
read := func() (int, bool) {
|
||||
if stored == 0 {
|
||||
return 0, false
|
||||
}
|
||||
return stored, true
|
||||
}
|
||||
write := func(p int) error { stored = p; return nil }
|
||||
return isFree, read, write, &stored
|
||||
}
|
||||
|
||||
func TestClaimPort_CleanContentionIdempotentExhaustion(t *testing.T) {
|
||||
cands := []int{8822, 2222, 8022, 62222}
|
||||
|
||||
// clean → first candidate
|
||||
isFree, read, write, stored := claimHarness(map[int]bool{8822: true, 2222: true, 8022: true, 62222: true}, 0)
|
||||
if p, err := claimPort(cands, isFree, read, write); err != nil || p != 8822 {
|
||||
t.Fatalf("clean claim = %d / %v, want 8822", p, err)
|
||||
}
|
||||
if *stored != 8822 {
|
||||
t.Fatalf("clean claim not persisted, stored=%d", *stored)
|
||||
}
|
||||
|
||||
// contention: 8822 busy → 2222
|
||||
isFree, read, write, _ = claimHarness(map[int]bool{8822: false, 2222: true, 8022: true, 62222: true}, 0)
|
||||
if p, err := claimPort(cands, isFree, read, write); err != nil || p != 2222 {
|
||||
t.Fatalf("contention claim = %d / %v, want 2222", p, err)
|
||||
}
|
||||
|
||||
// idempotent: a persisted port is KEPT unconditionally (even when isFree(cur) reports busy —
|
||||
// that's felhom-sshd itself holding it on a continuous reconcile; re-probing would flip-flop).
|
||||
isFree, read, write, _ = claimHarness(map[int]bool{8822: true, 2222: false, 8022: true, 62222: true}, 2222)
|
||||
if p, err := claimPort(cands, isFree, read, write); err != nil || p != 2222 {
|
||||
t.Fatalf("persisted port must be kept even when isFree says busy (own daemon), got %d / %v", p, err)
|
||||
}
|
||||
|
||||
// exhaustion: all busy → LOUD error, no fallback
|
||||
isFree, read, write, stored = claimHarness(map[int]bool{8822: false, 2222: false, 8022: false, 62222: false}, 0)
|
||||
p, err := claimPort(cands, isFree, read, write)
|
||||
if err != ErrPortsExhausted {
|
||||
t.Fatalf("exhaustion must return ErrPortsExhausted, got %d / %v", p, err)
|
||||
}
|
||||
if p == 22 || p != 0 {
|
||||
t.Fatalf("exhaustion must NOT yield a port (esp. :22), got %d", p)
|
||||
}
|
||||
if *stored != 0 {
|
||||
t.Fatalf("exhaustion must persist nothing, stored=%d", *stored)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package felhomsshd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// HealMarkerPath records the last felhom-sshd auto-heal (RFC3339), so the heartbeat surfaces a
|
||||
// recurring failure to the operator (the mgmt_plane pattern, scoped to the OOB daemon).
|
||||
const HealMarkerPath = "/run/felhom-sshd.healed"
|
||||
|
||||
// HealAndCheck restores felhom-sshd if it is down AND its config is valid — ONE restart per cooldown
|
||||
// [SF-5 / spike §6], so a persistently-broken instance is reported (Status), not restart-stormed. It
|
||||
// NEVER restarts onto a broken config [trap 8]: if `sshd -t` fails, it hands off (Status reports
|
||||
// config_invalid; the hub warns). A healthy/active instance is untouched.
|
||||
func (m *Manager) HealAndCheck(ctx context.Context, port int) {
|
||||
if m.isActive(ctx) {
|
||||
return // healthy — nothing to heal
|
||||
}
|
||||
// Down. Only restart if the config is VALID — never convert degraded into dead.
|
||||
if _, errOut, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil {
|
||||
m.logger.Warn("felhomsshd: down AND config invalid — NOT restarting (report-only)",
|
||||
"stderr", strings.TrimSpace(string(errOut)))
|
||||
return
|
||||
}
|
||||
// Cooldown: at most one deliberate restart per window.
|
||||
now := m.now()
|
||||
if !m.lastRestartAt.IsZero() && now.Sub(m.lastRestartAt) < restartCooldown {
|
||||
return
|
||||
}
|
||||
m.lastRestartAt = now
|
||||
// reset-failed BEFORE restart [SF-5]: a start-limit lockout otherwise refuses the restart.
|
||||
if m.isFailed(ctx) {
|
||||
_, _, _ = m.runner.Run(ctx, "systemctl", "reset-failed", Unit)
|
||||
}
|
||||
if err := m.systemctl(ctx, "restart"); err != nil {
|
||||
return
|
||||
}
|
||||
// record the heal (best-effort; a failed marker write never fails the heal)
|
||||
_ = os.WriteFile(HealMarkerPath, []byte(now.UTC().Format(time.RFC3339)+"\n"), 0o644)
|
||||
m.logger.Warn("felhomsshd: was down with a valid config — restarted (heal)", "port", port)
|
||||
}
|
||||
|
||||
// Status builds the OOB heartbeat stanza (Part 4). Read-only. Discovers the effective port(s) via
|
||||
// `sshd -T` (authoritative — catches a non-default/multi-Port config), dials the port locally to
|
||||
// prove reachability, reads the wg-felhom handshake age, and reflects operator-peer/key config from
|
||||
// the desired-state block. NEVER `wg show dump` (the S1 ban) — `latest-handshakes` only.
|
||||
func (m *Manager) Status(ctx context.Context, block *hub.WireWireguard) *hub.OOBStatus {
|
||||
st := &hub.OOBStatus{
|
||||
FelhomSshdActive: m.isActive(ctx),
|
||||
FelhomSshdPort: m.port,
|
||||
}
|
||||
// Authoritative port(s) from `sshd -T` (may differ from m.port if the config was hand-edited).
|
||||
if ports := m.sshdEffectivePorts(ctx); len(ports) > 0 {
|
||||
st.FelhomSshdPort = ports[0]
|
||||
}
|
||||
// config validity
|
||||
if _, _, err := m.runner.Run(ctx, "sshd", "-t", "-f", ConfPath); err != nil {
|
||||
st.ConfigInvalid = true
|
||||
}
|
||||
// Reachability = a LISTENER is bound on the OOB port (catches "active but crashed post-fork"). We
|
||||
// do NOT dial: the belt (correctly) drops even localhost→felhom-sshd (tunnel-only), so a local
|
||||
// dial always fails and would misreport a healthy daemon as unreachable.
|
||||
if st.FelhomSshdPort > 0 {
|
||||
st.Reachable = listenerPresent(st.FelhomSshdPort)
|
||||
}
|
||||
// wg-felhom handshake age (the OOB path rides the tunnel)
|
||||
if age, ok := m.wgHandshakeAge(ctx); ok {
|
||||
st.WGHandshakeAgeS = &age
|
||||
}
|
||||
// Operator-peer configured: from the in-memory block when fetched, OR (robust across an agent
|
||||
// restart, before the next desired-state fetch) from the PERSISTENT belt @operator_ips set — the
|
||||
// agent can read it via the sudo nft-list grant (unlike the 0600 root-owned wg-felhom.conf). This
|
||||
// keeps the report + the oob_degraded alert gate accurate immediately after a restart.
|
||||
st.OperatorPeerConfigured = m.beltOperatorConfigured(ctx)
|
||||
if block != nil {
|
||||
if block.OOBPeerIP != "" {
|
||||
st.OperatorPeerConfigured = true
|
||||
}
|
||||
st.OperatorKeyConfigured = strings.TrimSpace(block.OOBOperatorSSHKey) != ""
|
||||
}
|
||||
// Operator key: robust across restart via the installed authorized_keys file.
|
||||
if fi, err := os.Stat(AuthKeysUserPath); err == nil && fi.Size() > 0 {
|
||||
st.OperatorKeyConfigured = true
|
||||
}
|
||||
// last auto-heal
|
||||
if raw, err := os.ReadFile(HealMarkerPath); err == nil {
|
||||
st.HealedAt = strings.TrimSpace(string(raw))
|
||||
}
|
||||
return st
|
||||
}
|
||||
|
||||
// sshdEffectivePorts parses `sshd -T -f <conf>` for the effective Port line(s) (handles multi-Port).
|
||||
func (m *Manager) sshdEffectivePorts(ctx context.Context) []int {
|
||||
out, _, err := m.runner.Run(ctx, "sshd", "-T", "-f", ConfPath)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var ports []int
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
f := strings.Fields(strings.ToLower(line))
|
||||
if len(f) == 2 && f[0] == "port" {
|
||||
if p, err := strconv.Atoi(f[1]); err == nil && p > 0 && p <= 65535 {
|
||||
ports = append(ports, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ports
|
||||
}
|
||||
|
||||
// wgHandshakeAge reads wg-felhom's latest-handshake age in seconds (latest-handshakes ONLY — the
|
||||
// dump ban). ok=false when the tunnel is down or unreadable.
|
||||
func (m *Manager) wgHandshakeAge(ctx context.Context) (int64, bool) {
|
||||
out, _, err := m.runner.Run(ctx, "wg", "show", "wg-felhom", "latest-handshakes")
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) != 2 {
|
||||
continue
|
||||
}
|
||||
epoch, err := strconv.ParseInt(f[1], 10, 64)
|
||||
if err != nil || epoch <= 0 {
|
||||
continue
|
||||
}
|
||||
age := m.now().Unix() - epoch
|
||||
if age < 0 {
|
||||
age = 0
|
||||
}
|
||||
return age, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// listenerPresent reports whether something is LISTENing on the port (via ss — reads kernel state,
|
||||
// so the belt never blocks it, unlike a real dial).
|
||||
func listenerPresent(port int) bool {
|
||||
out, err := exec.Command("ss", "-Htln", "sport = :"+strconv.Itoa(port)).Output()
|
||||
return err == nil && strings.TrimSpace(string(out)) != ""
|
||||
}
|
||||
|
||||
// beltOperatorConfigured reports whether the belt's @operator_ips set is non-empty (an operator /32
|
||||
// is allowed to reach felhom-sshd). Read via the sudo nft-list grant — persistent + agent-readable.
|
||||
func (m *Manager) beltOperatorConfigured(ctx context.Context) bool {
|
||||
out, _, err := m.runner.Run(ctx, "nft", "list", "set", "inet", "felhom_oob", "operator_ips")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(string(out), "elements")
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package felhomsshd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// Loop drives the felhom-sshd Manager on its own cadence (the wgtunnel/lanresolver shape) and
|
||||
// consumes the hub desired-state's wireguard block via the desired.Syncer raw-consumer seam (for
|
||||
// oob_peer_ip → the belt, and oob_operator_ssh_key → authorized_keys). Each tick: claim/render/reload
|
||||
// the instance, sync the belt sets, and run the health/heal check.
|
||||
type Loop struct {
|
||||
mgr *Manager
|
||||
belt *Belt // Part 3 (nil-safe: no belt sync when unset)
|
||||
interval time.Duration
|
||||
logger *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
fetched bool // a desired-state document has been delivered (false = never a teardown signal)
|
||||
block *hub.WireWireguard
|
||||
|
||||
nudge chan struct{}
|
||||
}
|
||||
|
||||
// NewLoop builds the loop. interval defaults to 60s. belt may be nil (belt sync skipped).
|
||||
func NewLoop(mgr *Manager, belt *Belt, interval time.Duration, logger *slog.Logger) *Loop {
|
||||
if interval <= 0 {
|
||||
interval = 60 * time.Second
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Loop{mgr: mgr, belt: belt, interval: interval, logger: logger, nudge: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
// OnDesiredState implements desired.RawConsumer: store the latest wireguard block and nudge.
|
||||
func (l *Loop) OnDesiredState(_ context.Context, resp *hub.DesiredStateResponse) {
|
||||
if resp == nil {
|
||||
return
|
||||
}
|
||||
l.mu.Lock()
|
||||
l.fetched = true
|
||||
l.block = resp.DesiredState.Wireguard
|
||||
l.mu.Unlock()
|
||||
select {
|
||||
case l.nudge <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Loop) snapshot() (bool, *hub.WireWireguard) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.fetched, l.block
|
||||
}
|
||||
|
||||
// Run reconciles immediately, then on every tick or desired-state nudge, until ctx is cancelled.
|
||||
func (l *Loop) Run(ctx context.Context) error {
|
||||
l.reconcile(ctx)
|
||||
t := time.NewTicker(l.interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
case <-l.nudge:
|
||||
}
|
||||
l.reconcile(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcile runs one full pass: instance apply → belt sync → health/heal. The `fetched` gate mirrors
|
||||
// wgtunnel: until a desired-state document arrives, the operator inputs (authorized_keys, @operator_ips)
|
||||
// are LEFT UNTOUCHED — an unfetched block must never empty the belt (operator lockout) or wipe the key.
|
||||
func (l *Loop) reconcile(ctx context.Context) {
|
||||
fetched, block := l.snapshot()
|
||||
// Pass the block to Apply only when fetched, so authorized_keys is applied only from real desired
|
||||
// state (a nil block on Apply skips the authorized_keys write, leaving the existing file).
|
||||
applyBlock := block
|
||||
if !fetched {
|
||||
applyBlock = nil
|
||||
}
|
||||
port, err := l.mgr.Apply(ctx, applyBlock)
|
||||
if err != nil {
|
||||
return // Apply logged; a claim/exhaustion or install error — retry next tick
|
||||
}
|
||||
if l.belt != nil {
|
||||
l.belt.SyncPort(ctx, port) // always — the port is the agent's own claim
|
||||
if fetched {
|
||||
l.belt.SyncOperator(ctx, oobPeerIP(block)) // only from real desired state
|
||||
}
|
||||
}
|
||||
l.mgr.HealAndCheck(ctx, port)
|
||||
}
|
||||
|
||||
// oobPeerIP extracts the operator /32 source (bare IP) from the block, or "" when OOB is off.
|
||||
func oobPeerIP(block *hub.WireWireguard) string {
|
||||
if block == nil {
|
||||
return ""
|
||||
}
|
||||
return block.OOBPeerIP
|
||||
}
|
||||
|
||||
// OOBStatus implements the hub collector's reporter seam (Part 4).
|
||||
func (l *Loop) OOBStatus(ctx context.Context) *hub.OOBStatus {
|
||||
_, block := l.snapshot()
|
||||
return l.mgr.Status(ctx, block)
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package felhomsshd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// AuthKeysUserPath is felhom-op's authorized_keys (the default operator identity — key OUTSIDE
|
||||
// ~/.ssh so the customer's sshd never honours it [SF-3]). FIXED (sudoers install target).
|
||||
const AuthKeysUserPath = AuthKeysDir + "/" + OperatorUser
|
||||
|
||||
const stagedConfName = "sshd_config"
|
||||
|
||||
// Manager renders + applies the felhom-sshd config and drives the unit through the narrow-sudoers
|
||||
// runner. Config changes go write→`sshd -t`→`reload` (NEVER restart-on-change [SF-2]); a deliberate
|
||||
// restart is `reset-failed`-then-`restart` [SF-5], rate-limited by a cooldown (no flap).
|
||||
type Manager struct {
|
||||
runner proxmox.Runner
|
||||
stateDir string
|
||||
logger *slog.Logger
|
||||
|
||||
// injectable seams (tests)
|
||||
isActive func(ctx context.Context) bool
|
||||
isFailed func(ctx context.Context) bool
|
||||
isFree portProbe
|
||||
now func() time.Time
|
||||
|
||||
port int // the claimed port (0 until Apply claims it)
|
||||
hash string // sha256 of the last-applied config (steady-state zero-exec gate)
|
||||
akHash string // sha256 of the last-applied felhom-op authorized_keys
|
||||
lastRestartAt time.Time // heal cooldown (Part 4)
|
||||
}
|
||||
|
||||
// restartCooldown bounds deliberate felhom-sshd restarts (heal path) — one attempt per window, so a
|
||||
// persistently-broken instance is reported, not restart-stormed (spike §6 / notification-cooldown shape).
|
||||
const restartCooldown = 10 * time.Minute
|
||||
|
||||
// NewManager builds a Manager. stateDir is the agent state dir (staged config lives under it).
|
||||
func NewManager(runner proxmox.Runner, stateDir string, logger *slog.Logger) *Manager {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Manager{
|
||||
runner: runner,
|
||||
stateDir: stateDir,
|
||||
logger: logger,
|
||||
isActive: func(ctx context.Context) bool {
|
||||
out, _ := exec.CommandContext(ctx, "systemctl", "is-active", Unit).Output()
|
||||
return strings.TrimSpace(string(out)) == "active"
|
||||
},
|
||||
isFailed: func(ctx context.Context) bool {
|
||||
out, _ := exec.CommandContext(ctx, "systemctl", "is-failed", Unit).Output()
|
||||
return strings.TrimSpace(string(out)) == "failed"
|
||||
},
|
||||
isFree: probeFree,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// Port returns the claimed port (0 before the first successful Apply).
|
||||
func (m *Manager) Port() int { return m.port }
|
||||
|
||||
func (m *Manager) sshdDir() string { return filepath.Join(m.stateDir, "felhom-sshd") }
|
||||
func (m *Manager) stagedConfPath() string { return filepath.Join(m.sshdDir(), stagedConfName) }
|
||||
func (m *Manager) portFilePath() string { return filepath.Join(m.sshdDir(), "port") }
|
||||
|
||||
// readPort/writePort persist the claimed port in the AGENT-OWNED state dir (the agent is non-root and
|
||||
// cannot write the root-owned /etc/felhom-sshd). A port is not a secret.
|
||||
func (m *Manager) readPort() (int, bool) {
|
||||
raw, err := os.ReadFile(m.portFilePath())
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
p, err := strconv.Atoi(strings.TrimSpace(string(raw)))
|
||||
if err != nil || p < 1 || p > 65535 {
|
||||
return 0, false
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
|
||||
func (m *Manager) writePort(port int) error {
|
||||
if err := os.MkdirAll(m.sshdDir(), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(m.portFilePath(), []byte(strconv.Itoa(port)+"\n"), 0o600)
|
||||
}
|
||||
|
||||
// Apply claims the port, renders the config, reconciles the running unit, and (when the desired-state
|
||||
// block carries it) installs the operator's authorized_keys. Idempotent: no change → at most an
|
||||
// is-active check. block may be nil (no desired-state yet) — the instance still runs; only the
|
||||
// operator login/belt inputs are skipped. Returns the claimed port (0 on a claim/exhaustion error) —
|
||||
// the caller (belt sync) needs it.
|
||||
func (m *Manager) Apply(ctx context.Context, block *hub.WireWireguard) (int, error) {
|
||||
port, err := claimPort(Candidates, m.isFree, m.readPort, m.writePort)
|
||||
if err != nil {
|
||||
m.logger.Error("felhomsshd: port claim failed", "err", err)
|
||||
return 0, err
|
||||
}
|
||||
m.port = port
|
||||
|
||||
// Operator authorized_keys (from the hub-driven block) — written to felhom-op's dedicated file,
|
||||
// outside ~/.ssh [SF-3]. Independent of the config-reload path (a key change never reloads sshd).
|
||||
if block != nil {
|
||||
m.applyAuthorizedKeys(ctx, block.OOBOperatorSSHKey)
|
||||
}
|
||||
|
||||
conf, err := renderConfig(port)
|
||||
if err != nil {
|
||||
m.logger.Error("felhomsshd: refusing to apply invalid config", "err", err)
|
||||
return port, err
|
||||
}
|
||||
sum := sha256.Sum256([]byte(conf))
|
||||
hash := hex.EncodeToString(sum[:])
|
||||
active := m.isActive(ctx)
|
||||
|
||||
if m.hash == hash && active {
|
||||
return port, nil // steady state: zero execs
|
||||
}
|
||||
|
||||
// Stage → validate → install → reload/enable. Never restart on a config change [SF-2].
|
||||
if err := os.MkdirAll(m.sshdDir(), 0o700); err != nil {
|
||||
m.logger.Error("felhomsshd: state dir", "err", err)
|
||||
return port, err
|
||||
}
|
||||
if err := os.WriteFile(m.stagedConfPath(), []byte(conf), 0o600); err != nil {
|
||||
m.logger.Error("felhomsshd: staging config", "err", err)
|
||||
return port, err
|
||||
}
|
||||
// Validate the STAGED config before it is installed — a bad render never reaches the live path.
|
||||
if _, errOut, err := m.runner.Run(ctx, "sshd", "-t", "-f", m.stagedConfPath()); err != nil {
|
||||
m.logger.Error("felhomsshd: staged config failed sshd -t — NOT installing", "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||
return port, err
|
||||
}
|
||||
if _, errOut, err := m.runner.Run(ctx, "install", "-o", "root", "-g", "root", "-m", "0644", "--", m.stagedConfPath(), ConfPath); err != nil {
|
||||
m.logger.Error("felhomsshd: config install failed", "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||
return port, err
|
||||
}
|
||||
verb := "reload"
|
||||
if !active {
|
||||
verb = "enable" // first apply / down → enable --now brings it up
|
||||
}
|
||||
if err := m.systemctl(ctx, verb); err != nil {
|
||||
return port, err
|
||||
}
|
||||
m.hash = hash
|
||||
m.logger.Info("felhomsshd: config applied", "port", port, "action", verb)
|
||||
return port, nil
|
||||
}
|
||||
|
||||
// applyAuthorizedKeys installs (or clears) felhom-op's authorized_keys from the hub-delivered
|
||||
// operator SSH key. Hash-gated (no churn); the key is public, never a secret. A write failure is
|
||||
// logged, not fatal — the instance keeps running. NO sshd reload needed (sshd reads the file per-auth).
|
||||
func (m *Manager) applyAuthorizedKeys(ctx context.Context, sshKey string) {
|
||||
content := ""
|
||||
if k := strings.TrimSpace(sshKey); k != "" {
|
||||
content = k + "\n"
|
||||
}
|
||||
sum := sha256.Sum256([]byte(content))
|
||||
h := hex.EncodeToString(sum[:])
|
||||
if h == m.akHash {
|
||||
return // unchanged
|
||||
}
|
||||
staged := filepath.Join(m.sshdDir(), "authorized_keys."+OperatorUser)
|
||||
if err := os.MkdirAll(m.sshdDir(), 0o700); err != nil {
|
||||
m.logger.Error("felhomsshd: state dir for authorized_keys", "err", err)
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile(staged, []byte(content), 0o600); err != nil {
|
||||
m.logger.Error("felhomsshd: staging authorized_keys", "err", err)
|
||||
return
|
||||
}
|
||||
if _, errOut, err := m.runner.Run(ctx, "install", "-o", "root", "-g", "root", "-m", "0644", "--", staged, AuthKeysUserPath); err != nil {
|
||||
m.logger.Error("felhomsshd: authorized_keys install failed", "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||
return
|
||||
}
|
||||
m.akHash = h
|
||||
m.logger.Info("felhomsshd: operator authorized_keys updated", "user", OperatorUser, "present", content != "")
|
||||
}
|
||||
|
||||
// systemctl runs the reconcile verbs through the narrow runner. `enable` = `enable --now`; `reload`
|
||||
// HUPs (config change, running instance survives a bad reload via the unit's ExecReload sshd -t gate).
|
||||
func (m *Manager) systemctl(ctx context.Context, verb string) error {
|
||||
var args []string
|
||||
switch verb {
|
||||
case "enable":
|
||||
args = []string{"enable", "--now", Unit}
|
||||
case "reload", "restart":
|
||||
args = []string{verb, Unit}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
if _, errOut, err := m.runner.Run(ctx, "systemctl", args...); err != nil {
|
||||
m.logger.Error("felhomsshd: systemctl "+verb+" failed", "err", err, "stderr", strings.TrimSpace(string(errOut)))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeFree is the production port probe: nothing LISTENing (ss) AND a real bind succeeds (a bind
|
||||
// that succeeds-then-closes proves the port is actually claimable, not just ss-silent).
|
||||
func probeFree(port int) bool {
|
||||
p := strconv.Itoa(port)
|
||||
out, err := exec.Command("ss", "-Htln", "sport = :"+p).Output()
|
||||
if err == nil && strings.TrimSpace(string(out)) != "" {
|
||||
return false // something is listening
|
||||
}
|
||||
ln, err := net.Listen("tcp", "0.0.0.0:"+p)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_ = ln.Close()
|
||||
return true
|
||||
}
|
||||
@@ -30,6 +30,10 @@ import (
|
||||
// PhasePreStart is the PVE hook phase at which we self-heal (before the container mounts are set up).
|
||||
const PhasePreStart = "pre-start"
|
||||
|
||||
// PhasePostStart is the PVE hook phase after the container started — the NAS automount reassert
|
||||
// point (the fresh guest namespace has no idle autofs triggers; see netreassert.go).
|
||||
const PhasePostStart = "post-start"
|
||||
|
||||
// placeholderMode is the mode for a created bind-source placeholder. Host-root-owned + this mode =
|
||||
// fail-closed against the unprivileged guest (host uid 0 is unmapped in the guest userns).
|
||||
const placeholderMode = 0o755
|
||||
|
||||
@@ -28,9 +28,14 @@ var SnippetPath = filepath.Join(SnippetDir, SnippetName)
|
||||
|
||||
// snippetBody is the tiny wrapper PVE execs as `<script> <vmid> <phase>`. It delegates to the agent
|
||||
// binary so the heal LOGIC is the unit-tested Go, never duplicated (divergence-proof) shell. Executable.
|
||||
// The wrapper NEVER exec's and ALWAYS exits 0 (CAMPAIGN-3 F10/rc255 belt): a hook that exits nonzero
|
||||
// aborts the guest start. `exec` would surface the binary's exit code to PVE; instead we run it as a
|
||||
// child, swallow any nonzero (missing/crashed binary, OOM-kill), and `exit 0` unconditionally. The Go
|
||||
// side has its own recover + per-phase timeout — this is the second belt at the shell layer.
|
||||
const snippetBody = `#!/bin/sh
|
||||
# felhom-agent guest pre-start self-heal hook (C1 net). PVE calls: <script> <vmid> <phase>.
|
||||
exec ` + AgentBin + ` guest-hook "$1" "$2"
|
||||
` + AgentBin + ` guest-hook "$1" "$2" || true
|
||||
exit 0
|
||||
`
|
||||
|
||||
// InstallSnippet writes the pre-start hook wrapper into the PVE snippets dir (idempotent, root-owned,
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package guesthook
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
|
||||
// post-start network-storage reassert (RCA AUDIT-nas-cwa-rca-2026-07-11 fix 1, hook leg).
|
||||
//
|
||||
// A freshly started guest's namespace does NOT inherit an idle NAS autofs trigger (only real
|
||||
// mounts), so its /mnt/felhom-drives/<share> path is a silent local stub until the trigger is
|
||||
// re-created host-side. PVE runs the hookscript as root in the start task, so this leg calls
|
||||
// systemctl DIRECTLY (no sudo) — the daemon leg (localapi.ReassertNetworkMounts) is the sudo path.
|
||||
// Like the pre-start heal, this must NEVER fail the hook: all errors go to stderr (the PVE task
|
||||
// log) and the guest start proceeds regardless.
|
||||
|
||||
// netReasserter is the reassert capability (satisfied by *storage.SudoHostOps; faked in tests).
|
||||
type netReasserter interface {
|
||||
ReassertNetworkAutomounts(ctx context.Context) []storage.NetReassertResult
|
||||
RearmNetworkAutomount(ctx context.Context, where string) error
|
||||
}
|
||||
|
||||
// PostStartNetworkReassert re-arms idle NAS automount triggers after vmid started, then verifies
|
||||
// the (now running) guest actually sees each share path. Best-effort throughout.
|
||||
func PostStartNetworkReassert(ctx context.Context, vmid string) {
|
||||
runner := &proxmox.ExecRunner{Mode: proxmox.RunnerDirect}
|
||||
ops := storage.NewSudoHostOps(storage.SudoHostOpsConfig{
|
||||
Runner: runner,
|
||||
Logger: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})),
|
||||
})
|
||||
postStartNetworkReassert(ctx, vmid, ops, func(ctx context.Context, vmid, path string) bool {
|
||||
return GuestSeesPath(ctx, runner, vmid, path)
|
||||
})
|
||||
}
|
||||
|
||||
// postStartNetworkReassert is the seam-injected core (unit-tested; the wrapper above binds the
|
||||
// real host surface).
|
||||
func postStartNetworkReassert(ctx context.Context, vmid string, ops netReasserter, sees func(ctx context.Context, vmid, path string) bool) {
|
||||
for _, res := range ops.ReassertNetworkAutomounts(ctx) {
|
||||
if !res.Remediates() {
|
||||
continue // foreign/errored rows expect nothing in the guest (already logged by the ops layer)
|
||||
}
|
||||
if sees(ctx, vmid, res.Where) {
|
||||
fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s post-start — network share %s visible in guest (%s)\n",
|
||||
vmid, res.Name, res.Action)
|
||||
continue
|
||||
}
|
||||
// F11 matrix-correction: the guest is blind even though the pass classified this share
|
||||
// skip-active/rearmed. A freshly-started namespace does NOT inherit an ACTIVE host mount — only
|
||||
// a FRESH trigger event propagates in. Re-arm this specific share and re-verify once.
|
||||
fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s post-start — network share %s not visible after reassert (%s) — re-arming\n",
|
||||
vmid, res.Name, res.Action)
|
||||
if err := ops.RearmNetworkAutomount(ctx, res.Where); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s post-start — WARNING: re-arm of %s failed: %v\n", vmid, res.Name, err)
|
||||
continue
|
||||
}
|
||||
if sees(ctx, vmid, res.Where) {
|
||||
fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s post-start — network share %s visible in guest after re-arm (healed)\n", vmid, res.Name)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s post-start — WARNING: network share %s STILL NOT visible after re-arm\n", vmid, res.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GuestSeesPath reports whether vmid's guest has `path` as a mount target in its own namespace —
|
||||
// the hook-process mirror of localapi's GuestBinder.GuestSeesMount (which is method-bound to the
|
||||
// daemon's binder and unavailable here). Resolution/read errors → false.
|
||||
func GuestSeesPath(ctx context.Context, runner proxmox.Runner, vmid, path string) bool {
|
||||
out, _, err := runner.Run(ctx, "lxc-info", "-n", vmid, "-p", "-H")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
pid := strings.TrimSpace(string(out))
|
||||
if pid == "" {
|
||||
return false
|
||||
}
|
||||
data, err := os.ReadFile("/proc/" + pid + "/mountinfo")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) >= 5 && f[4] == path {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package guesthook
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
|
||||
type fakeReasserter struct {
|
||||
invoked int
|
||||
rearmed []string
|
||||
rearmErr error
|
||||
results []storage.NetReassertResult
|
||||
}
|
||||
|
||||
func (f *fakeReasserter) ReassertNetworkAutomounts(context.Context) []storage.NetReassertResult {
|
||||
f.invoked++
|
||||
return f.results
|
||||
}
|
||||
|
||||
func (f *fakeReasserter) RearmNetworkAutomount(_ context.Context, where string) error {
|
||||
f.rearmed = append(f.rearmed, where)
|
||||
return f.rearmErr
|
||||
}
|
||||
|
||||
// The post-start core must run the reassert pass and verify guest visibility for every share the
|
||||
// pass acted on (or found actively mounted) — and never for foreign/errored rows.
|
||||
func TestPostStartNetworkReassert_Core(t *testing.T) {
|
||||
ops := &fakeReasserter{results: []storage.NetReassertResult{
|
||||
{Name: "media", Where: "/mnt/felhom-drives/media", Action: storage.NetReassertRearmed},
|
||||
{Name: "active", Where: "/mnt/felhom-drives/active", Action: storage.NetReassertSkipActive},
|
||||
{Name: "foreign", Where: "/mnt/felhom-drives/foreign", Action: storage.NetReassertSkipForeign},
|
||||
}}
|
||||
var verified []string
|
||||
postStartNetworkReassert(context.Background(), "9201", ops, func(_ context.Context, vmid, path string) bool {
|
||||
if vmid != "9201" {
|
||||
t.Errorf("verify called with vmid %q, want 9201", vmid)
|
||||
}
|
||||
verified = append(verified, path)
|
||||
return true
|
||||
})
|
||||
if ops.invoked != 1 {
|
||||
t.Fatalf("reassert pass invoked %d times, want 1", ops.invoked)
|
||||
}
|
||||
if len(verified) != 2 || verified[0] != "/mnt/felhom-drives/media" || verified[1] != "/mnt/felhom-drives/active" {
|
||||
t.Fatalf("verify must cover rearmed + skip-active only, got %v", verified)
|
||||
}
|
||||
}
|
||||
|
||||
// F11 matrix-correction: a guest that does NOT see a share (even a skip-active one) must trigger a
|
||||
// re-arm of that specific share, then a re-verify. Encodes the live 2026-07-12 finding that a rebooted
|
||||
// guest inherits neither an idle trigger nor an active mount.
|
||||
func TestPostStartNetworkReassert_RearmsWhenGuestBlind(t *testing.T) {
|
||||
ops := &fakeReasserter{results: []storage.NetReassertResult{
|
||||
{Name: "media", Where: "/mnt/felhom-drives/media", Action: storage.NetReassertSkipActive},
|
||||
}}
|
||||
// sees: false the first time (blind), true the second (post-rearm) — the heal path.
|
||||
calls := 0
|
||||
postStartNetworkReassert(context.Background(), "9201", ops, func(context.Context, string, string) bool {
|
||||
calls++
|
||||
return calls >= 2
|
||||
})
|
||||
if len(ops.rearmed) != 1 || ops.rearmed[0] != "/mnt/felhom-drives/media" {
|
||||
t.Fatalf("a blind guest must trigger exactly one re-arm of the share, got %v", ops.rearmed)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed verify must be non-fatal even if the re-arm also fails: the core returns normally (hook
|
||||
// exits 0 regardless).
|
||||
func TestPostStartNetworkReassert_VerifyFailureNonFatal(t *testing.T) {
|
||||
ops := &fakeReasserter{
|
||||
rearmErr: context.DeadlineExceeded,
|
||||
results: []storage.NetReassertResult{
|
||||
{Name: "media", Where: "/mnt/felhom-drives/media", Action: storage.NetReassertRearmed},
|
||||
},
|
||||
}
|
||||
// Must not panic or abort; the WARNING goes to stderr (PVE task log). Guest never sees it, re-arm errors.
|
||||
postStartNetworkReassert(context.Background(), "9201", ops, func(context.Context, string, string) bool {
|
||||
return false
|
||||
})
|
||||
if ops.invoked != 1 {
|
||||
t.Fatalf("reassert pass invoked %d times, want 1", ops.invoked)
|
||||
}
|
||||
if len(ops.rearmed) != 1 {
|
||||
t.Fatalf("a blind guest must attempt one re-arm, got %d", len(ops.rearmed))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
// Package guestnet implements R-54: the host-tier watchdog for each customer guest's own network.
|
||||
//
|
||||
// Origin — INCIDENT-guest-dhclient-killed-2026-07-20 §5 "OPEN RISK". The guest's DHCP client is
|
||||
// started once by ifupdown at boot and NOTHING supervises it. When it was killed on 2026-07-20 the
|
||||
// guest kept working for another ~80 minutes on its unexpired lease; only when the lease expired did
|
||||
// the address and default route vanish, taking the Cloudflare tunnel, the hub reports, the catalog
|
||||
// sync and the controller→agent channel with them. Total outage ~1h15m, and for the first 80 minutes
|
||||
// every observable signal said healthy.
|
||||
//
|
||||
// The design consequence is the whole point of this package: **liveness of the DHCP client process
|
||||
// is itself a probe**, not a detail. Waiting for the IP to disappear is waiting out the exact silent
|
||||
// window the incident proved exists. See TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy.
|
||||
//
|
||||
// The agent is the right tier for this: it lives on the host, keeps its own line to the hub, and can
|
||||
// still see and repair a guest that has gone completely mute. The controller cannot fix its own
|
||||
// missing default route.
|
||||
package guestnet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Runner is the privileged-exec seam (satisfied by *proxmox.ExecRunner). Declared consumer-side so
|
||||
// tests inject a scripted runner and no unit test goes near pct.
|
||||
type Runner interface {
|
||||
Run(ctx context.Context, name string, args ...string) (stdout, stderr []byte, err error)
|
||||
}
|
||||
|
||||
// Mode is how the guest is configured to get its address.
|
||||
type Mode string
|
||||
|
||||
const (
|
||||
ModeDHCP Mode = "dhcp"
|
||||
ModeStatic Mode = "static"
|
||||
ModeUnknown Mode = "unknown"
|
||||
)
|
||||
|
||||
// State is a guest's verdict for one cycle.
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateHealthy State = "healthy"
|
||||
// StateUnhealthy: DHCP-configured and something is wrong that dhclient can fix.
|
||||
StateUnhealthy State = "unhealthy"
|
||||
// StateStaticFault: a static guest missing its address/route. Reported loudly, NEVER healed —
|
||||
// re-running dhclient on a statically-configured guest would fight its own configuration, and
|
||||
// the durable answer is R-50 (island bridge), not a point fix here.
|
||||
StateStaticFault State = "static_fault"
|
||||
// StateUnknown: the guest could not be probed at all (pct exec failed, an interface file we
|
||||
// cannot read, a probe tool missing). Never healed — acting blind is how the incident happened.
|
||||
StateUnknown State = "unknown"
|
||||
)
|
||||
|
||||
// Probe is one guest's observed network facts.
|
||||
type Probe struct {
|
||||
VMID int
|
||||
Mode Mode
|
||||
IP string // empty when absent
|
||||
HasRoute bool
|
||||
DHCPAlive bool
|
||||
Reachable bool // pct exec worked at all
|
||||
Detail string // human-readable reason, operator-tier English
|
||||
}
|
||||
|
||||
// eth0 is the guest interface every felhom guest uses (the LXC veth peer inside the guest).
|
||||
const eth0 = "eth0"
|
||||
|
||||
// probe runs the four fixed-shape reads. Every argv is a constant plus the vmid — no guest-supplied
|
||||
// data is ever interpolated into a command, and there is no shell anywhere in this path.
|
||||
func (w *Watchdog) probe(ctx context.Context, vmid int) Probe {
|
||||
p := Probe{VMID: vmid, Mode: ModeUnknown}
|
||||
id := itoa(vmid)
|
||||
|
||||
// 1. Address. This also settles reachability: if pct exec cannot run here, nothing else is
|
||||
// worth attempting.
|
||||
out, errOut, err := w.runner.Run(ctx, "pct", "exec", id, "--", "ip", "-4", "-o", "addr", "show", "dev", eth0)
|
||||
if err != nil {
|
||||
p.Detail = "address probe failed: " + firstLine(string(errOut))
|
||||
return p // Reachable stays false → StateUnknown
|
||||
}
|
||||
p.Reachable = true
|
||||
p.IP = parseInet(string(out))
|
||||
|
||||
// 2. Default route.
|
||||
out, _, err = w.runner.Run(ctx, "pct", "exec", id, "--", "ip", "route", "show", "default")
|
||||
if err == nil {
|
||||
p.HasRoute = hasDefaultRoute(string(out))
|
||||
}
|
||||
|
||||
// 3. Configured mode. An unreadable interfaces file leaves ModeUnknown, which never heals.
|
||||
out, _, err = w.runner.Run(ctx, "pct", "exec", id, "--", "cat", "/etc/network/interfaces")
|
||||
if err == nil {
|
||||
p.Mode = parseMode(string(out), eth0)
|
||||
}
|
||||
|
||||
// 4. DHCP client liveness — the probe the incident was invisible to.
|
||||
// pgrep exits 1 with EMPTY stderr when there is no match; anything on stderr means the probe
|
||||
// itself failed (pgrep absent, guest wedged), which must read as unknown rather than as a
|
||||
// dead client, or a missing tool would trigger heals forever.
|
||||
out, errOut, err = w.runner.Run(ctx, "pct", "exec", id, "--", "pgrep", "-x", "dhclient")
|
||||
switch {
|
||||
case err == nil && strings.TrimSpace(string(out)) != "":
|
||||
p.DHCPAlive = true
|
||||
case err != nil && strings.TrimSpace(string(errOut)) != "":
|
||||
p.Reachable = false
|
||||
p.Detail = "dhclient liveness probe failed: " + firstLine(string(errOut))
|
||||
default:
|
||||
p.DHCPAlive = false
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// classify turns observed facts into the verdict. Pure — table-tested.
|
||||
func classify(p Probe) (State, string) {
|
||||
if !p.Reachable {
|
||||
d := p.Detail
|
||||
if d == "" {
|
||||
d = "guest not reachable via pct exec"
|
||||
}
|
||||
return StateUnknown, d
|
||||
}
|
||||
switch p.Mode {
|
||||
case ModeStatic:
|
||||
if p.IP != "" && p.HasRoute {
|
||||
return StateHealthy, "static address and default route present"
|
||||
}
|
||||
return StateStaticFault, "statically configured guest is missing its address or default route — reported only; dhclient must never be run against a static configuration (R-50 owns the durable fix)"
|
||||
case ModeDHCP:
|
||||
switch {
|
||||
case p.IP == "":
|
||||
return StateUnhealthy, "no IPv4 address on " + eth0
|
||||
case !p.HasRoute:
|
||||
return StateUnhealthy, "no default route"
|
||||
case !p.DHCPAlive:
|
||||
// THE incident state: address and route still present on an unexpired lease, with
|
||||
// nothing left to renew them. Damage is ~1-2 h in the future and invisible today.
|
||||
return StateUnhealthy, "dhclient is not running — the lease will not be renewed (the 2026-07-20 failure mode; address still present, renewal already dead)"
|
||||
default:
|
||||
return StateHealthy, "address, default route and dhclient all present"
|
||||
}
|
||||
default:
|
||||
return StateUnknown, "interface configuration mode could not be determined — not healing"
|
||||
}
|
||||
}
|
||||
|
||||
// --- parsing (fixtures captured live from guest 9201 on 2026-07-21, probe P3) -------------------
|
||||
|
||||
// parseInet extracts the address from `ip -4 -o addr show dev eth0` output, e.g.
|
||||
//
|
||||
// 2: eth0 inet 192.168.0.104/24 brd 192.168.0.255 scope global dynamic eth0\ valid_lft ...
|
||||
//
|
||||
// Returns "" when there is no inet line at all (the post-lease-expiry state: the command succeeds
|
||||
// and prints NOTHING).
|
||||
func parseInet(out string) string {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
for i, f := range fields {
|
||||
if f == "inet" && i+1 < len(fields) {
|
||||
addr := fields[i+1]
|
||||
if idx := strings.IndexByte(addr, '/'); idx > 0 {
|
||||
addr = addr[:idx]
|
||||
}
|
||||
return addr
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// hasDefaultRoute parses `ip route show default`, e.g. "default via 192.168.0.1 dev eth0 ".
|
||||
// Empty output = no default route (the incident state).
|
||||
func hasDefaultRoute(out string) bool {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if strings.HasPrefix(strings.TrimSpace(line), "default ") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseMode reads the iface stanza for dev out of /etc/network/interfaces:
|
||||
//
|
||||
// auto eth0
|
||||
// iface eth0 inet dhcp
|
||||
//
|
||||
// Anything else (no stanza, a manual/loopback mode, a file we could not read) is ModeUnknown, and
|
||||
// unknown never heals.
|
||||
func parseMode(out, dev string) Mode {
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
f := strings.Fields(strings.TrimSpace(line))
|
||||
// iface <dev> inet <mode>
|
||||
if len(f) >= 4 && f[0] == "iface" && f[1] == dev && f[2] == "inet" {
|
||||
switch f[3] {
|
||||
case "dhcp":
|
||||
return ModeDHCP
|
||||
case "static":
|
||||
return ModeStatic
|
||||
default:
|
||||
return ModeUnknown
|
||||
}
|
||||
}
|
||||
}
|
||||
return ModeUnknown
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
if len(s) > 200 {
|
||||
s = s[:200]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// itoa avoids pulling strconv into every call site's readability.
|
||||
func itoa(i int) string {
|
||||
if i == 0 {
|
||||
return "0"
|
||||
}
|
||||
neg := i < 0
|
||||
if neg {
|
||||
i = -i
|
||||
}
|
||||
var b [20]byte
|
||||
pos := len(b)
|
||||
for i > 0 {
|
||||
pos--
|
||||
b[pos] = byte('0' + i%10)
|
||||
i /= 10
|
||||
}
|
||||
if neg {
|
||||
pos--
|
||||
b[pos] = '-'
|
||||
}
|
||||
return string(b[pos:])
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package guestnet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// GuestNetStatus implements hub.GuestNetReporter: the heartbeat stanza built from the last sweep.
|
||||
// Pure read of already-collected state — it never probes, so a hub report can never trigger a pct
|
||||
// exec storm.
|
||||
//
|
||||
// It returns a stanza even when no guest has been probed yet (empty guests + a checked_at), because
|
||||
// "the watchdog is running and has nothing to say" must be distinguishable on the hub from "the
|
||||
// watchdog is not wired", which is the shape the v0.91.0 inert seam hid behind.
|
||||
func (w *Watchdog) GuestNetStatus(context.Context) *hub.GuestNetStatus {
|
||||
snap := w.Snapshot()
|
||||
out := &hub.GuestNetStatus{CheckedAt: w.now().UTC().Format(time.RFC3339)}
|
||||
for _, g := range snap {
|
||||
out.Guests = append(out.Guests, hub.GuestNetGuest{
|
||||
VMID: g.VMID,
|
||||
State: g.State,
|
||||
Mode: g.Mode,
|
||||
IP: g.IP,
|
||||
HasRoute: g.HasRoute,
|
||||
DHClientAlive: g.DHClientAlive,
|
||||
CheckedAt: g.CheckedAt,
|
||||
Healed: g.Healed,
|
||||
HealSucceeded: g.HealSucceeded,
|
||||
LastHealAt: g.LastHealAt,
|
||||
HealsLastHour: g.HealsLastHour,
|
||||
Damped: g.Damped,
|
||||
Message: g.Message,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
package guestnet
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// GuestSource yields the guests this agent OWNS. Production passes the pool-verified source
|
||||
// (ListLXC ∩ pool membership, audit A1) — never a bare ListLXC, which under a broad token would let
|
||||
// the watchdog run dhclient inside a co-tenant's container.
|
||||
type GuestSource interface {
|
||||
Guests(ctx context.Context) ([]proxmox.Guest, error)
|
||||
}
|
||||
|
||||
// Defaults. Every one of these is a damper: this watchdog runs a privileged command inside a
|
||||
// customer's container, so it is designed to under-act.
|
||||
const (
|
||||
DefaultInterval = 60 * time.Second
|
||||
// DefaultMinHealInterval is the per-guest cool-off between heals.
|
||||
DefaultMinHealInterval = 10 * time.Minute
|
||||
// DefaultMaxHealsPerHour caps a guest's heals; beyond it the watchdog only reports, because a
|
||||
// guest needing a fourth heal in an hour has a problem dhclient cannot fix.
|
||||
DefaultMaxHealsPerHour = 3
|
||||
// DefaultSettle is the boot-race guard, applied to BOTH the guest's uptime and the agent's own.
|
||||
// A guest that booted 40 s ago legitimately has no lease yet.
|
||||
DefaultSettle = 3 * time.Minute
|
||||
// requiredBadProbes: two CONSECUTIVE bad cycles before any heal. One blip is not a diagnosis.
|
||||
requiredBadProbes = 2
|
||||
)
|
||||
|
||||
// Watchdog probes each owned, running guest's network every interval and heals a DHCP guest whose
|
||||
// client has died. It never returns an error: a guest-network fault is a reported fact, not an agent
|
||||
// failure.
|
||||
type Watchdog struct {
|
||||
runner Runner
|
||||
guests GuestSource
|
||||
logger *slog.Logger
|
||||
|
||||
interval time.Duration
|
||||
minHealInterval time.Duration
|
||||
maxHealsPerHour int
|
||||
settle time.Duration
|
||||
|
||||
now func() time.Time
|
||||
startedAt time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
state map[int]*guestState
|
||||
}
|
||||
|
||||
type guestState struct {
|
||||
badProbes int
|
||||
heals []time.Time // heal timestamps, pruned to the last hour
|
||||
lastHealAt time.Time
|
||||
lastState State
|
||||
report GuestReport
|
||||
}
|
||||
|
||||
// New builds a Watchdog with the shipped dampers. interval <= 0 uses DefaultInterval.
|
||||
func New(runner Runner, guests GuestSource, interval time.Duration, logger *slog.Logger) *Watchdog {
|
||||
if interval <= 0 {
|
||||
interval = DefaultInterval
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
w := &Watchdog{
|
||||
runner: runner,
|
||||
guests: guests,
|
||||
logger: logger,
|
||||
interval: interval,
|
||||
minHealInterval: DefaultMinHealInterval,
|
||||
maxHealsPerHour: DefaultMaxHealsPerHour,
|
||||
settle: DefaultSettle,
|
||||
now: time.Now,
|
||||
state: map[int]*guestState{},
|
||||
}
|
||||
w.startedAt = w.now()
|
||||
return w
|
||||
}
|
||||
|
||||
// SetDampers overrides the three rate limits from config. Non-positive values keep the default,
|
||||
// the same "0 = package default" convention the storage watchdog and wg loop use.
|
||||
func (w *Watchdog) SetDampers(minHealInterval time.Duration, maxHealsPerHour int, settle time.Duration) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if minHealInterval > 0 {
|
||||
w.minHealInterval = minHealInterval
|
||||
}
|
||||
if maxHealsPerHour > 0 {
|
||||
w.maxHealsPerHour = maxHealsPerHour
|
||||
}
|
||||
if settle > 0 {
|
||||
w.settle = settle
|
||||
}
|
||||
}
|
||||
|
||||
// Watch runs until ctx is cancelled. Started with `go wd.Watch(ctx)` — deliberately not part of the
|
||||
// errc fan-out, because a guest-network watchdog must never be able to bring the agent down.
|
||||
func (w *Watchdog) Watch(ctx context.Context) {
|
||||
w.logger.Info("guestnet: watchdog starting",
|
||||
"interval", w.interval, "min_heal_interval", w.minHealInterval,
|
||||
"max_heals_per_hour", w.maxHealsPerHour, "settle", w.settle)
|
||||
t := time.NewTicker(w.interval)
|
||||
defer t.Stop()
|
||||
w.Tick(ctx) // immediate baseline
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.logger.Info("guestnet: watchdog shutting down", "reason", ctx.Err())
|
||||
return
|
||||
case <-t.C:
|
||||
w.Tick(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tick performs one full sweep. Exported so the wiring test and the live STOP leg can drive exactly
|
||||
// one cycle instead of waiting on a ticker.
|
||||
func (w *Watchdog) Tick(ctx context.Context) {
|
||||
guests, err := w.guests.Guests(ctx)
|
||||
if err != nil {
|
||||
// Unknown ownership ⇒ do nothing. Never fall back to an unfiltered guest list.
|
||||
w.logger.Warn("guestnet: guest list unavailable — skipping sweep (ownership unproven)", "err", err)
|
||||
return
|
||||
}
|
||||
for _, g := range guests {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if g.Status != "running" {
|
||||
w.forget(g.VMID)
|
||||
continue
|
||||
}
|
||||
w.checkGuest(ctx, g)
|
||||
}
|
||||
}
|
||||
|
||||
// forget drops state for a guest that is no longer running, so a stopped-and-restarted guest starts
|
||||
// from a clean slate rather than inheriting a stale bad-probe count.
|
||||
func (w *Watchdog) forget(vmid int) {
|
||||
w.mu.Lock()
|
||||
delete(w.state, vmid)
|
||||
w.mu.Unlock()
|
||||
}
|
||||
|
||||
func (w *Watchdog) checkGuest(ctx context.Context, g proxmox.Guest) {
|
||||
now := w.now()
|
||||
p := w.probe(ctx, g.VMID)
|
||||
state, detail := classify(p)
|
||||
|
||||
w.mu.Lock()
|
||||
st := w.state[g.VMID]
|
||||
if st == nil {
|
||||
st = &guestState{}
|
||||
w.state[g.VMID] = st
|
||||
}
|
||||
prev := st.lastState
|
||||
st.lastState = state
|
||||
st.pruneHeals(now)
|
||||
rep := GuestReport{
|
||||
VMID: g.VMID, Mode: string(p.Mode), IP: p.IP, HasRoute: p.HasRoute,
|
||||
DHClientAlive: p.DHCPAlive, State: string(state), Message: detail,
|
||||
HealsLastHour: len(st.heals), CheckedAt: now.UTC().Format(time.RFC3339),
|
||||
}
|
||||
if !st.lastHealAt.IsZero() {
|
||||
rep.LastHealAt = st.lastHealAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
switch state {
|
||||
case StateHealthy:
|
||||
st.badProbes = 0
|
||||
w.mu.Unlock()
|
||||
// The healthy path MUST be observable. v0.91.2's lesson, learned the hard way one day
|
||||
// earlier: if a healthy cycle logs nothing, "no alarms" and "never probed" are the same
|
||||
// line of evidence, and an inert watchdog is indistinguishable from a working one.
|
||||
w.logger.Debug("guestnet: guest network healthy", "vmid", g.VMID, "mode", string(p.Mode),
|
||||
"has_route", p.HasRoute, "dhclient_alive", p.DHCPAlive)
|
||||
if prev != "" && prev != StateHealthy {
|
||||
w.logger.Info("guestnet: guest network recovered", "vmid", g.VMID, "previous_state", string(prev))
|
||||
}
|
||||
w.record(g.VMID, rep)
|
||||
return
|
||||
|
||||
case StateUnknown, StateStaticFault:
|
||||
st.badProbes = 0 // neither is a dhclient fault; don't accumulate toward a heal
|
||||
w.mu.Unlock()
|
||||
if prev != state { // loud once per transition, not once per minute
|
||||
w.logger.Warn("guestnet: guest network not actionable — reporting only",
|
||||
"vmid", g.VMID, "state", string(state), "mode", string(p.Mode),
|
||||
"has_ip", p.IP != "", "has_route", p.HasRoute, "detail", detail)
|
||||
}
|
||||
w.record(g.VMID, rep)
|
||||
return
|
||||
}
|
||||
|
||||
// --- StateUnhealthy: a DHCP guest with something dhclient can fix ---------------------------
|
||||
|
||||
st.badProbes++
|
||||
bad := st.badProbes
|
||||
lastHeal := st.lastHealAt
|
||||
healsInHour := len(st.heals)
|
||||
w.mu.Unlock()
|
||||
|
||||
if reason, ok := w.observeOnly(g, now); !ok {
|
||||
rep.Damped = true
|
||||
rep.Message = detail + " — observing only: " + reason
|
||||
w.logger.Info("guestnet: guest network unhealthy but not acting", "vmid", g.VMID,
|
||||
"reason", reason, "detail", detail)
|
||||
w.record(g.VMID, rep)
|
||||
return
|
||||
}
|
||||
if bad < requiredBadProbes {
|
||||
rep.Message = detail + " — awaiting a second consecutive bad probe before healing"
|
||||
w.logger.Info("guestnet: guest network unhealthy (first bad probe — not acting yet)",
|
||||
"vmid", g.VMID, "detail", detail, "bad_probes", bad, "required", requiredBadProbes)
|
||||
w.record(g.VMID, rep)
|
||||
return
|
||||
}
|
||||
if damped, reason := w.damped(lastHeal, healsInHour, now); damped {
|
||||
rep.Damped = true
|
||||
rep.Message = detail + " — heal damped: " + reason
|
||||
w.logger.Warn("guestnet: guest network unhealthy but healing is DAMPED — reporting only",
|
||||
"vmid", g.VMID, "reason", reason, "heals_last_hour", healsInHour, "detail", detail)
|
||||
w.record(g.VMID, rep)
|
||||
return
|
||||
}
|
||||
|
||||
// --- heal ----------------------------------------------------------------------------------
|
||||
w.logger.Warn("guestnet: guest network unhealthy — healing",
|
||||
"vmid", g.VMID, "detail", detail, "bad_probes", bad)
|
||||
healed, healErr := w.heal(ctx, g.VMID)
|
||||
|
||||
w.mu.Lock()
|
||||
st = w.state[g.VMID]
|
||||
if st != nil {
|
||||
st.heals = append(st.heals, now)
|
||||
st.lastHealAt = now
|
||||
st.badProbes = 0 // the post-heal probe below is the new evidence
|
||||
healsInHour = len(st.heals)
|
||||
}
|
||||
w.mu.Unlock()
|
||||
|
||||
after := w.probe(ctx, g.VMID)
|
||||
afterState, afterDetail := classify(after)
|
||||
|
||||
rep = GuestReport{
|
||||
VMID: g.VMID, Mode: string(after.Mode), IP: after.IP, HasRoute: after.HasRoute,
|
||||
DHClientAlive: after.DHCPAlive, State: string(afterState), Message: afterDetail,
|
||||
Healed: true, HealSucceeded: afterState == StateHealthy,
|
||||
LastHealAt: now.UTC().Format(time.RFC3339), HealsLastHour: healsInHour,
|
||||
CheckedAt: w.now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if healErr != nil {
|
||||
rep.Message = "heal command failed: " + healErr.Error() + "; " + afterDetail
|
||||
}
|
||||
|
||||
if afterState == StateHealthy {
|
||||
w.logger.Info("guestnet: guest network healed", "vmid", g.VMID, "ip", after.IP,
|
||||
"has_route", after.HasRoute, "dhclient_alive", after.DHCPAlive, "heals_last_hour", healsInHour)
|
||||
} else {
|
||||
w.logger.Error("guestnet: heal did not restore the guest network", "vmid", g.VMID,
|
||||
"state", string(afterState), "detail", afterDetail, "heal_ran", healed, "err", healErr)
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
if st = w.state[g.VMID]; st != nil {
|
||||
st.lastState = afterState
|
||||
}
|
||||
w.mu.Unlock()
|
||||
w.record(g.VMID, rep)
|
||||
}
|
||||
|
||||
// heal runs the incident's restored invocation, VERBATIM (INCIDENT-guest-dhclient-killed-2026-07-20
|
||||
// §5) — the same argv that brought guest 9201 back at 10:04:3x UTC. Fixed shape, no shell, no guest
|
||||
// data interpolated. Logged at INFO before it runs so the operator sees the exact command.
|
||||
func (w *Watchdog) heal(ctx context.Context, vmid int) (bool, error) {
|
||||
args := []string{"exec", itoa(vmid), "--", "dhclient",
|
||||
"-pf", "/run/dhclient." + eth0 + ".pid",
|
||||
"-lf", "/var/lib/dhcp/dhclient." + eth0 + ".leases", eth0}
|
||||
w.logger.Info("guestnet: running heal command", "vmid", vmid, "cmd", "pct "+joinArgs(args))
|
||||
_, errOut, err := w.runner.Run(ctx, "pct", args...)
|
||||
if err != nil {
|
||||
w.logger.Error("guestnet: heal command failed", "vmid", vmid, "stderr", firstLine(string(errOut)), "err", err)
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// observeOnly reports whether a boot race means this cycle must look and not touch.
|
||||
func (w *Watchdog) observeOnly(g proxmox.Guest, now time.Time) (string, bool) {
|
||||
if now.Sub(w.startedAt) < w.settle {
|
||||
return "agent started less than " + w.settle.String() + " ago", false
|
||||
}
|
||||
if g.Uptime > 0 && time.Duration(g.Uptime)*time.Second < w.settle {
|
||||
return "guest has been up for less than " + w.settle.String(), false
|
||||
}
|
||||
return "", true
|
||||
}
|
||||
|
||||
// damped applies the two rate limits.
|
||||
func (w *Watchdog) damped(lastHeal time.Time, healsInHour int, now time.Time) (bool, string) {
|
||||
if !lastHeal.IsZero() && now.Sub(lastHeal) < w.minHealInterval {
|
||||
return true, "last heal was less than " + w.minHealInterval.String() + " ago"
|
||||
}
|
||||
if healsInHour >= w.maxHealsPerHour {
|
||||
return true, "heal budget for the hour is spent (a guest needing more than this has a fault dhclient cannot fix)"
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
func (s *guestState) pruneHeals(now time.Time) {
|
||||
kept := s.heals[:0]
|
||||
for _, t := range s.heals {
|
||||
if now.Sub(t) < time.Hour {
|
||||
kept = append(kept, t)
|
||||
}
|
||||
}
|
||||
s.heals = kept
|
||||
}
|
||||
|
||||
func (w *Watchdog) record(vmid int, rep GuestReport) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if st := w.state[vmid]; st != nil {
|
||||
st.report = rep
|
||||
}
|
||||
}
|
||||
|
||||
func joinArgs(args []string) string {
|
||||
out := ""
|
||||
for i, a := range args {
|
||||
if i > 0 {
|
||||
out += " "
|
||||
}
|
||||
out += a
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// --- the hub report block ------------------------------------------------------------------------
|
||||
|
||||
// GuestReport is one guest's last observed network state, mirrored to the hub.
|
||||
type GuestReport struct {
|
||||
VMID int `json:"vmid"`
|
||||
State string `json:"state"` // healthy | unhealthy | static_fault | unknown
|
||||
Mode string `json:"mode"` // dhcp | static | unknown
|
||||
IP string `json:"ip,omitempty"`
|
||||
HasRoute bool `json:"has_route"`
|
||||
DHClientAlive bool `json:"dhclient_alive"`
|
||||
CheckedAt string `json:"checked_at,omitempty"`
|
||||
Healed bool `json:"healed,omitempty"` // a heal ran on THIS cycle
|
||||
HealSucceeded bool `json:"heal_succeeded,omitempty"` // and the re-probe came back healthy
|
||||
LastHealAt string `json:"last_heal_at,omitempty"`
|
||||
HealsLastHour int `json:"heals_last_hour,omitempty"`
|
||||
Damped bool `json:"damped,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
// Snapshot returns the per-guest blocks for the heartbeat, VMID-sorted for a stable wire shape.
|
||||
func (w *Watchdog) Snapshot() []GuestReport {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
out := make([]GuestReport, 0, len(w.state))
|
||||
for _, st := range w.state {
|
||||
if st.report.VMID != 0 {
|
||||
out = append(out, st.report)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].VMID < out[j].VMID })
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,568 @@
|
||||
package guestnet
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// --- fixtures captured LIVE from guest 9201 on 2026-07-21 (probe P3, via `ssh felhom-pve`) -------
|
||||
//
|
||||
// These are the byte shapes the parser must survive; note the literal backslash `ip -o` emits and
|
||||
// the trailing space on the route line.
|
||||
|
||||
const (
|
||||
fxAddr = "2: eth0 inet 192.168.0.104/24 brd 192.168.0.255 scope global dynamic eth0\\ valid_lft 4916sec preferred_lft 4916sec\n"
|
||||
fxRoute = "default via 192.168.0.1 dev eth0 \n"
|
||||
fxPgrep = "235839\n"
|
||||
fxIfacesDHCP = "auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet dhcp\n"
|
||||
fxIfacesStat = "auto lo\niface lo inet loopback\n\nauto eth0\niface eth0 inet static\n\taddress 192.168.0.162/24\n\tgateway 192.168.0.1\n"
|
||||
// pct exec against a guest that does not exist / is not running (rc=2, message on stderr).
|
||||
fxNoGuestErr = "Configuration file 'nodes/demo-felhom/lxc/9999.conf' does not exist\n"
|
||||
)
|
||||
|
||||
// scriptedRunner answers per probe kind and records EVERY argv. The counts are the assertions that
|
||||
// matter: a watchdog that heals when it must not is worse than one that never heals.
|
||||
type scriptedRunner struct {
|
||||
mu sync.Mutex
|
||||
out map[string]string // kind → stdout
|
||||
fail map[string]error // kind → error
|
||||
errs map[string]string // kind → stderr
|
||||
call [][]string
|
||||
// healFixes models what a successful dhclient actually does: the client is running again and
|
||||
// the lease is renewed. Set false to model a guest whose network is broken beyond dhclient.
|
||||
healFixes bool
|
||||
}
|
||||
|
||||
func newRunner() *scriptedRunner {
|
||||
return &scriptedRunner{
|
||||
out: map[string]string{
|
||||
"addr": fxAddr, "route": fxRoute, "iface": fxIfacesDHCP, "pgrep": fxPgrep, "dhclient": "",
|
||||
},
|
||||
fail: map[string]error{},
|
||||
errs: map[string]string{},
|
||||
healFixes: true,
|
||||
}
|
||||
}
|
||||
|
||||
// kind classifies a `pct exec <vmid> -- <cmd> ...` argv.
|
||||
func kind(args []string) string {
|
||||
if len(args) < 4 || args[0] != "exec" {
|
||||
return "other"
|
||||
}
|
||||
rest := args[3:]
|
||||
switch rest[0] {
|
||||
case "ip":
|
||||
if len(rest) > 1 && rest[1] == "route" {
|
||||
return "route"
|
||||
}
|
||||
return "addr"
|
||||
case "cat":
|
||||
return "iface"
|
||||
case "pgrep":
|
||||
return "pgrep"
|
||||
case "dhclient":
|
||||
return "dhclient"
|
||||
}
|
||||
return "other"
|
||||
}
|
||||
|
||||
func (r *scriptedRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.call = append(r.call, append([]string{name}, args...))
|
||||
k := kind(args)
|
||||
if k == "dhclient" && r.fail[k] == nil && r.healFixes {
|
||||
// a real dhclient re-acquires the lease and stays resident
|
||||
r.out["pgrep"], r.out["addr"], r.out["route"] = fxPgrep, fxAddr, fxRoute
|
||||
delete(r.fail, "pgrep")
|
||||
}
|
||||
return []byte(r.out[k]), []byte(r.errs[k]), r.fail[k]
|
||||
}
|
||||
|
||||
func (r *scriptedRunner) countOf(k string) int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
n := 0
|
||||
for _, c := range r.call {
|
||||
if len(c) > 1 && kind(c[1:]) == k {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (r *scriptedRunner) lastOf(k string) []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for i := len(r.call) - 1; i >= 0; i-- {
|
||||
if len(r.call[i]) > 1 && kind(r.call[i][1:]) == k {
|
||||
return r.call[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// killDHClient models the exact 2026-07-20 state: lease still valid (address AND route present),
|
||||
// dhclient gone. This is the fixture the whole feature exists for.
|
||||
func (r *scriptedRunner) killDHClient() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.out["pgrep"] = ""
|
||||
r.fail["pgrep"] = errors.New("exit status 1") // pgrep: no match, EMPTY stderr
|
||||
}
|
||||
|
||||
func (r *scriptedRunner) reviveDHClient() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.out["pgrep"] = fxPgrep
|
||||
delete(r.fail, "pgrep")
|
||||
}
|
||||
|
||||
// expireLease models the state 80 minutes later: address and route gone too.
|
||||
func (r *scriptedRunner) expireLease() {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.out["addr"] = ""
|
||||
r.out["route"] = ""
|
||||
}
|
||||
|
||||
type fakeGuests struct {
|
||||
guests []proxmox.Guest
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeGuests) Guests(context.Context) ([]proxmox.Guest, error) { return f.guests, f.err }
|
||||
|
||||
func running9201() *fakeGuests {
|
||||
return &fakeGuests{guests: []proxmox.Guest{
|
||||
{VMID: 9201, Name: "felhom-demo", Status: "running", Type: "lxc", Uptime: 7200},
|
||||
}}
|
||||
}
|
||||
|
||||
// newTestWatchdog wires a watchdog with a manual clock the test advances, and captures the log so
|
||||
// the "healthy cycles are observable" contract can be asserted rather than assumed.
|
||||
func newTestWatchdog(r Runner, g GuestSource) (*Watchdog, *time.Time, *bytes.Buffer) {
|
||||
clock := time.Unix(1_784_000_000, 0).UTC()
|
||||
buf := &bytes.Buffer{}
|
||||
logger := slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
|
||||
w := New(r, g, time.Minute, logger)
|
||||
w.now = func() time.Time { return clock }
|
||||
// The agent's own settle window is measured from startedAt, which New stamped with the REAL
|
||||
// clock; restamp it against the fake one, well in the past.
|
||||
w.startedAt = clock.Add(-time.Hour)
|
||||
return w, &clock, buf
|
||||
}
|
||||
|
||||
// --- Scenario E: the TIMED failure, detected instantly ------------------------------------------
|
||||
//
|
||||
// RED-PROOF (recorded in REPORT.md): reverting classify()'s dhcp arm to IP-presence-only —
|
||||
//
|
||||
// case ModeDHCP:
|
||||
// if p.IP == "" { return StateUnhealthy, ... }
|
||||
// return StateHealthy, ...
|
||||
//
|
||||
// makes TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy report "healthy" for the July-20 fixture, and
|
||||
// TestWatchdog_HealsTheIncidentState records ZERO heals. That is the 80-minute silent window, exactly
|
||||
// as it happened.
|
||||
|
||||
func TestProbe_DeadDHClientWithLiveLeaseIsUnhealthy(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.killDHClient()
|
||||
w, _, _ := newTestWatchdog(r, running9201())
|
||||
|
||||
p := w.probe(context.Background(), 9201)
|
||||
if !p.Reachable {
|
||||
t.Fatalf("guest must read as reachable: %+v", p)
|
||||
}
|
||||
if p.IP != "192.168.0.104" || !p.HasRoute {
|
||||
t.Fatalf("the lease is still live in this fixture — IP/route must be present: %+v", p)
|
||||
}
|
||||
if p.DHCPAlive {
|
||||
t.Fatalf("dhclient must read as dead: %+v", p)
|
||||
}
|
||||
|
||||
state, detail := classify(p)
|
||||
if state != StateUnhealthy {
|
||||
t.Fatalf("classify = %q, want %q — waiting for the IP to vanish is the 80-minute silent "+
|
||||
"window the incident proved (detail: %s)", state, StateUnhealthy, detail)
|
||||
}
|
||||
if !strings.Contains(detail, "dhclient") {
|
||||
t.Fatalf("the reason must name the dead client, got %q", detail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_HealsTheIncidentState(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.killDHClient()
|
||||
w, clock, logBuf := newTestWatchdog(r, running9201())
|
||||
ctx := context.Background()
|
||||
|
||||
// Cycle 1: unhealthy, but one bad probe is not a diagnosis.
|
||||
w.Tick(ctx)
|
||||
if n := r.countOf("dhclient"); n != 0 {
|
||||
t.Fatalf("healed after ONE bad probe (%d heals) — a single blip must never trigger a heal", n)
|
||||
}
|
||||
|
||||
// Cycle 2: second consecutive bad probe → heal. The heal makes the client live again.
|
||||
*clock = clock.Add(time.Minute)
|
||||
w.Tick(ctx)
|
||||
|
||||
if n := r.countOf("dhclient"); n != 1 {
|
||||
t.Fatalf("heal ran %d times, want exactly 1", n)
|
||||
}
|
||||
// The invocation must be the incident's, verbatim.
|
||||
want := []string{"pct", "exec", "9201", "--", "dhclient",
|
||||
"-pf", "/run/dhclient.eth0.pid", "-lf", "/var/lib/dhcp/dhclient.eth0.leases", "eth0"}
|
||||
got := r.lastOf("dhclient")
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("heal argv = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("heal argv[%d] = %q, want %q (full: %v)", i, got[i], want[i], got)
|
||||
}
|
||||
}
|
||||
|
||||
// The report must show the heal AND the verified-healthy re-probe.
|
||||
snap := w.Snapshot()
|
||||
if len(snap) != 1 {
|
||||
t.Fatalf("snapshot = %+v, want one guest", snap)
|
||||
}
|
||||
g := snap[0]
|
||||
if !g.Healed || !g.HealSucceeded {
|
||||
t.Fatalf("report must record a successful heal: %+v", g)
|
||||
}
|
||||
if g.State != string(StateHealthy) || g.LastHealAt == "" || g.HealsLastHour != 1 {
|
||||
t.Fatalf("post-heal report is wrong: %+v", g)
|
||||
}
|
||||
if !strings.Contains(logBuf.String(), "guestnet: guest network healed") {
|
||||
t.Fatalf("the heal was not logged: %s", logBuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A healthy box must be silent about alarms but NOT silent about having looked.
|
||||
func TestWatchdog_HealthyCycleProbesAndNeverHeals(t *testing.T) {
|
||||
r := newRunner()
|
||||
w, _, logBuf := newTestWatchdog(r, running9201())
|
||||
|
||||
w.Tick(context.Background())
|
||||
|
||||
if n := r.countOf("dhclient"); n != 0 {
|
||||
t.Fatalf("a healthy guest was healed %d times, want 0", n)
|
||||
}
|
||||
if r.countOf("pgrep") != 1 || r.countOf("addr") != 1 {
|
||||
t.Fatalf("the healthy path must still probe: %v", r.call)
|
||||
}
|
||||
if !strings.Contains(logBuf.String(), "guest network healthy") {
|
||||
t.Fatalf("a healthy cycle must be observable — otherwise 'no alarms' and 'never probed' "+
|
||||
"are the same evidence (v0.91.2's lesson). Log: %s", logBuf.String())
|
||||
}
|
||||
snap := w.Snapshot()
|
||||
if len(snap) != 1 || snap[0].State != string(StateHealthy) || !snap[0].DHClientAlive {
|
||||
t.Fatalf("healthy snapshot wrong: %+v", snap)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Scenario F: configuration and damping guards ------------------------------------------------
|
||||
|
||||
func TestWatchdog_StaticGuestIsNeverHealedWithDHClient(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.out["iface"] = fxIfacesStat
|
||||
r.killDHClient() // on a static guest this is NORMAL
|
||||
w, clock, _ := newTestWatchdog(r, running9201())
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
w.Tick(context.Background())
|
||||
*clock = clock.Add(time.Minute)
|
||||
}
|
||||
if n := r.countOf("dhclient"); n != 0 {
|
||||
t.Fatalf("a static guest was healed with dhclient %d times, want 0", n)
|
||||
}
|
||||
if s := w.Snapshot(); len(s) != 1 || s[0].State != string(StateHealthy) {
|
||||
t.Fatalf("a static guest with address+route is healthy, got %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_StaticGuestMissingAddressReportsButNeverHeals(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.out["iface"] = fxIfacesStat
|
||||
r.expireLease() // no address, no route on a STATIC guest → R-50 territory, not ours
|
||||
w, clock, logBuf := newTestWatchdog(r, running9201())
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
w.Tick(context.Background())
|
||||
*clock = clock.Add(time.Minute)
|
||||
}
|
||||
if n := r.countOf("dhclient"); n != 0 {
|
||||
t.Fatalf("healed a static guest %d times, want 0 — dhclient must never fight a static config", n)
|
||||
}
|
||||
s := w.Snapshot()
|
||||
if len(s) != 1 || s[0].State != string(StateStaticFault) {
|
||||
t.Fatalf("state = %+v, want static_fault", s)
|
||||
}
|
||||
if !strings.Contains(logBuf.String(), "not actionable") {
|
||||
t.Fatalf("a static fault must be reported loudly: %s", logBuf.String())
|
||||
}
|
||||
// Loud ONCE per transition, not once per cycle.
|
||||
if n := strings.Count(logBuf.String(), "not actionable"); n != 1 {
|
||||
t.Fatalf("static fault logged %d times over 5 cycles, want 1 (per transition)", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_UnreachableGuestIsUnknownAndNeverHealed(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.fail["addr"] = errors.New("exit status 2")
|
||||
r.errs["addr"] = fxNoGuestErr
|
||||
w, clock, _ := newTestWatchdog(r, running9201())
|
||||
|
||||
for i := 0; i < 4; i++ {
|
||||
w.Tick(context.Background())
|
||||
*clock = clock.Add(time.Minute)
|
||||
}
|
||||
if n := r.countOf("dhclient"); n != 0 {
|
||||
t.Fatalf("healed a guest we could not probe %d times, want 0 — acting blind is how the "+
|
||||
"incident happened", n)
|
||||
}
|
||||
if s := w.Snapshot(); len(s) != 1 || s[0].State != string(StateUnknown) {
|
||||
t.Fatalf("state = %+v, want unknown", s)
|
||||
}
|
||||
}
|
||||
|
||||
// A missing pgrep (or any probe tool) must read as unknown, never as a dead client — otherwise a
|
||||
// broken probe would heal forever.
|
||||
func TestWatchdog_ProbeToolFailureIsUnknownNotDead(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.fail["pgrep"] = errors.New("exit status 127")
|
||||
r.errs["pgrep"] = "pgrep: command not found\n"
|
||||
w, clock, _ := newTestWatchdog(r, running9201())
|
||||
|
||||
for i := 0; i < 4; i++ {
|
||||
w.Tick(context.Background())
|
||||
*clock = clock.Add(time.Minute)
|
||||
}
|
||||
if n := r.countOf("dhclient"); n != 0 {
|
||||
t.Fatalf("a failed liveness probe caused %d heals, want 0", n)
|
||||
}
|
||||
if s := w.Snapshot(); len(s) != 1 || s[0].State != string(StateUnknown) {
|
||||
t.Fatalf("state = %+v, want unknown", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_DampingCeilings(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.killDHClient()
|
||||
r.healFixes = false // the heal "works" but the client dies again immediately
|
||||
w, clock, _ := newTestWatchdog(r, running9201())
|
||||
ctx := context.Background()
|
||||
|
||||
// 10 hours of one-minute cycles against a permanently broken guest.
|
||||
for i := 0; i < 600; i++ {
|
||||
w.Tick(ctx)
|
||||
*clock = clock.Add(time.Minute)
|
||||
}
|
||||
|
||||
heals := r.countOf("dhclient")
|
||||
// Ceiling: 3 per hour AND ≥10 min apart ⇒ at most 3 in any rolling hour. Over 10 h the
|
||||
// min-interval rule dominates: 6 slots/hour capped to 3/hour ⇒ ≤ 30.
|
||||
if heals > 30 {
|
||||
t.Fatalf("heals = %d over 10 h, want ≤ 30 (≤3/hour) — the damper is not holding", heals)
|
||||
}
|
||||
if heals == 0 {
|
||||
t.Fatalf("heals = 0 — the damper has become a mute")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_MinimumIntervalBetweenHeals(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.killDHClient()
|
||||
r.healFixes = false
|
||||
w, clock, _ := newTestWatchdog(r, running9201())
|
||||
ctx := context.Background()
|
||||
|
||||
w.Tick(ctx) // bad probe 1
|
||||
*clock = clock.Add(time.Minute)
|
||||
w.Tick(ctx) // bad probe 2 → heal #1
|
||||
if r.countOf("dhclient") != 1 {
|
||||
t.Fatalf("expected exactly one heal by now, got %d", r.countOf("dhclient"))
|
||||
}
|
||||
// Nine more minutes of failure: still inside the 10-minute cool-off.
|
||||
for i := 0; i < 9; i++ {
|
||||
*clock = clock.Add(time.Minute)
|
||||
w.Tick(ctx)
|
||||
}
|
||||
if n := r.countOf("dhclient"); n != 1 {
|
||||
t.Fatalf("heals = %d within the 10-minute cool-off, want 1", n)
|
||||
}
|
||||
if s := w.Snapshot(); len(s) != 1 || !s[0].Damped {
|
||||
t.Fatalf("a damped cycle must say so in the report: %+v", s)
|
||||
}
|
||||
// Past the cool-off, one more heal is allowed.
|
||||
*clock = clock.Add(2 * time.Minute)
|
||||
w.Tick(ctx)
|
||||
if n := r.countOf("dhclient"); n != 2 {
|
||||
t.Fatalf("heals = %d after the cool-off expired, want 2", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_BootRacesObserveOnly(t *testing.T) {
|
||||
t.Run("young guest", func(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.killDHClient()
|
||||
g := running9201()
|
||||
g.guests[0].Uptime = 40 // seconds
|
||||
w, clock, _ := newTestWatchdog(r, g)
|
||||
for i := 0; i < 4; i++ {
|
||||
w.Tick(context.Background())
|
||||
*clock = clock.Add(time.Minute)
|
||||
}
|
||||
if n := r.countOf("dhclient"); n != 0 {
|
||||
t.Fatalf("healed a guest that booted 40 s ago %d times, want 0 — it has no lease YET", n)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("young agent", func(t *testing.T) {
|
||||
r := newRunner()
|
||||
r.killDHClient()
|
||||
w, clock, _ := newTestWatchdog(r, running9201())
|
||||
w.startedAt = *clock // the agent just started
|
||||
for i := 0; i < 2; i++ {
|
||||
w.Tick(context.Background())
|
||||
*clock = clock.Add(time.Minute)
|
||||
}
|
||||
if n := r.countOf("dhclient"); n != 0 {
|
||||
t.Fatalf("healed %d times within the agent's own settle window, want 0", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWatchdog_StoppedGuestIsNotProbed(t *testing.T) {
|
||||
r := newRunner()
|
||||
g := running9201()
|
||||
g.guests[0].Status = "stopped"
|
||||
w, _, _ := newTestWatchdog(r, g)
|
||||
|
||||
w.Tick(context.Background())
|
||||
|
||||
if len(r.call) != 0 {
|
||||
t.Fatalf("a stopped guest was probed: %v", r.call)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchdog_GuestListFailureSkipsTheSweep(t *testing.T) {
|
||||
r := newRunner()
|
||||
w, _, logBuf := newTestWatchdog(r, &fakeGuests{err: errors.New("pool membership read: 403")})
|
||||
|
||||
w.Tick(context.Background())
|
||||
|
||||
if len(r.call) != 0 {
|
||||
t.Fatalf("acted with unproven ownership: %v", r.call)
|
||||
}
|
||||
if !strings.Contains(logBuf.String(), "ownership unproven") {
|
||||
t.Fatalf("the skip must be logged: %s", logBuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A transient bad probe followed by recovery must never heal, and must log the recovery.
|
||||
func TestWatchdog_SingleBlipNeverHeals(t *testing.T) {
|
||||
r := newRunner()
|
||||
w, clock, logBuf := newTestWatchdog(r, running9201())
|
||||
ctx := context.Background()
|
||||
|
||||
w.Tick(ctx) // healthy
|
||||
r.killDHClient()
|
||||
*clock = clock.Add(time.Minute)
|
||||
w.Tick(ctx) // bad probe 1
|
||||
r.reviveDHClient()
|
||||
*clock = clock.Add(time.Minute)
|
||||
w.Tick(ctx) // healthy again
|
||||
|
||||
if n := r.countOf("dhclient"); n != 0 {
|
||||
t.Fatalf("a single blip caused %d heals, want 0", n)
|
||||
}
|
||||
if !strings.Contains(logBuf.String(), "guest network recovered") {
|
||||
t.Fatalf("the recovery must be visible: %s", logBuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// --- parsers over the live P3 fixtures ------------------------------------------------------------
|
||||
|
||||
func TestParsers_OverLiveFixtures(t *testing.T) {
|
||||
if got := parseInet(fxAddr); got != "192.168.0.104" {
|
||||
t.Fatalf("parseInet = %q, want 192.168.0.104", got)
|
||||
}
|
||||
if got := parseInet(""); got != "" {
|
||||
t.Fatalf("parseInet(empty) = %q, want empty (the post-expiry state prints nothing)", got)
|
||||
}
|
||||
if !hasDefaultRoute(fxRoute) {
|
||||
t.Fatalf("hasDefaultRoute(%q) = false", fxRoute)
|
||||
}
|
||||
if hasDefaultRoute("") || hasDefaultRoute("172.17.0.0/16 dev docker0 proto kernel scope link\n") {
|
||||
t.Fatal("docker bridge routes must not read as a default route (the incident's exact leftovers)")
|
||||
}
|
||||
if got := parseMode(fxIfacesDHCP, "eth0"); got != ModeDHCP {
|
||||
t.Fatalf("parseMode(dhcp) = %q", got)
|
||||
}
|
||||
if got := parseMode(fxIfacesStat, "eth0"); got != ModeStatic {
|
||||
t.Fatalf("parseMode(static) = %q", got)
|
||||
}
|
||||
if got := parseMode("auto lo\niface lo inet loopback\n", "eth0"); got != ModeUnknown {
|
||||
t.Fatalf("parseMode(no eth0 stanza) = %q, want unknown", got)
|
||||
}
|
||||
if got := parseMode("iface eth0 inet manual\n", "eth0"); got != ModeUnknown {
|
||||
t.Fatalf("parseMode(manual) = %q, want unknown", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassify_Table(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
p Probe
|
||||
want State
|
||||
}{
|
||||
{"dhcp all green", Probe{Reachable: true, Mode: ModeDHCP, IP: "1.2.3.4", HasRoute: true, DHCPAlive: true}, StateHealthy},
|
||||
{"dhcp dead client, live lease", Probe{Reachable: true, Mode: ModeDHCP, IP: "1.2.3.4", HasRoute: true}, StateUnhealthy},
|
||||
{"dhcp no address", Probe{Reachable: true, Mode: ModeDHCP, DHCPAlive: true}, StateUnhealthy},
|
||||
{"dhcp no route", Probe{Reachable: true, Mode: ModeDHCP, IP: "1.2.3.4", DHCPAlive: true}, StateUnhealthy},
|
||||
{"static green", Probe{Reachable: true, Mode: ModeStatic, IP: "1.2.3.4", HasRoute: true}, StateHealthy},
|
||||
{"static no client is normal", Probe{Reachable: true, Mode: ModeStatic, IP: "1.2.3.4", HasRoute: true}, StateHealthy},
|
||||
{"static broken", Probe{Reachable: true, Mode: ModeStatic}, StateStaticFault},
|
||||
{"unreachable", Probe{Mode: ModeDHCP}, StateUnknown},
|
||||
{"unknown mode", Probe{Reachable: true, Mode: ModeUnknown, IP: "1.2.3.4", HasRoute: true}, StateUnknown},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got, _ := classify(tc.p); got != tc.want {
|
||||
t.Fatalf("classify = %q, want %q", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var _ io.Writer = (*bytes.Buffer)(nil)
|
||||
|
||||
// A3 (R-50): the guestnet healer is eth0-only and MUST stay blind to the island NIC. A guest on an
|
||||
// island host presents eth0 DHCP (the LAN leg the healer owns) PLUS eth1 static (the island). Because
|
||||
// parseMode is interface-scoped, adding eth1 static cannot flip eth0's detected mode — so the healer
|
||||
// keeps treating eth0 as DHCP and never runs dhclient against the static island NIC (which would
|
||||
// sabotage it). This is the verify-only guarantee that let R-50 ship the island NIC without a healer
|
||||
// change. Red-proof: make parseMode scan globally instead of per-dev and the eth0 assertion fails.
|
||||
func TestParseMode_IslandStaticNICDoesNotConfuseEth0(t *testing.T) {
|
||||
interfaces := "auto lo\niface lo inet loopback\n\n" +
|
||||
"auto eth0\niface eth0 inet dhcp\n\n" +
|
||||
"auto eth1\niface eth1 inet static\n address 169.254.253.2/30\n"
|
||||
if got := parseMode(interfaces, "eth0"); got != ModeDHCP {
|
||||
t.Errorf("eth0 must classify DHCP even with an island eth1 static present, got %q", got)
|
||||
}
|
||||
if got := parseMode(interfaces, "eth1"); got != ModeStatic {
|
||||
t.Errorf("eth1 (island) must classify static when asked directly (dev-scoped), got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -193,6 +193,50 @@ func (c *Client) RegisterWG(ctx context.Context, pubkey string) (*WGRegisterResp
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// ErrNoPBSSecret is the typed "404: no unconsumed PBS token secret staged for this host" outcome
|
||||
// (PBS DR slice 2). Absent-or-already-consumed are indistinguishable by design (consume-once).
|
||||
var ErrNoPBSSecret = fmt.Errorf("hub: no unconsumed PBS token secret staged for this host")
|
||||
|
||||
// ConsumePBSToken fetches this host's one-time PBS token secret — EXACTLY ONCE (PBS DR slice 2;
|
||||
// POST /api/v1/hosts/{host_id}/pbs/consume-token, per-host key, self-scoped; NOTE the PLURAL
|
||||
// /hosts/ — the slice-1 route). A 200 burns the secret hub-side: the caller MUST apply it or
|
||||
// surface a loud consumed-but-failed state (never silent-retry). The secret is returned to the
|
||||
// caller only — never logged, never in an error.
|
||||
func (c *Client) ConsumePBSToken(ctx context.Context) (string, error) {
|
||||
if c.hostID == "" {
|
||||
return "", fmt.Errorf("hub: ConsumePBSToken requires a configured host_id")
|
||||
}
|
||||
url := c.baseURL + "/api/v1/hosts/" + c.hostID + "/pbs/consume-token"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+c.apiKey)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := c.hc.Do(req)
|
||||
if err != nil {
|
||||
return "", &TransportError{Err: err}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return "", ErrNoPBSSecret
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return "", &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
|
||||
}
|
||||
var out struct {
|
||||
TokenSecret string `json:"token_secret"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return "", fmt.Errorf("hub: decoding consume-token response (body withheld — secret channel)")
|
||||
}
|
||||
if out.TokenSecret == "" {
|
||||
return "", fmt.Errorf("hub: consume-token returned an empty secret")
|
||||
}
|
||||
return out.TokenSecret, nil
|
||||
}
|
||||
|
||||
// JobWire is one queued signed-op job as served by GET /hosts/{id}/jobs (slice 10A). The blob is
|
||||
// OPAQUE to the hub — for slice 10B it is a base64 `SignedJobEnvelope` (op-blob + armored SSHSIG)
|
||||
// the agent verifies before executing.
|
||||
@@ -263,3 +307,50 @@ func tail(b []byte, max int) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// IdentityEscrowResponse mirrors GET /api/v1/hosts/{host_id}/escrow (hub >= v0.94.0, R-199).
|
||||
// Present=false is a CLEAN answer, not a fault: the host simply has no sealed bundle yet.
|
||||
type IdentityEscrowResponse struct {
|
||||
HostID string `json:"host_id"`
|
||||
Present bool `json:"present"`
|
||||
IdentityEscrowB64 string `json:"identity_escrow_b64"`
|
||||
}
|
||||
|
||||
// FetchIdentityEscrow reads back THIS host's own opaque identity-escrow blob (R-199 link 6 — the
|
||||
// mirror of UploadEscrow, self-scoped server-side by the per-host key). The bytes are ciphertext: they
|
||||
// are useless without the customer's recovery code R, which neither the hub nor this agent ever holds.
|
||||
//
|
||||
// It is the ONLY retrieval this client performs, and it is deliberately narrow — no directive, no
|
||||
// K-escrow, no key rotation. The operator-driven DR path (recovery-mode re-enroll) is a different
|
||||
// endpoint with a different gate and is not reached from here.
|
||||
//
|
||||
// Errors are typed (transport vs HTTP) and never include the bearer token. The BLOB is never logged —
|
||||
// only its length.
|
||||
func (c *Client) FetchIdentityEscrow(ctx context.Context) (*IdentityEscrowResponse, error) {
|
||||
if c.hostID == "" {
|
||||
return nil, fmt.Errorf("hub: FetchIdentityEscrow requires a configured host_id")
|
||||
}
|
||||
url := c.baseURL + "/api/v1/hosts/" + c.hostID + "/escrow"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hub: building escrow-fetch request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+c.apiKey)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := c.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, &TransportError{Err: err}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
|
||||
}
|
||||
var out IdentityEscrowResponse
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("hub: decoding escrow fetch: %w", err)
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
+221
-7
@@ -2,8 +2,12 @@ package hub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
|
||||
@@ -43,6 +47,22 @@ type RestoreTestReporter interface {
|
||||
RestoreTests(ctx context.Context) []RestoreTest
|
||||
}
|
||||
|
||||
// ProvenRestoreTestReporter is the DURABLE half of the restore-test signal (R-189).
|
||||
//
|
||||
// RestoreTestReporter above is backed by an in-memory store whose own comment used to read "lost on
|
||||
// restart; the cadence re-populates". That was true while a timer re-tested every tier daily. It
|
||||
// stopped being true on 2026-08-03: under per-archive due-ness the agent will not re-test an archive
|
||||
// it has already proven, so a proof lost to a restart is not repeated for a whole archive generation
|
||||
// — a week on the offsite tier — and the hub reports the tier unproven the entire time.
|
||||
//
|
||||
// Observed, not predicted: a real 14.5 GB offsite restore passed at 15:25:14, the agent was restarted
|
||||
// 2 m 43 s later for a deploy, and the hub logged `0 restore-tests` on the next two reports.
|
||||
//
|
||||
// (*backup.RestoreTestState).ProvenRestoreTests satisfies this. nil → the merge is a no-op.
|
||||
type ProvenRestoreTestReporter interface {
|
||||
ProvenRestoreTests(ctx context.Context) []RestoreTest
|
||||
}
|
||||
|
||||
// PBSReporter is the slice-6-Phase-B seam the pbs verify loop plugs into (same pattern).
|
||||
// Returns the agent's latest-known PBS snapshot inventory + verify-state. nil → empty.
|
||||
type PBSReporter interface {
|
||||
@@ -55,6 +75,18 @@ type WireguardReporter interface {
|
||||
WireguardStatus(ctx context.Context) *WireguardStatus
|
||||
}
|
||||
|
||||
// PBSDRReporter is the slice-2 seam the pbsdr bridge loop plugs into (same consumer-side
|
||||
// pattern — hub does not import pbsdr). nil (feature not wired) → no pbs_dr stanza.
|
||||
type PBSDRReporter interface {
|
||||
PBSDRStatus(ctx context.Context) *PBSDRStatus
|
||||
}
|
||||
|
||||
// GuestNetReporter is the R-54 seam the guestnet watchdog plugs into (same consumer-side pattern —
|
||||
// hub does not import guestnet). nil (feature not wired) → no guest_net stanza.
|
||||
type GuestNetReporter interface {
|
||||
GuestNetStatus(ctx context.Context) *GuestNetStatus
|
||||
}
|
||||
|
||||
// Collector builds a HostReport from read-only sources. All deps are behind narrow
|
||||
// interfaces for unit testing.
|
||||
type Collector struct {
|
||||
@@ -63,12 +95,19 @@ type Collector struct {
|
||||
storage StorageObserver
|
||||
backups BackupReporter
|
||||
restoreTests RestoreTestReporter
|
||||
provenTests ProvenRestoreTestReporter
|
||||
pbs PBSReporter
|
||||
temp TempReader // slice 9: host CPU/chassis temp (nil-safe → nil temp)
|
||||
capProbe func(ctx context.Context) []capability.Status // v0.44.0: privileged-capability self-check (nil → empty)
|
||||
leafFP string // v0.48.0: served local-API leaf fp (static per process; "" when local API disabled)
|
||||
addrEnum AddressEnumerator // v0.119.0: host interface enumeration; nil => the REAL one (see collectAddresses)
|
||||
wg WireguardReporter // S3: offsite-tunnel status (nil → stanza omitted)
|
||||
pbsdr PBSDRReporter // slice 2: PBS DR tier bridge state (nil → stanza omitted)
|
||||
guestNet GuestNetReporter // R-54: per-guest network watchdog (nil → stanza omitted)
|
||||
selfUpdate SelfUpdateReporter // D1: agent self-update pending status (nil → false)
|
||||
mgmtPlane MgmtPlaneReporter // G1: management-plane health (nil → stanza omitted)
|
||||
oob OOBReporter // H1: operator-access health (nil → stanza omitted)
|
||||
backupTarget func() ConfiguredBackupTarget // R-109: primary backup tier id (nil → recipe records unknown)
|
||||
hostID string
|
||||
agentVersion string
|
||||
logger *slog.Logger
|
||||
@@ -103,6 +142,30 @@ func (c *Collector) SetTempReader(t TempReader) *Collector {
|
||||
return c
|
||||
}
|
||||
|
||||
// SetBackupTargetResolver wires the DR recipe to the agent's own backup config (R-109), so the recipe
|
||||
// can name WHICH storage holds the local whole-guest archives. Returns the collector for chaining.
|
||||
//
|
||||
// The resolver MUST report the tier that is IN EFFECT, which is the daemon-start snapshot — NOT the
|
||||
// current contents of agent.json. A backup-target move rewrites that file and deliberately does not
|
||||
// restart the agent (the E-1 lesson: restarting mid-backup records a spurious failure for a run that
|
||||
// succeeded), so between the write and the restart the file names a target no backup is writing to yet.
|
||||
// Re-reading the file here — the live-reload shape used for escrow.pbs_storage_id — would make the
|
||||
// recipe point at the new storage while every archive still landed on the old one. One state, one
|
||||
// owner: the recipe follows what performs the backup.
|
||||
func (c *Collector) SetBackupTargetResolver(f func() ConfiguredBackupTarget) *Collector {
|
||||
c.backupTarget = f
|
||||
return c
|
||||
}
|
||||
|
||||
// configuredBackupTarget consults the resolver. An unwired seam is reported as NOT KNOWN — never as a
|
||||
// guess — so the recipe records an explicit unknown instead of a target the agent never verified.
|
||||
func (c *Collector) configuredBackupTarget() ConfiguredBackupTarget {
|
||||
if c.backupTarget == nil {
|
||||
return ConfiguredBackupTarget{}
|
||||
}
|
||||
return c.backupTarget()
|
||||
}
|
||||
|
||||
// SetCapabilityProber wires the privileged-capability self-check (v0.44.0): each collect runs it
|
||||
// and attaches the snapshot. nil → the report carries an empty []. Returns the collector for chaining.
|
||||
func (c *Collector) SetCapabilityProber(probe func(ctx context.Context) []capability.Status) *Collector {
|
||||
@@ -125,6 +188,20 @@ func (c *Collector) SetWireguardReporter(w WireguardReporter) *Collector {
|
||||
return c
|
||||
}
|
||||
|
||||
// SetPBSDRReporter wires the PBS-DR-tier bridge state source (slice 2; nil-safe → stanza
|
||||
// omitted). Returns the collector for chaining.
|
||||
func (c *Collector) SetPBSDRReporter(p PBSDRReporter) *Collector {
|
||||
c.pbsdr = p
|
||||
return c
|
||||
}
|
||||
|
||||
// SetGuestNetReporter wires the R-54 guest-network watchdog as a report source (nil-safe → stanza
|
||||
// omitted). Returns the collector for chaining.
|
||||
func (c *Collector) SetGuestNetReporter(g GuestNetReporter) *Collector {
|
||||
c.guestNet = g
|
||||
return c
|
||||
}
|
||||
|
||||
// SelfUpdateReporter is the D1 seam the selfupdate commit-manager plugs into (same consumer-side
|
||||
// pattern — hub does not import selfupdate). nil (feature not wired) → pending=false on the report.
|
||||
type SelfUpdateReporter interface {
|
||||
@@ -140,6 +217,30 @@ func (c *Collector) SetSelfUpdateReporter(s SelfUpdateReporter) *Collector {
|
||||
return c
|
||||
}
|
||||
|
||||
// MgmtPlaneReporter is the G1 seam the mgmtplane observer plugs into (same consumer-side pattern —
|
||||
// hub does not import mgmtplane). nil (feature not wired) → no mgmt_plane stanza on the report.
|
||||
type MgmtPlaneReporter interface {
|
||||
MgmtPlaneStatus(ctx context.Context) *MgmtPlaneStatus
|
||||
}
|
||||
|
||||
// SetMgmtPlaneReporter wires the management-plane health source (G1; nil-safe → stanza omitted).
|
||||
// Returns the collector for chaining.
|
||||
func (c *Collector) SetMgmtPlaneReporter(m MgmtPlaneReporter) *Collector {
|
||||
c.mgmtPlane = m
|
||||
return c
|
||||
}
|
||||
|
||||
// OOBReporter is the H1 seam the felhom-sshd loop plugs into (nil → no oob stanza).
|
||||
type OOBReporter interface {
|
||||
OOBStatus(ctx context.Context) *OOBStatus
|
||||
}
|
||||
|
||||
// SetOOBReporter wires the operator-access health source (H1; nil-safe → stanza omitted).
|
||||
func (c *Collector) SetOOBReporter(o OOBReporter) *Collector {
|
||||
c.oob = o
|
||||
return c
|
||||
}
|
||||
|
||||
// Collect builds the report. Best-effort liveness: a failed NodeStatus is a hard
|
||||
// error (no useful report — the cycle skips the POST); a failed per-guest
|
||||
// GuestConfig degrades that guest to status="unknown" without spec but still sends;
|
||||
@@ -151,7 +252,8 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
|
||||
}
|
||||
|
||||
host := hostMetrics(c.px.Node(), ns)
|
||||
host.CPUTempC = c.cpuTempC(ctx) // slice 9: operator freebie — temp now rides the hub report too
|
||||
host.CPUTempC = c.cpuTempC(ctx) // slice 9: operator freebie — temp now rides the hub report too
|
||||
host.WrapperSHA256 = pbsWrapperSHA256() // R-50b(a): make privileged-artifact drift answerable
|
||||
report := &HostReport{
|
||||
HostID: c.hostID,
|
||||
ReportedAt: c.now().Format(time.RFC3339),
|
||||
@@ -169,19 +271,36 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) {
|
||||
Cloudflared: Cloudflared{Status: c.cloudflaredStatus(ctx)},
|
||||
Capabilities: c.capabilities(ctx),
|
||||
LeafFingerprint: c.leafFP,
|
||||
Addresses: c.collectAddresses(),
|
||||
}
|
||||
// DR recipe host-half — derived from the just-collected guest/storage/PBS facts (no new reads).
|
||||
// Secret-free by construction (identifiers/intents/sizes/coordinates only).
|
||||
report.DRRecipe = BuildDRRecipeHostHalf(report.Guests, report.StorageTargets, report.PBSSnapshots)
|
||||
report.DRRecipe = BuildDRRecipeHostHalf(report.Guests, report.StorageTargets, report.PBSSnapshots, c.configuredBackupTarget())
|
||||
// S3: offsite-tunnel status stanza (nil reporter = feature disabled → omitted; the pubkey in
|
||||
// it is the operator's revocation-recovery handle).
|
||||
if c.wg != nil {
|
||||
report.Wireguard = c.wg.WireguardStatus(ctx)
|
||||
}
|
||||
// Slice 2: PBS DR tier bridge state (nil reporter = feature not wired → stanza omitted).
|
||||
if c.pbsdr != nil {
|
||||
report.PBSDR = c.pbsdr.PBSDRStatus(ctx)
|
||||
}
|
||||
// R-54: guest-network watchdog state (nil reporter = feature not wired → stanza omitted).
|
||||
if c.guestNet != nil {
|
||||
report.GuestNet = c.guestNet.GuestNetStatus(ctx)
|
||||
}
|
||||
// D1: agent self-update pending status (nil reporter → pending=false, the steady state).
|
||||
if c.selfUpdate != nil {
|
||||
report.SelfUpdatePending, report.SelfUpdatePendingVersion = c.selfUpdate.SelfUpdatePending()
|
||||
}
|
||||
// G1: management-plane health (nil reporter = feature not wired → stanza omitted).
|
||||
if c.mgmtPlane != nil {
|
||||
report.MgmtPlane = c.mgmtPlane.MgmtPlaneStatus(ctx)
|
||||
}
|
||||
// H1: operator-access (OOB) health (nil reporter = feature not wired → stanza omitted).
|
||||
if c.oob != nil {
|
||||
report.OOB = c.oob.OOBStatus(ctx)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
@@ -220,6 +339,27 @@ func (c *Collector) cpuTempC(ctx context.Context) *int {
|
||||
return c.temp.CPUTempC(ctx)
|
||||
}
|
||||
|
||||
// pbsWrapperPath is the installed PBS-DR apply wrapper. Duplicated from internal/pbsdr.WrapperPath
|
||||
// rather than imported, to keep the report collector free of a dependency on the DR bridge.
|
||||
const pbsWrapperPath = "/usr/local/sbin/felhom-pbs-apply"
|
||||
|
||||
// pbsWrapperSHA256 hashes the installed wrapper for the report (R-50b(a)). Best-effort: a missing or
|
||||
// unreadable file yields "", which the hub reads as UNKNOWN rather than as drift — a host that
|
||||
// legitimately has no DR wrapper must not light up amber. The file is 0755, so no privilege is
|
||||
// needed to read it.
|
||||
func pbsWrapperSHA256() string {
|
||||
f, err := os.Open(pbsWrapperPath)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return ""
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
func hostMetrics(node string, ns proxmox.NodeStatus) HostMetrics {
|
||||
h := HostMetrics{
|
||||
Node: node,
|
||||
@@ -304,16 +444,90 @@ func (c *Collector) collectBackups(ctx context.Context) []Backup {
|
||||
return []Backup{}
|
||||
}
|
||||
|
||||
// collectRestoreTests merges the in-memory result with the PERSISTED per-tier proofs (R-189).
|
||||
//
|
||||
// The rule is ONE ENTRY PER TIER, NEWEST WINS, and it falls out of what each source means rather
|
||||
// than from a preference between them:
|
||||
//
|
||||
// - the in-memory store holds this process's latest run, pass OR fail. A failure exists nowhere
|
||||
// else and must always reach the hub — a failing tier is retried at the next evaluation, so its
|
||||
// record is short-lived by design;
|
||||
// - the persisted state holds the last SUCCESS per tier and survives a restart.
|
||||
//
|
||||
// Comparing by TestedAt gives the right answer in every case without special-casing: a fresh failure
|
||||
// beats an older stored success (the failure is the news), a stored success beats a stale in-memory
|
||||
// entry after a restart, and a tier proved twice never appears twice — two entries for one tier would
|
||||
// read at the hub as two tests.
|
||||
//
|
||||
// A tier with no usable persisted proof contributes NOTHING. Reporting an unproven tier as proven
|
||||
// would be a worse defect than the one this closes.
|
||||
func (c *Collector) collectRestoreTests(ctx context.Context) []RestoreTest {
|
||||
if c.restoreTests == nil {
|
||||
return []RestoreTest{}
|
||||
out := []RestoreTest{}
|
||||
if c.restoreTests != nil {
|
||||
if r := c.restoreTests.RestoreTests(ctx); r != nil {
|
||||
out = append(out, r...)
|
||||
}
|
||||
}
|
||||
if r := c.restoreTests.RestoreTests(ctx); r != nil {
|
||||
return r
|
||||
if c.provenTests == nil {
|
||||
return out
|
||||
}
|
||||
return []RestoreTest{}
|
||||
|
||||
// Index what we already have by tier, keeping the newest per tier.
|
||||
best := map[string]int{} // tier → index into out
|
||||
for i, rt := range out {
|
||||
if rt.SourceTier == "" {
|
||||
continue // untiered entry: never deduped, never overwritten — we cannot say what it is
|
||||
}
|
||||
if j, seen := best[rt.SourceTier]; !seen || newerRestoreTest(rt, out[j]) {
|
||||
best[rt.SourceTier] = i
|
||||
}
|
||||
}
|
||||
for _, p := range c.provenTests.ProvenRestoreTests(ctx) {
|
||||
if p.SourceTier == "" {
|
||||
continue // not usable as a per-tier proof; the state layer already filters these
|
||||
}
|
||||
i, seen := best[p.SourceTier]
|
||||
if !seen {
|
||||
out = append(out, p)
|
||||
best[p.SourceTier] = len(out) - 1
|
||||
continue
|
||||
}
|
||||
if newerRestoreTest(p, out[i]) {
|
||||
out[i] = p
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// newerRestoreTest reports whether a was tested after b. An unparseable or absent timestamp is
|
||||
// treated as OLDER, so a malformed entry can never displace a good one.
|
||||
func newerRestoreTest(a, b RestoreTest) bool {
|
||||
ta, aok := parseRestoreTestedAt(a.TestedAt)
|
||||
tb, bok := parseRestoreTestedAt(b.TestedAt)
|
||||
if !aok {
|
||||
return false
|
||||
}
|
||||
if !bok {
|
||||
return true
|
||||
}
|
||||
return ta.After(tb)
|
||||
}
|
||||
|
||||
func parseRestoreTestedAt(s string) (time.Time, bool) {
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return t.UTC(), true
|
||||
}
|
||||
|
||||
// SetProvenRestoreTests wires the durable proof source. It is a setter rather than a constructor
|
||||
// argument because the persisted state is opened later in main() than the collector is built; the
|
||||
// same shape as the other late-wired seams here. **The wiring is asserted by an AST test** — the
|
||||
// method it feeds carried a doc comment naming a "host-report gauge" for weeks with no caller at
|
||||
// all, and this fix must not become the next instance of that.
|
||||
func (c *Collector) SetProvenRestoreTests(p ProvenRestoreTestReporter) { c.provenTests = p }
|
||||
|
||||
// collectPBSSnapshots reads the latest PBS snapshot inventory via the seam (nil → empty).
|
||||
func (c *Collector) collectPBSSnapshots(ctx context.Context) []PBSSnapshot {
|
||||
if c.pbs == nil {
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-54 §9 rule 6: the guest_net stanza must appear in a report built through the PRODUCTION collect
|
||||
// path, not only in a struct a test constructed. The v0.91.0 defect was exactly this gap — a seam
|
||||
// with green tests and no caller.
|
||||
|
||||
type fakeGuestNet struct{ st *GuestNetStatus }
|
||||
|
||||
func (f fakeGuestNet) GuestNetStatus(context.Context) *GuestNetStatus { return f.st }
|
||||
|
||||
func TestCollect_GuestNetOmittedWhenReporterNil(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
|
||||
r, err := c.Collect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Collect: %v", err)
|
||||
}
|
||||
if r.GuestNet != nil {
|
||||
t.Fatalf("no reporter wired → guest_net must be omitted, got %+v", r.GuestNet)
|
||||
}
|
||||
// And it must be absent from the WIRE, not merely nil in Go — an always-present empty stanza
|
||||
// would make "watchdog not wired" indistinguishable from "watchdog found nothing".
|
||||
b, _ := json.Marshal(r)
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := m["guest_net"]; ok {
|
||||
t.Fatalf("guest_net key present on the wire with no reporter wired: %s", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollect_GuestNetPopulatedWhenWired(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.92.0", quietLogger())
|
||||
c.SetGuestNetReporter(fakeGuestNet{st: &GuestNetStatus{
|
||||
CheckedAt: "2026-07-21T10:00:00Z",
|
||||
Guests: []GuestNetGuest{{
|
||||
VMID: 9201, State: "healthy", Mode: "dhcp", IP: "192.168.0.104",
|
||||
HasRoute: true, DHClientAlive: true, CheckedAt: "2026-07-21T10:00:00Z",
|
||||
}},
|
||||
}})
|
||||
r, err := c.Collect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Collect: %v", err)
|
||||
}
|
||||
if r.GuestNet == nil || len(r.GuestNet.Guests) != 1 {
|
||||
t.Fatalf("guest_net stanza missing from a collected report: %+v", r.GuestNet)
|
||||
}
|
||||
|
||||
// The wire keys are the contract the hub will read; pin the ones an operator diagnoses with.
|
||||
b, _ := json.Marshal(r)
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gn, ok := m["guest_net"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("guest_net missing or wrong shape on the wire: %s", b)
|
||||
}
|
||||
guests, ok := gn["guests"].([]any)
|
||||
if !ok || len(guests) != 1 {
|
||||
t.Fatalf("guest_net.guests wrong on the wire: %v", gn)
|
||||
}
|
||||
g := guests[0].(map[string]any)
|
||||
for _, key := range []string{"vmid", "state", "mode", "ip", "has_route", "dhclient_alive"} {
|
||||
if _, ok := g[key]; !ok {
|
||||
t.Fatalf("guest_net.guests[0] is missing the %q key: %v", key, g)
|
||||
}
|
||||
}
|
||||
if g["dhclient_alive"] != true {
|
||||
t.Fatalf("dhclient_alive must survive the round trip: %v", g)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeMgmtPlane is a MgmtPlaneReporter returning a fixed stanza (or nil).
|
||||
type fakeMgmtPlane struct{ st *MgmtPlaneStatus }
|
||||
|
||||
func (f fakeMgmtPlane) MgmtPlaneStatus(context.Context) *MgmtPlaneStatus { return f.st }
|
||||
|
||||
func TestCollect_MgmtPlaneOmittedWhenReporterNil(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger())
|
||||
r, err := c.Collect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Collect: %v", err)
|
||||
}
|
||||
if r.MgmtPlane != nil {
|
||||
t.Fatalf("no reporter wired → mgmt_plane must be omitted (nil), got %+v", r.MgmtPlane)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollect_MgmtPlanePopulatedWhenWired(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.71.0", quietLogger())
|
||||
c.SetMgmtPlaneReporter(fakeMgmtPlane{st: &MgmtPlaneStatus{
|
||||
PrivsepDirOK: true, SshdReachable: true, HealedRecently: true, PrivsepHealedAt: "2026-07-05T16:42:17Z",
|
||||
}})
|
||||
r, err := c.Collect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Collect: %v", err)
|
||||
}
|
||||
if r.MgmtPlane == nil {
|
||||
t.Fatal("wired reporter → mgmt_plane must be present")
|
||||
}
|
||||
if !r.MgmtPlane.HealedRecently || r.MgmtPlane.PrivsepHealedAt != "2026-07-05T16:42:17Z" {
|
||||
t.Fatalf("mgmt_plane not carried through: %+v", r.MgmtPlane)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeOOB is an OOBReporter returning a fixed stanza (or nil).
|
||||
type fakeOOB struct{ st *OOBStatus }
|
||||
|
||||
func (f fakeOOB) OOBStatus(context.Context) *OOBStatus { return f.st }
|
||||
|
||||
func TestCollect_OOBOmittedWhenNil(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.72.0", quietLogger())
|
||||
r, _ := c.Collect(context.Background())
|
||||
if r.OOB != nil {
|
||||
t.Fatalf("no reporter → oob omitted, got %+v", r.OOB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollect_OOBPopulatedWhenWired(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{}, nil, nil, nil, "h", "0.72.0", quietLogger())
|
||||
c.SetOOBReporter(fakeOOB{st: &OOBStatus{FelhomSshdActive: true, FelhomSshdPort: 8822, Reachable: true}})
|
||||
r, _ := c.Collect(context.Background())
|
||||
if r.OOB == nil || r.OOB.FelhomSshdPort != 8822 || !r.OOB.Reachable {
|
||||
t.Fatalf("oob not carried through: %+v", r.OOB)
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,14 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) {
|
||||
State: StorageStateAttached, Reachable: true, MountPath: "/mnt/usb-backup", TotalBytes: 2000000000000,
|
||||
Smart: SmartSummary{Health: SmartUnknown},
|
||||
},
|
||||
// A pbs target so the recipe's pbs coord has a storage.cfg row to resolve its namespace
|
||||
// from (R-106). Without one the fixture produces namespace_state=unknown while the golden
|
||||
// pins a resolved coord — key-set-equal but semantically a fiction.
|
||||
{
|
||||
Name: "felhom-pbs", Type: StorageTypePBS, DurableID: "repo+fp", Content: "backup",
|
||||
State: StorageStateAttached, Reachable: true, PBSNamespace: "felhom-spike",
|
||||
Smart: SmartSummary{Health: SmartUnknown},
|
||||
},
|
||||
},
|
||||
Backups: []Backup{
|
||||
{
|
||||
@@ -72,9 +80,13 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) {
|
||||
},
|
||||
AuditTail: []AuditEntry{},
|
||||
Cloudflared: Cloudflared{Status: "active"},
|
||||
// v0.119.0: host addresses. Populated so the bidirectional key-set guard exercises the new
|
||||
// element keys, not just the presence of the array.
|
||||
Addresses: []HostAddress{{Iface: "vmbr0", CIDR: "192.168.0.162/24"}},
|
||||
}
|
||||
// dr_recipe host-half: built from the same guest/storage/pbs facts (the production path).
|
||||
report.DRRecipe = BuildDRRecipeHostHalf(report.Guests, report.StorageTargets, report.PBSSnapshots)
|
||||
report.DRRecipe = BuildDRRecipeHostHalf(report.Guests, report.StorageTargets, report.PBSSnapshots,
|
||||
ConfiguredBackupTarget{StorageID: "usb-backup", Known: true})
|
||||
b, _ := json.Marshal(report)
|
||||
var got map[string]any
|
||||
json.Unmarshal(b, &got)
|
||||
@@ -97,6 +109,9 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) {
|
||||
assertSameKeys(t, "restore_tests[0]", firstElem(golden["restore_tests"]), firstElem(got["restore_tests"]))
|
||||
// slice-6-Phase-B addition — pbs_snapshots[0] key set.
|
||||
assertSameKeys(t, "pbs_snapshots[0]", firstElem(golden["pbs_snapshots"]), firstElem(got["pbs_snapshots"]))
|
||||
// v0.119.0 addition — addresses[0] key set (iface/cidr), the cross-repo wire for the hub's
|
||||
// Network card.
|
||||
assertSameKeys(t, "addresses[0]", firstElem(golden["addresses"]), firstElem(got["addresses"]))
|
||||
|
||||
// DR-recipe host-half — the agent's secret-free reconstruction-scaffolding section. Assert the
|
||||
// dr_recipe key set + each sub-array's element key set (the cross-repo wire pinned in the golden).
|
||||
@@ -106,6 +121,7 @@ func TestHostReport_ContractMatchesGolden(t *testing.T) {
|
||||
assertSameKeys(t, "dr_recipe.guests[0]", firstElem(field(grec, "guests")), firstElem(field(srec, "guests")))
|
||||
assertSameKeys(t, "dr_recipe.drives[0]", firstElem(field(grec, "drives")), firstElem(field(srec, "drives")))
|
||||
assertSameKeys(t, "dr_recipe.pve_storage[0]", firstElem(field(grec, "pve_storage")), firstElem(field(srec, "pve_storage")))
|
||||
assertSameKeys(t, "dr_recipe.backup_target", field(grec, "backup_target"), field(srec, "backup_target"))
|
||||
}
|
||||
|
||||
// field extracts a nested object value from a decoded JSON map (nil if absent/not a map).
|
||||
|
||||
+138
-10
@@ -30,6 +30,36 @@ import "sort"
|
||||
// ignore-unknown (encoding/json default) for forward-compat, mirroring storage_manifest.
|
||||
const DRRecipeVersion = 1
|
||||
|
||||
// Recipe field states (R-106/R-109). A recipe is read at the worst possible moment — by an operator
|
||||
// rebuilding a machine that is gone — so a field the agent cannot resolve must SAY SO rather than emit
|
||||
// a default, an empty string, or a plausible-looking placeholder. A guess read as fact costs more than
|
||||
// an admitted gap: it sends the restore at the wrong archive and nothing contradicts it. This is the
|
||||
// same cannot-tell-must-not-lie rule R-117 needed a third state for.
|
||||
const (
|
||||
DRStateResolved = "resolved"
|
||||
DRStateUnknown = "unknown"
|
||||
)
|
||||
|
||||
// Reasons a resolved-value field is unknown. Enum-shaped, never free text, so the wire stays pinnable
|
||||
// and TestDRRecipeHostHalf_NoSecrets has a fixed vocabulary to walk.
|
||||
const (
|
||||
// DRReasonNoBackupConfig: the collector was built without a backup-config seam, so the agent could
|
||||
// not consult the very config its own scheduler reads. Nothing is guessed.
|
||||
DRReasonNoBackupConfig = "agent_backup_config_unavailable"
|
||||
// DRReasonNoSuchStorage: the configured target id matches no storage this host observes. The id is
|
||||
// still recorded (it IS what the config says) and the state says it could not be corroborated.
|
||||
DRReasonNoSuchStorage = "not_a_known_storage"
|
||||
// DRReasonNoPBSStorage: snapshots exist but no pbs storage was observed, so there is no storage.cfg
|
||||
// row to read the namespace from.
|
||||
DRReasonNoPBSStorage = "no_pbs_storage_observed"
|
||||
)
|
||||
|
||||
// PBSRootNamespace is how the recipe spells PBS's root namespace. The PBS API spells it as the EMPTY
|
||||
// string (and `pct restore --ns root` would name a namespace that does not exist) — "root" is a display
|
||||
// convention this wire has always used, kept here so the field's meaning did not change under R-106.
|
||||
// Only a box with no `namespace` line in its pbs storage.cfg stanza ever emits it.
|
||||
const PBSRootNamespace = "root"
|
||||
|
||||
// DRRecipeHostHalf is the agent-emitted half (guest/drive/storage/PBS scaffolding). Derived entirely
|
||||
// from facts the report already collects — no new privileged reads.
|
||||
type DRRecipeHostHalf struct {
|
||||
@@ -38,6 +68,45 @@ type DRRecipeHostHalf struct {
|
||||
PBS *DRPBSCoord `json:"pbs,omitempty"`
|
||||
Drives []DRDrive `json:"drives"`
|
||||
PVEStorage []DRPVEStorage `json:"pve_storage"`
|
||||
// BackupTarget names WHICH storage holds the local whole-guest archives (R-109). Always present —
|
||||
// its own State field carries "I could not tell", so the section is never simply absent.
|
||||
BackupTarget *DRBackupTarget `json:"backup_target"`
|
||||
}
|
||||
|
||||
// DRBackupTarget answers the one question pve_storage cannot: of every storage listed there, WHICH one
|
||||
// does this box's primary backup tier actually write its whole-guest archives to?
|
||||
//
|
||||
// Before R-109 the recipe listed each storage's name/type/content and said nothing about the target.
|
||||
// That was harmless while the target was the well-known `local`; the 2026-07-28 vzdump-target move
|
||||
// ended that. Every box now carries TWO content=backup dir storages — `felhom-backup` (live) and
|
||||
// `local` (archives frozen at the move, never refreshed since) — and they are indistinguishable by
|
||||
// name, type and content alone. A restorer picking the frozen one gets a guest that restores cleanly
|
||||
// and is silently months out of date, which is the worst shape a backup defect can take.
|
||||
type DRBackupTarget struct {
|
||||
// State is DRStateResolved | DRStateUnknown. A reader MUST consult it before trusting StorageID:
|
||||
// the id is also recorded in one unknown case (see DRReasonNoSuchStorage).
|
||||
State string `json:"state"`
|
||||
// StorageID is the PVE storage id of the PRIMARY backup tier. Empty only when the config could not
|
||||
// be consulted at all.
|
||||
StorageID string `json:"storage_id,omitempty"`
|
||||
// MountPath is where that storage's archives land on the host — the disambiguation a restorer
|
||||
// actually needs, since it is what separates felhom-backup's /mnt/hdd_1 from local's /var/lib/vz.
|
||||
// "" for a pbs target (no host mount) and when unresolved.
|
||||
MountPath string `json:"mount_path,omitempty"`
|
||||
// Reason is why State is unknown (one of the DRReason* constants); "" when resolved.
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// ConfiguredBackupTarget is what the agent's own backup config says the PRIMARY tier writes to.
|
||||
//
|
||||
// Known=false is a REAL state, not a nil-guard: it means the collector was constructed without the
|
||||
// backup-config seam (the --selftest one-shots did exactly this before v0.118.0), and the recipe then
|
||||
// records unknown instead of inventing a target. Deliberately a struct rather than a `(string, bool)`
|
||||
// return — the (value, ok) shape is what made "errors degrade to unknown, never to no-backup"
|
||||
// unimplementable in newestArchiveOn, and this field has the same three-way reading.
|
||||
type ConfiguredBackupTarget struct {
|
||||
StorageID string
|
||||
Known bool
|
||||
}
|
||||
|
||||
// DRGuest is the sizing needed to recreate the LXC at the right size (GuestSpec, already on the wire).
|
||||
@@ -51,8 +120,21 @@ type DRGuest struct {
|
||||
// DRPBSCoord is WHERE the whole-CT snapshot lives — COORDINATES ONLY. The encryption key is escrow-only;
|
||||
// the access token is identity-escrow-only. Neither is here.
|
||||
type DRPBSCoord struct {
|
||||
RepoID string `json:"repo_id"` // the PVE pbs storage id (e.g. "felhom-pbs") — not a token
|
||||
Namespace string `json:"namespace"` // PBS namespace the restore targets
|
||||
RepoID string `json:"repo_id"` // the PVE pbs storage id (e.g. "felhom-pbs") — not a token
|
||||
// Namespace is the PBS namespace the restore targets, resolved from the pbs storage's storage.cfg
|
||||
// stanza — the same field `vzdump --storage <pbs>` makes PVE read, so the recipe cannot disagree
|
||||
// with the backup that produced the snapshot. PBSRootNamespace when the box has no namespace
|
||||
// configured; "" when NamespaceState is unknown.
|
||||
//
|
||||
// R-106: this used to come from the listed snapshot's own `ns`, which PBS does not echo per item once
|
||||
// the request is already namespace-scoped via `?ns=` (internal/pbs/client.go). The field was
|
||||
// therefore always empty, ToHub normalised empty → "root", and every per-customer box reported the
|
||||
// root namespace while its backups were really in `demo-hp` / `demo-felhom`.
|
||||
Namespace string `json:"namespace"`
|
||||
// NamespaceState is DRStateResolved | DRStateUnknown — consult it before trusting Namespace.
|
||||
NamespaceState string `json:"namespace_state"`
|
||||
// NamespaceReason is why NamespaceState is unknown; "" when resolved.
|
||||
NamespaceReason string `json:"namespace_reason,omitempty"`
|
||||
LatestSnapshotID string `json:"latest_snapshot_id"` // most-recent snapshot's backup_id (a coordinate)
|
||||
}
|
||||
|
||||
@@ -83,7 +165,7 @@ const driveIntentEnrolled = "enrolled"
|
||||
// it is unit-tested directly (no live reads). drives[] = the user-data external drives (usb/local-dir
|
||||
// with a durable-id); pve_storage[] = every storage target (the storage.cfg scaffolding); pbs = the
|
||||
// latest PBS snapshot's coordinates; guests[] = each guest's sizing (skip guests with no spec).
|
||||
func BuildDRRecipeHostHalf(guests []Guest, targets []StorageTarget, pbs []PBSSnapshot) *DRRecipeHostHalf {
|
||||
func BuildDRRecipeHostHalf(guests []Guest, targets []StorageTarget, pbs []PBSSnapshot, backupTarget ConfiguredBackupTarget) *DRRecipeHostHalf {
|
||||
h := &DRRecipeHostHalf{
|
||||
RecipeVersion: DRRecipeVersion,
|
||||
Guests: []DRGuest{},
|
||||
@@ -103,11 +185,14 @@ func BuildDRRecipeHostHalf(guests []Guest, targets []StorageTarget, pbs []PBSSna
|
||||
})
|
||||
}
|
||||
|
||||
var pbsRepoID string
|
||||
var pbsRepoID, pbsNamespace string
|
||||
var pbsStorageFound bool
|
||||
for _, t := range targets {
|
||||
h.PVEStorage = append(h.PVEStorage, DRPVEStorage{Name: t.Name, Type: t.Type, Content: t.Content})
|
||||
if t.Type == StorageTypePBS && pbsRepoID == "" {
|
||||
pbsRepoID = t.Name // the pbs storage id is a coordinate, not the key
|
||||
if t.Type == StorageTypePBS && !pbsStorageFound {
|
||||
pbsStorageFound = true
|
||||
pbsRepoID = t.Name // the pbs storage id is a coordinate, not the key
|
||||
pbsNamespace = t.PBSNamespace // storage.cfg's namespace — "" here means the ROOT namespace
|
||||
}
|
||||
if isUserDataDrive(t) {
|
||||
h.Drives = append(h.Drives, DRDrive{
|
||||
@@ -119,12 +204,41 @@ func BuildDRRecipeHostHalf(guests []Guest, targets []StorageTarget, pbs []PBSSna
|
||||
}
|
||||
}
|
||||
|
||||
if c := latestPBSCoord(pbs, pbsRepoID); c != nil {
|
||||
h.BackupTarget = resolveBackupTarget(targets, backupTarget)
|
||||
|
||||
if c := latestPBSCoord(pbs, pbsRepoID, pbsNamespace, pbsStorageFound); c != nil {
|
||||
h.PBS = c
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// resolveBackupTarget records WHICH storage the primary backup tier writes to (R-109), or records
|
||||
// explicitly that it could not tell. Three outcomes, and the two unknowns are deliberately distinct —
|
||||
// "I could not read my own config" and "my config names a storage that is not here" send an operator
|
||||
// to different places.
|
||||
//
|
||||
// MountPath prefers the live mount and falls back to the CONFIGURED path: during a rebuild the drive is
|
||||
// frequently absent, and when it is, MountPath empties out while ConfigPath is the only thing left that
|
||||
// still says which drive the row was about (the R-116 lesson). The storage's absence from the host is a
|
||||
// separate signal (E-2's backup_target_absent); it does not make the recipe's answer unknown, because
|
||||
// the question here is which storage.cfg row to restore FROM, and that is still known.
|
||||
func resolveBackupTarget(targets []StorageTarget, cfg ConfiguredBackupTarget) *DRBackupTarget {
|
||||
if !cfg.Known || cfg.StorageID == "" {
|
||||
return &DRBackupTarget{State: DRStateUnknown, Reason: DRReasonNoBackupConfig}
|
||||
}
|
||||
for _, t := range targets {
|
||||
if t.Name != cfg.StorageID {
|
||||
continue
|
||||
}
|
||||
mount := t.MountPath
|
||||
if mount == "" {
|
||||
mount = t.ConfigPath
|
||||
}
|
||||
return &DRBackupTarget{State: DRStateResolved, StorageID: cfg.StorageID, MountPath: mount}
|
||||
}
|
||||
return &DRBackupTarget{State: DRStateUnknown, StorageID: cfg.StorageID, Reason: DRReasonNoSuchStorage}
|
||||
}
|
||||
|
||||
// isUserDataDrive selects the external user-data drives the recipe enumerates (felhom-usb / felhom-flash
|
||||
// class): a usb or local-dir storage with a filesystem-UUID durable id and a host mount path. local /
|
||||
// lvmthin / pbs / nfs / cifs are scaffolding (they land in pve_storage) but are not user-data drives.
|
||||
@@ -137,16 +251,30 @@ func isUserDataDrive(t StorageTarget) bool {
|
||||
|
||||
// latestPBSCoord picks the most-recent snapshot (lexical max of the RFC3339 backup_time) and returns
|
||||
// its coordinates. Returns nil when there is no snapshot to target.
|
||||
func latestPBSCoord(snaps []PBSSnapshot, repoID string) *DRPBSCoord {
|
||||
//
|
||||
// The namespace comes from the pbs STORAGE (storage.cfg), never from the snapshot — see DRPBSCoord's
|
||||
// Namespace comment for why the snapshot's own field cannot answer it (R-106). storageFound=false with
|
||||
// snapshots present is a genuine unknown: something listed snapshots, but there is no storage row to
|
||||
// read a namespace from, so the recipe says so rather than defaulting to root.
|
||||
func latestPBSCoord(snaps []PBSSnapshot, repoID, namespace string, storageFound bool) *DRPBSCoord {
|
||||
if len(snaps) == 0 {
|
||||
return nil
|
||||
}
|
||||
sorted := append([]PBSSnapshot(nil), snaps...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].BackupTime > sorted[j].BackupTime })
|
||||
latest := sorted[0]
|
||||
return &DRPBSCoord{
|
||||
c := &DRPBSCoord{
|
||||
RepoID: repoID,
|
||||
Namespace: latest.Namespace,
|
||||
LatestSnapshotID: latest.BackupID,
|
||||
NamespaceState: DRStateUnknown,
|
||||
NamespaceReason: DRReasonNoPBSStorage,
|
||||
}
|
||||
if storageFound {
|
||||
c.NamespaceState, c.NamespaceReason = DRStateResolved, ""
|
||||
// An empty configured namespace is not a missing answer — it IS the root namespace.
|
||||
if c.Namespace = namespace; c.Namespace == "" {
|
||||
c.Namespace = PBSRootNamespace
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -30,7 +32,7 @@ func TestBuildDRRecipeHostHalf(t *testing.T) {
|
||||
{Namespace: "root", BackupID: "9201", BackupTime: "2026-06-16T08:00:00Z"}, // latest
|
||||
}
|
||||
|
||||
h := BuildDRRecipeHostHalf(guests, targets, pbs)
|
||||
h := BuildDRRecipeHostHalf(guests, targets, pbs, ConfiguredBackupTarget{StorageID: "felhom-flash", Known: true})
|
||||
|
||||
if h.RecipeVersion != 1 {
|
||||
t.Errorf("recipe_version=%d, want 1", h.RecipeVersion)
|
||||
@@ -68,7 +70,8 @@ func TestBuildDRRecipeHostHalf(t *testing.T) {
|
||||
|
||||
// TestBuildDRRecipeHostHalf_NoPBS: no snapshots → pbs omitted (nil), no panic.
|
||||
func TestBuildDRRecipeHostHalf_NoPBS(t *testing.T) {
|
||||
h := BuildDRRecipeHostHalf(nil, []StorageTarget{{Name: "local", Type: StorageTypeLocal}}, nil)
|
||||
h := BuildDRRecipeHostHalf(nil, []StorageTarget{{Name: "local", Type: StorageTypeLocal}}, nil,
|
||||
ConfiguredBackupTarget{StorageID: "local", Known: true})
|
||||
if h.PBS != nil {
|
||||
t.Errorf("pbs should be nil with no snapshots, got %+v", h.PBS)
|
||||
}
|
||||
@@ -89,6 +92,7 @@ func TestDRRecipeHostHalf_V1DriveShape(t *testing.T) {
|
||||
MountPath: "/mnt/felhom-usb", TotalBytes: 931 << 30},
|
||||
},
|
||||
nil,
|
||||
ConfiguredBackupTarget{StorageID: "felhom-usb", Known: true},
|
||||
)
|
||||
if len(h.Drives) != 1 {
|
||||
t.Fatalf("want 1 drive, got %d", len(h.Drives))
|
||||
@@ -124,6 +128,7 @@ func TestDRRecipeHostHalf_NoSecrets(t *testing.T) {
|
||||
{Name: "felhom-usb", Type: StorageTypeUSB, DurableID: "uuid:da9e7089", Role: "bulk-data", MountPath: "/mnt/felhom-usb", TotalBytes: 1},
|
||||
},
|
||||
[]PBSSnapshot{{Namespace: "root", BackupID: "9201", BackupTime: "2026-06-16T08:00:00Z"}},
|
||||
ConfiguredBackupTarget{StorageID: "felhom-usb", Known: true},
|
||||
)
|
||||
b, err := json.Marshal(h)
|
||||
if err != nil {
|
||||
@@ -132,6 +137,294 @@ func TestDRRecipeHostHalf_NoSecrets(t *testing.T) {
|
||||
assertNoSecretKeys(t, b)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// R-106 / R-109 — the recipe records the RESOLVED backup target and the REAL PBS namespace.
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
// capturedDemoFelhomTargets is the storage set demo-felhom really had on 2026-07-30, not an invented
|
||||
// one. PROVENANCE — every field was captured, none composed:
|
||||
//
|
||||
// - names/types/contents: the pve_storage block of the box's own PRE-FIX recipe, downloaded from the
|
||||
// hub at GET /customers/demo-felhom/dr-recipe.json (agent v0.115.0).
|
||||
// - paths + is_mountpoint + the pbs namespace: `cat /etc/pve/storage.cfg` on felhom-pve, same day —
|
||||
// `dir: local path /var/lib/vz`, `dir: felhom-backup path /mnt/hdd_1 is_mountpoint 1`,
|
||||
// `pbs: felhom-pbs ... namespace demo-felhom`.
|
||||
//
|
||||
// THE AMBIGUITY THIS PINS IS REAL, and assertBackupCandidateAmbiguity below refuses to let the fixture
|
||||
// quietly lose it: `local` and `felhom-backup` BOTH carry content=backup, and since the 2026-07-28
|
||||
// vzdump-target move `local` holds archives frozen at that date. Naming the wrong one restores a guest
|
||||
// that is silently months stale.
|
||||
func capturedDemoFelhomTargets() []StorageTarget {
|
||||
return []StorageTarget{
|
||||
{Name: "local-lvm", Type: StorageTypeLVMThin, DurableID: "pve/data", Content: "images,rootdir"},
|
||||
{
|
||||
Name: "felhom-backup", Type: StorageTypeLocalDir, Content: "backup",
|
||||
DurableID: "uuid:47a3361a-91e0-4831-a69d-27f540ed3f48",
|
||||
MountPath: "/mnt/hdd_1", ConfigPath: "/mnt/hdd_1", TotalBytes: 983351140352,
|
||||
},
|
||||
{
|
||||
Name: "felhom-pbs", Type: StorageTypePBS, Content: "backup",
|
||||
DurableID: "repo+fp", PBSNamespace: "demo-felhom",
|
||||
},
|
||||
// The decoy: same content, plausible name, historically THE vzdump target. ConfigPath only —
|
||||
// `local` lives on the LVM root and is not its own mount, so the observer leaves MountPath empty.
|
||||
{Name: "local", Type: StorageTypeLocal, Content: "backup,import,vztmpl,iso", ConfigPath: "/var/lib/vz"},
|
||||
}
|
||||
}
|
||||
|
||||
// capturedDemoFelhomSnapshots mirrors what the box's pre-fix recipe carried: latest_snapshot_id "9201".
|
||||
// Namespace is deliberately EMPTY on every element — that is exactly what the PBS API returns once the
|
||||
// list is namespace-scoped via `?ns=`, and it is the input that used to become the bogus "root".
|
||||
func capturedDemoFelhomSnapshots() []PBSSnapshot {
|
||||
return []PBSSnapshot{
|
||||
{Namespace: "", BackupID: "9201", BackupTime: "2026-07-29T22:00:00Z"},
|
||||
{Namespace: "", BackupID: "9201", BackupTime: "2026-07-30T22:00:00Z"}, // latest
|
||||
}
|
||||
}
|
||||
|
||||
// assertBackupCandidateAmbiguity fails if the fixture stopped containing TWO plausible content=backup
|
||||
// storages. Without this the consequence test below could pass on a fixture with only one candidate —
|
||||
// which is precisely the hollow shape that let two defects ship green earlier in this arc.
|
||||
func assertBackupCandidateAmbiguity(t *testing.T, h *DRRecipeHostHalf) {
|
||||
t.Helper()
|
||||
var candidates []string
|
||||
for _, s := range h.PVEStorage {
|
||||
if strings.Contains(s.Content, "backup") && (s.Type == StorageTypeLocalDir || s.Type == StorageTypeLocal) {
|
||||
candidates = append(candidates, s.Name)
|
||||
}
|
||||
}
|
||||
if len(candidates) < 2 {
|
||||
t.Fatalf("fixture no longer poses the R-109 problem: want >=2 content=backup dir storages, got %v", candidates)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDRRecipe_BackupTargetNamesTheLiveStorage is THE consequence assertion for R-109: given a box that
|
||||
// really carries two content=backup dir storages, the generated recipe names the LIVE one, gives its
|
||||
// mountpoint, and does not name the frozen one. Not "the function returned a non-empty string".
|
||||
func TestDRRecipe_BackupTargetNamesTheLiveStorage(t *testing.T) {
|
||||
h := BuildDRRecipeHostHalf(nil, capturedDemoFelhomTargets(), capturedDemoFelhomSnapshots(),
|
||||
ConfiguredBackupTarget{StorageID: "felhom-backup", Known: true})
|
||||
|
||||
assertBackupCandidateAmbiguity(t, h)
|
||||
|
||||
bt := h.BackupTarget
|
||||
if bt == nil {
|
||||
t.Fatal("backup_target is absent — the recipe still cannot say where the local archives are (R-109)")
|
||||
}
|
||||
if bt.State != DRStateResolved {
|
||||
t.Errorf("state=%q want %q (reason=%q)", bt.State, DRStateResolved, bt.Reason)
|
||||
}
|
||||
if bt.StorageID != "felhom-backup" {
|
||||
t.Errorf("storage_id=%q — the recipe must name the LIVE target, not %q", bt.StorageID, "felhom-backup")
|
||||
}
|
||||
if bt.MountPath != "/mnt/hdd_1" {
|
||||
t.Errorf("mount_path=%q want /mnt/hdd_1 — the mountpoint is what separates it from local's /var/lib/vz", bt.MountPath)
|
||||
}
|
||||
// Unambiguous: the frozen decoy must not be what the field names.
|
||||
if bt.StorageID == "local" || bt.MountPath == "/var/lib/vz" {
|
||||
t.Errorf("recipe names the FROZEN target (%q at %q) — a restore from it is silently stale", bt.StorageID, bt.MountPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDRRecipe_PBSNamespaceIsThePerCustomerOne is the consequence assertion for R-106: the recipe carries
|
||||
// the namespace the box's backups actually live in, resolved from storage.cfg, and specifically NOT the
|
||||
// "root" that every box used to report.
|
||||
func TestDRRecipe_PBSNamespaceIsThePerCustomerOne(t *testing.T) {
|
||||
h := BuildDRRecipeHostHalf(nil, capturedDemoFelhomTargets(), capturedDemoFelhomSnapshots(),
|
||||
ConfiguredBackupTarget{StorageID: "felhom-backup", Known: true})
|
||||
|
||||
if h.PBS == nil {
|
||||
t.Fatal("pbs coord absent with snapshots present")
|
||||
}
|
||||
if h.PBS.Namespace == PBSRootNamespace {
|
||||
t.Errorf("namespace=%q — this is the R-106 symptom: the snapshot's empty ns normalised to root "+
|
||||
"while the box's backups are in demo-felhom", h.PBS.Namespace)
|
||||
}
|
||||
if h.PBS.Namespace != "demo-felhom" {
|
||||
t.Errorf("namespace=%q want demo-felhom (storage.cfg's `namespace` on the pbs storage)", h.PBS.Namespace)
|
||||
}
|
||||
if h.PBS.NamespaceState != DRStateResolved {
|
||||
t.Errorf("namespace_state=%q want %q (reason=%q)", h.PBS.NamespaceState, DRStateResolved, h.PBS.NamespaceReason)
|
||||
}
|
||||
if h.PBS.RepoID != "felhom-pbs" || h.PBS.LatestSnapshotID != "9201" {
|
||||
t.Errorf("coord drifted: repo=%q snapshot=%q", h.PBS.RepoID, h.PBS.LatestSnapshotID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDRRecipe_PBSNamespaceRootIsResolvedNotUnknown: a box with a pbs storage and NO namespace line is
|
||||
// genuinely in the root namespace. That is an answer, not a gap — it must read resolved/"root", so the
|
||||
// honest root case is never confused with "I could not tell".
|
||||
func TestDRRecipe_PBSNamespaceRootIsResolvedNotUnknown(t *testing.T) {
|
||||
h := BuildDRRecipeHostHalf(nil,
|
||||
[]StorageTarget{{Name: "felhom-pbs", Type: StorageTypePBS, Content: "backup", PBSNamespace: ""}},
|
||||
capturedDemoFelhomSnapshots(),
|
||||
ConfiguredBackupTarget{StorageID: "felhom-pbs", Known: true})
|
||||
|
||||
if h.PBS.NamespaceState != DRStateResolved {
|
||||
t.Errorf("namespace_state=%q — an unconfigured namespace IS the root namespace, not an unknown", h.PBS.NamespaceState)
|
||||
}
|
||||
if h.PBS.Namespace != PBSRootNamespace {
|
||||
t.Errorf("namespace=%q want %q", h.PBS.Namespace, PBSRootNamespace)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDRRecipe_BackupTargetUnknownWhenConfigUnavailable is the WRONG case: the agent could not consult
|
||||
// its own backup config. The recipe must say so explicitly and emit NO storage_id key at all — an
|
||||
// absent value must not be representable as a plausible-looking answer.
|
||||
func TestDRRecipe_BackupTargetUnknownWhenConfigUnavailable(t *testing.T) {
|
||||
h := BuildDRRecipeHostHalf(nil, capturedDemoFelhomTargets(), nil, ConfiguredBackupTarget{})
|
||||
|
||||
bt := h.BackupTarget
|
||||
if bt == nil {
|
||||
t.Fatal("backup_target must be PRESENT and say unknown, not vanish")
|
||||
}
|
||||
if bt.State != DRStateUnknown || bt.Reason != DRReasonNoBackupConfig {
|
||||
t.Errorf("state=%q reason=%q want %q/%q", bt.State, bt.Reason, DRStateUnknown, DRReasonNoBackupConfig)
|
||||
}
|
||||
// Absence recorded as absence: no id, and no id KEY on the wire.
|
||||
if bt.StorageID != "" {
|
||||
t.Errorf("storage_id=%q — an unresolvable target must not be filled in", bt.StorageID)
|
||||
}
|
||||
b, err := json.Marshal(bt)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var keys map[string]json.RawMessage
|
||||
if err := json.Unmarshal(b, &keys); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, banned := range []string{"storage_id", "mount_path"} {
|
||||
if _, ok := keys[banned]; ok {
|
||||
t.Errorf("unknown backup_target must not carry a %q key; got %s", banned, b)
|
||||
}
|
||||
}
|
||||
// And nothing in it may read as one of the real candidates.
|
||||
for _, decoy := range []string{"felhom-backup", "local", "/var/lib/vz", "/mnt/hdd_1"} {
|
||||
if strings.Contains(string(b), decoy) {
|
||||
t.Errorf("unknown backup_target leaked a plausible value %q: %s", decoy, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDRRecipe_BackupTargetUnknownWhenStorageMissing: the config names a storage this host does not
|
||||
// have. That is unknown for a DIFFERENT reason — and the configured id IS still recorded, because
|
||||
// "config says felhom-backup, no such storage here" sends an operator somewhere useful while silence
|
||||
// does not.
|
||||
func TestDRRecipe_BackupTargetUnknownWhenStorageMissing(t *testing.T) {
|
||||
targets := []StorageTarget{{Name: "local", Type: StorageTypeLocal, Content: "backup", ConfigPath: "/var/lib/vz"}}
|
||||
h := BuildDRRecipeHostHalf(nil, targets, nil, ConfiguredBackupTarget{StorageID: "felhom-backup", Known: true})
|
||||
|
||||
bt := h.BackupTarget
|
||||
if bt.State != DRStateUnknown || bt.Reason != DRReasonNoSuchStorage {
|
||||
t.Errorf("state=%q reason=%q want %q/%q", bt.State, bt.Reason, DRStateUnknown, DRReasonNoSuchStorage)
|
||||
}
|
||||
if bt.StorageID != "felhom-backup" {
|
||||
t.Errorf("storage_id=%q want the CONFIGURED id recorded even though it matched nothing", bt.StorageID)
|
||||
}
|
||||
// It must NOT silently fall back to the only content=backup storage present.
|
||||
if bt.StorageID == "local" || bt.MountPath == "/var/lib/vz" {
|
||||
t.Error("resolution fell back to the wrong storage instead of reporting unknown")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDRRecipe_PBSNamespaceUnknownWithoutPBSStorage: snapshots exist but no pbs storage was observed, so
|
||||
// there is no storage.cfg row to read a namespace from. The recipe must NOT default to root — that
|
||||
// default is the entire R-106 defect.
|
||||
func TestDRRecipe_PBSNamespaceUnknownWithoutPBSStorage(t *testing.T) {
|
||||
h := BuildDRRecipeHostHalf(nil,
|
||||
[]StorageTarget{{Name: "local", Type: StorageTypeLocal, Content: "backup"}},
|
||||
capturedDemoFelhomSnapshots(),
|
||||
ConfiguredBackupTarget{StorageID: "local", Known: true})
|
||||
|
||||
if h.PBS == nil {
|
||||
t.Fatal("pbs coord should still be emitted (the snapshot id is a real coordinate)")
|
||||
}
|
||||
if h.PBS.NamespaceState != DRStateUnknown || h.PBS.NamespaceReason != DRReasonNoPBSStorage {
|
||||
t.Errorf("namespace_state=%q reason=%q want %q/%q",
|
||||
h.PBS.NamespaceState, h.PBS.NamespaceReason, DRStateUnknown, DRReasonNoPBSStorage)
|
||||
}
|
||||
if h.PBS.Namespace != "" {
|
||||
t.Errorf("namespace=%q — with no storage row to read, the field must be empty, never %q",
|
||||
h.PBS.Namespace, PBSRootNamespace)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDRRecipe_BackupTargetUsesConfigPathWhenDeviceGone is the DR-shaped case: the recipe is read while
|
||||
// the target drive is absent, so MountPath has emptied out. ConfigPath is then the only thing that still
|
||||
// says where the archives live (the R-116 lesson) — and the target is still RESOLVED, because which
|
||||
// storage.cfg row to restore from is known regardless of whether its device is currently present.
|
||||
func TestDRRecipe_BackupTargetUsesConfigPathWhenDeviceGone(t *testing.T) {
|
||||
targets := []StorageTarget{{
|
||||
Name: "felhom-backup", Type: StorageTypeLocalDir, Content: "backup",
|
||||
MountPath: "", ConfigPath: "/mnt/hdd_1", // device gone: observer empties MountPath, keeps ConfigPath
|
||||
}}
|
||||
h := BuildDRRecipeHostHalf(nil, targets, nil, ConfiguredBackupTarget{StorageID: "felhom-backup", Known: true})
|
||||
|
||||
bt := h.BackupTarget
|
||||
if bt.State != DRStateResolved {
|
||||
t.Errorf("state=%q — an absent device does not make the TARGET unknown", bt.State)
|
||||
}
|
||||
if bt.MountPath != "/mnt/hdd_1" {
|
||||
t.Errorf("mount_path=%q want the configured path /mnt/hdd_1", bt.MountPath)
|
||||
}
|
||||
}
|
||||
|
||||
// fakePBSReporter is a PBSReporter returning fixed snapshots (the verify loop's seam).
|
||||
type fakePBSReporter struct{ snaps []PBSSnapshot }
|
||||
|
||||
func (f fakePBSReporter) PBSSnapshots(context.Context) []PBSSnapshot { return f.snaps }
|
||||
|
||||
// TestCollectDRRecipe_ProductionPath runs the REAL generation path — Collector.Collect(), the method the
|
||||
// daemon calls every cycle — rather than BuildDRRecipeHostHalf directly. It is here because both defects
|
||||
// this file fixes were invisible to a direct-call test: the namespace one lived in what the observer put
|
||||
// on StorageTarget, and the target one lived in whether anything wired the config seam at all. A seam
|
||||
// that is correct and never wired is the failure mode this repo has hit four times.
|
||||
func TestCollectDRRecipe_ProductionPath(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
obs := fakeObserver{targets: capturedDemoFelhomTargets()}
|
||||
pbsRep := fakePBSReporter{snaps: capturedDemoFelhomSnapshots()}
|
||||
|
||||
c := NewCollector(px, fakeProber{status: "active"}, obs, nil, nil, pbsRep, "h", "0.118.0", quietLogger())
|
||||
c.SetBackupTargetResolver(func() ConfiguredBackupTarget {
|
||||
return ConfiguredBackupTarget{StorageID: "felhom-backup", Known: true}
|
||||
})
|
||||
|
||||
r, err := c.Collect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Collect: %v", err)
|
||||
}
|
||||
if r.DRRecipe == nil {
|
||||
t.Fatal("collect produced no dr_recipe")
|
||||
}
|
||||
if bt := r.DRRecipe.BackupTarget; bt == nil || bt.State != DRStateResolved || bt.StorageID != "felhom-backup" {
|
||||
t.Errorf("backup_target through Collect = %+v, want resolved/felhom-backup", bt)
|
||||
}
|
||||
if p := r.DRRecipe.PBS; p == nil || p.Namespace != "demo-felhom" || p.NamespaceState != DRStateResolved {
|
||||
t.Errorf("pbs namespace through Collect = %+v, want demo-felhom/resolved", p)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCollectDRRecipe_UnwiredSeamReportsUnknown: a Collector built WITHOUT the resolver (every
|
||||
// --selftest one-shot did exactly this before v0.118.0) must produce an explicit unknown. This is the
|
||||
// test that would have caught shipping the seam without wiring it.
|
||||
func TestCollectDRRecipe_UnwiredSeamReportsUnknown(t *testing.T) {
|
||||
px := &fakePx{node: "n", ns: newTestNodeStatus()}
|
||||
c := NewCollector(px, fakeProber{status: "active"}, fakeObserver{targets: capturedDemoFelhomTargets()},
|
||||
nil, nil, nil, "h", "0.118.0", quietLogger())
|
||||
|
||||
r, err := c.Collect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Collect: %v", err)
|
||||
}
|
||||
bt := r.DRRecipe.BackupTarget
|
||||
if bt == nil || bt.State != DRStateUnknown || bt.Reason != DRReasonNoBackupConfig {
|
||||
t.Fatalf("unwired resolver must yield unknown/%s, got %+v", DRReasonNoBackupConfig, bt)
|
||||
}
|
||||
if bt.StorageID != "" {
|
||||
t.Errorf("unwired resolver invented a target %q", bt.StorageID)
|
||||
}
|
||||
}
|
||||
|
||||
// assertNoSecretKeys walks decoded JSON and fails on any object key matching secretNameRe. Shared by
|
||||
// the agent boundary assertions. (durable_id/repo_id/latest_snapshot_id are identifiers/coordinates —
|
||||
// none match the credential regex.)
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/netip"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Host addresses (v0.119.0) — "which addresses does this box actually hold?"
|
||||
//
|
||||
// The hub could not answer that at all: HostMetrics carried node/cpu/mem/disk/load/uptime/temp and
|
||||
// no address of any kind, so the LAN IP of a managed host was invisible in every operator surface.
|
||||
// Two sources looked like answers and are not: `lan_resolver.host_ip` is an OPTIONAL config value
|
||||
// (absent unless that feature is configured), and DeriveHostIP(local_api.listen_addr) yields the
|
||||
// R-50 island literal 169.254.253.1 — a link-local address that is the same on every box. Reporting
|
||||
// either would have produced a confident wrong answer, which is worse than the blank it replaces.
|
||||
//
|
||||
// This reads the kernel's own view instead, and it issues NO block I/O (the CLAUDE.md health-check
|
||||
// rule): net.Interfaces() is a netlink/procfs read, needs no privilege, and touches no filesystem.
|
||||
|
||||
// HostAddress is one routable address the host holds, tagged with the interface carrying it.
|
||||
//
|
||||
// Deliberately iface+cidr rather than a single `lan_ip`: a Proxmox host legitimately holds several
|
||||
// (a management bridge, a tailnet, the WG tunnel), and picking one of them to call "the" LAN IP is a
|
||||
// guess the agent is not entitled to make — on a box whose management bridge is not vmbr0 that guess
|
||||
// is silently wrong. The agent reports what exists; the hub does the labelling.
|
||||
type HostAddress struct {
|
||||
Iface string `json:"iface"` // e.g. "vmbr0", "wg-felhom", "tailscale0"
|
||||
CIDR string `json:"cidr"` // e.g. "192.168.0.162/24" — prefix length kept, it is operator-relevant
|
||||
}
|
||||
|
||||
// ifaceAddrs is one enumerated interface: the ONLY facts the filter needs. Keeping the seam this
|
||||
// narrow is what lets the filter be tested against real measured shapes without a network stack.
|
||||
type ifaceAddrs struct {
|
||||
Name string
|
||||
Up bool
|
||||
Loopback bool
|
||||
CIDRs []string
|
||||
}
|
||||
|
||||
// AddressEnumerator returns the host's interfaces. Injectable so the filter can be driven with the
|
||||
// shapes measured on real hardware (see hostaddr_test.go) instead of whatever the test box happens
|
||||
// to have.
|
||||
type AddressEnumerator func() ([]ifaceAddrs, error)
|
||||
|
||||
// systemInterfaces is the production enumerator: the kernel's interface table.
|
||||
func systemInterfaces() ([]ifaceAddrs, error) {
|
||||
ifaces, err := net.Interfaces()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]ifaceAddrs, 0, len(ifaces))
|
||||
for _, i := range ifaces {
|
||||
e := ifaceAddrs{
|
||||
Name: i.Name,
|
||||
Up: i.Flags&net.FlagUp != 0,
|
||||
Loopback: i.Flags&net.FlagLoopback != 0,
|
||||
}
|
||||
// A per-interface error is not fatal: one unreadable interface must not cost the report
|
||||
// every other address (serve-degraded, as everywhere else in the collector).
|
||||
addrs, aerr := i.Addrs()
|
||||
if aerr != nil {
|
||||
out = append(out, e)
|
||||
continue
|
||||
}
|
||||
for _, a := range addrs {
|
||||
e.CIDRs = append(e.CIDRs, a.String())
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// filterHostAddresses keeps every GLOBAL UNICAST address on an up, non-loopback interface.
|
||||
//
|
||||
// IsGlobalUnicast() is the whole rule, and it was chosen by measuring both demo hosts rather than by
|
||||
// listing interface names to exclude. It drops, in one predicate:
|
||||
// - loopback (127.0.0.1, ::1)
|
||||
// - IPv6 link-local (fe80::/10) — every bridge carries one, pure noise
|
||||
// - IPv4 link-local (169.254.0.0/16) — which is exactly the R-50 island address on vmbr9, an
|
||||
// identical constant on every box and therefore actively misleading if surfaced
|
||||
//
|
||||
// It needs NO veth/fwbr/tap denylist: on a Proxmox host that per-guest plumbing carries no IP at
|
||||
// all, so it self-excludes by having nothing to report. Verified on demo-felhom and demo-hp —
|
||||
// veth9201i0/i1, fwbr*, and the unused NICs all appear in `ip link` and in no `ip addr` output.
|
||||
//
|
||||
// What survives on a real box: vmbr0's LAN address, wg-felhom's tunnel address, and tailscale0's
|
||||
// tailnet addresses. All three are true and useful; none is labelled here.
|
||||
func filterHostAddresses(in []ifaceAddrs) []HostAddress {
|
||||
out := []HostAddress{}
|
||||
for _, i := range in {
|
||||
if i.Loopback || !i.Up {
|
||||
continue
|
||||
}
|
||||
for _, c := range i.CIDRs {
|
||||
p, err := netip.ParsePrefix(c)
|
||||
if err != nil {
|
||||
continue // not a CIDR we understand — skip it, never fail the report
|
||||
}
|
||||
if !p.Addr().IsGlobalUnicast() {
|
||||
continue
|
||||
}
|
||||
out = append(out, HostAddress{Iface: i.Name, CIDR: p.String()})
|
||||
}
|
||||
}
|
||||
// Deterministic order so a report diff reflects a real change, not interface-table ordering.
|
||||
sort.Slice(out, func(a, b int) bool {
|
||||
if out[a].Iface != out[b].Iface {
|
||||
return out[a].Iface < out[b].Iface
|
||||
}
|
||||
return out[a].CIDR < out[b].CIDR
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// collectAddresses is the collector's entry point. It returns a non-nil slice so the field always
|
||||
// marshals as [] — an absent key and "this box has no routable address" must not look alike to the
|
||||
// hub, and [] is the honest encoding of the latter.
|
||||
func (c *Collector) collectAddresses() []HostAddress {
|
||||
enum := c.addrEnum
|
||||
if enum == nil {
|
||||
// Default to the REAL enumerator, deliberately inverting the nil-reporter-means-off
|
||||
// convention used by the optional stanzas above. Those gate on a config feature; this has
|
||||
// no dependency and no feature flag, so a forgotten wiring call in main.go would produce a
|
||||
// silently empty field — the inert-seam failure this repo has shipped four times.
|
||||
enum = systemInterfaces
|
||||
}
|
||||
ifaces, err := enum()
|
||||
if err != nil {
|
||||
c.logger.Warn("host addresses: interface enumeration failed", "err", err)
|
||||
return []HostAddress{}
|
||||
}
|
||||
return filterHostAddresses(ifaces)
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The fixtures below are MEASURED, not invented: `ip -o addr show` on demo-felhom (N100) and
|
||||
// demo-hp (HP t740) on 2026-07-31, transcribed verbatim including the interfaces that carry no
|
||||
// address. That matters — the filter's claim that it needs no veth/fwbr denylist rests on those
|
||||
// interfaces genuinely having nothing to report, and a hand-written fixture that omitted them would
|
||||
// have proved the claim by assuming it.
|
||||
|
||||
// demoFelhomIfaces is demo-felhom's real interface table.
|
||||
func demoFelhomIfaces() []ifaceAddrs {
|
||||
return []ifaceAddrs{
|
||||
{Name: "lo", Up: true, Loopback: true, CIDRs: []string{"127.0.0.1/8", "::1/128"}},
|
||||
{Name: "enp1s0", Up: false}, // physical NIC, no address
|
||||
{Name: "wlp2s0", Up: false}, // wifi, no address
|
||||
{Name: "tailscale0", Up: true, CIDRs: []string{
|
||||
"100.70.170.35/32", "fd7a:115c:a1e0::5236:aa24/128", "fe80::4197:26fc:ccba:b0d9/64"}},
|
||||
{Name: "vmbr0", Up: true, CIDRs: []string{"192.168.0.162/24", "fe80::6a1d:efff:fe5d:a664/64"}},
|
||||
{Name: "veth9201i0", Up: true}, // per-guest plumbing — no address
|
||||
{Name: "veth9201i1", Up: true}, // per-guest plumbing — no address
|
||||
{Name: "vmbr9", Up: true, CIDRs: []string{"169.254.253.1/30", "fe80::48d4:f6ff:fe05:2f98/64"}},
|
||||
{Name: "wg-felhom", Up: true, CIDRs: []string{"10.77.0.2/32"}},
|
||||
}
|
||||
}
|
||||
|
||||
func hasAddr(got []HostAddress, iface, cidr string) bool {
|
||||
for _, a := range got {
|
||||
if a.Iface == iface && a.CIDR == cidr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func flatten(got []HostAddress) string {
|
||||
var b strings.Builder
|
||||
for _, a := range got {
|
||||
b.WriteString(a.Iface + "=" + a.CIDR + " ")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// The LAN address is the whole point of the feature — it must survive the filter.
|
||||
// RED-PROOF 1: drop the `!p.Addr().IsGlobalUnicast()` continue → the vmbr9 + fe80 assertions below
|
||||
// go red (the LAN one still passes, which is exactly why the negatives are asserted too).
|
||||
func TestFilterHostAddresses_RealHost(t *testing.T) {
|
||||
got := filterHostAddresses(demoFelhomIfaces())
|
||||
|
||||
// --- what MUST be there ---
|
||||
if !hasAddr(got, "vmbr0", "192.168.0.162/24") {
|
||||
t.Fatalf("the LAN address was filtered away — the feature reports nothing: %s", flatten(got))
|
||||
}
|
||||
if !hasAddr(got, "wg-felhom", "10.77.0.2/32") {
|
||||
t.Errorf("the WireGuard address was filtered away: %s", flatten(got))
|
||||
}
|
||||
if !hasAddr(got, "tailscale0", "100.70.170.35/32") {
|
||||
t.Errorf("the tailnet address was filtered away: %s", flatten(got))
|
||||
}
|
||||
|
||||
// --- what MUST NOT be there, each for its own reason ---
|
||||
for _, bad := range []struct{ iface, cidr, why string }{
|
||||
{"lo", "127.0.0.1/8", "loopback is not an address of the host on any network"},
|
||||
{"lo", "::1/128", "IPv6 loopback"},
|
||||
{"vmbr9", "169.254.253.1/30", "the R-50 island literal — IDENTICAL on every box, so surfacing it is actively misleading"},
|
||||
{"vmbr0", "fe80::6a1d:efff:fe5d:a664/64", "IPv6 link-local, one per bridge, pure noise"},
|
||||
{"tailscale0", "fe80::4197:26fc:ccba:b0d9/64", "IPv6 link-local"},
|
||||
} {
|
||||
if hasAddr(got, bad.iface, bad.cidr) {
|
||||
t.Errorf("%s %s must be filtered (%s); got: %s", bad.iface, bad.cidr, bad.why, flatten(got))
|
||||
}
|
||||
}
|
||||
|
||||
// The no-denylist claim: not one veth/physical interface contributed a row.
|
||||
for _, a := range got {
|
||||
if strings.HasPrefix(a.Iface, "veth") || a.Iface == "enp1s0" || a.Iface == "wlp2s0" {
|
||||
t.Errorf("%s produced a row — the fixture says it has no address, so the filter invented one", a.Iface)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// demo-hp is different hardware (4 unused NICs, different ordering) and must filter identically —
|
||||
// the rule is about address CLASS, not about one box's interface names.
|
||||
func TestFilterHostAddresses_SecondHostFiltersIdentically(t *testing.T) {
|
||||
got := filterHostAddresses([]ifaceAddrs{
|
||||
{Name: "lo", Up: true, Loopback: true, CIDRs: []string{"127.0.0.1/8", "::1/128"}},
|
||||
{Name: "enp2s0f0", Up: false}, {Name: "enp1s0f0", Up: false},
|
||||
{Name: "enp1s0f1", Up: false}, {Name: "enp1s0f2", Up: false},
|
||||
{Name: "enp1s0f3", Up: false}, {Name: "wlo1", Up: false},
|
||||
{Name: "tailscale0", Up: true, CIDRs: []string{
|
||||
"100.76.96.79/32", "fd7a:115c:a1e0::ce36:6051/128", "fe80::e06a:ce64:80e1:7821/64"}},
|
||||
{Name: "vmbr0", Up: true, CIDRs: []string{"192.168.0.87/24", "fe80::7ed3:aff:fe77:d976/64"}},
|
||||
{Name: "wg-felhom", Up: true, CIDRs: []string{"10.77.0.3/32"}},
|
||||
{Name: "vmbr9", Up: true, CIDRs: []string{"169.254.253.1/30", "fe80::2484:92ff:fe7d:52a5/64"}},
|
||||
{Name: "veth9201i0", Up: true}, {Name: "veth9201i1", Up: true},
|
||||
})
|
||||
if !hasAddr(got, "vmbr0", "192.168.0.87/24") {
|
||||
t.Fatalf("demo-hp's LAN address was filtered away: %s", flatten(got))
|
||||
}
|
||||
if hasAddr(got, "vmbr9", "169.254.253.1/30") {
|
||||
t.Errorf("demo-hp's island address leaked through: %s", flatten(got))
|
||||
}
|
||||
// The island address is byte-identical on both boxes — the strongest argument for excluding it.
|
||||
if strings.Contains(flatten(got), "169.254.") {
|
||||
t.Errorf("a link-local IPv4 survived: %s", flatten(got))
|
||||
}
|
||||
}
|
||||
|
||||
// A DOWN interface holding a stale address must not be reported as if the box were reachable there.
|
||||
// RED-PROOF 2: drop `|| !i.Up` → this goes red.
|
||||
func TestFilterHostAddresses_DownInterfaceExcluded(t *testing.T) {
|
||||
got := filterHostAddresses([]ifaceAddrs{
|
||||
{Name: "vmbr0", Up: true, CIDRs: []string{"192.168.0.162/24"}},
|
||||
{Name: "vmbr1", Up: false, CIDRs: []string{"10.9.9.9/24"}},
|
||||
})
|
||||
if hasAddr(got, "vmbr1", "10.9.9.9/24") {
|
||||
t.Errorf("a DOWN interface's address was reported: %s", flatten(got))
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Errorf("want exactly the one up interface, got: %s", flatten(got))
|
||||
}
|
||||
}
|
||||
|
||||
// Order must be deterministic, or every report diff shows phantom churn.
|
||||
func TestFilterHostAddresses_DeterministicOrder(t *testing.T) {
|
||||
a := filterHostAddresses(demoFelhomIfaces())
|
||||
// Same facts, opposite enumeration order.
|
||||
rev := demoFelhomIfaces()
|
||||
for i, j := 0, len(rev)-1; i < j; i, j = i+1, j-1 {
|
||||
rev[i], rev[j] = rev[j], rev[i]
|
||||
}
|
||||
b := filterHostAddresses(rev)
|
||||
if flatten(a) != flatten(b) {
|
||||
t.Errorf("interface-table order changed the report:\n a=%s\n b=%s", flatten(a), flatten(b))
|
||||
}
|
||||
}
|
||||
|
||||
// A host with nothing routable yields [] and never nil — an absent key and "no addresses" must not
|
||||
// look alike on the wire.
|
||||
func TestFilterHostAddresses_EmptyIsNonNil(t *testing.T) {
|
||||
got := filterHostAddresses([]ifaceAddrs{{Name: "lo", Up: true, Loopback: true, CIDRs: []string{"127.0.0.1/8"}}})
|
||||
if got == nil {
|
||||
t.Fatal("filter returned nil — it would marshal as null, not []")
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("want no addresses, got %s", flatten(got))
|
||||
}
|
||||
}
|
||||
|
||||
// A malformed entry is skipped, never fatal — one bad address must not cost the report the others.
|
||||
func TestFilterHostAddresses_MalformedSkipped(t *testing.T) {
|
||||
got := filterHostAddresses([]ifaceAddrs{
|
||||
{Name: "vmbr0", Up: true, CIDRs: []string{"not-an-address", "192.168.0.162/24"}},
|
||||
})
|
||||
if len(got) != 1 || !hasAddr(got, "vmbr0", "192.168.0.162/24") {
|
||||
t.Errorf("a malformed sibling address broke the good one: %s", flatten(got))
|
||||
}
|
||||
}
|
||||
|
||||
// --- the WIRING half: the collector must actually call the filter ---
|
||||
|
||||
// The seam defaults to the REAL enumerator, so a forgotten wiring call cannot make this inert.
|
||||
// RED-PROOF 3: replace the collectAddresses body with `return []HostAddress{}` → red.
|
||||
func TestCollectAddresses_UsesTheInjectedEnumerator(t *testing.T) {
|
||||
c := &Collector{logger: slog.Default()}
|
||||
c.addrEnum = func() ([]ifaceAddrs, error) { return demoFelhomIfaces(), nil }
|
||||
got := c.collectAddresses()
|
||||
if !hasAddr(got, "vmbr0", "192.168.0.162/24") {
|
||||
t.Fatalf("the collector did not run the filter over the enumerator's output: %s", flatten(got))
|
||||
}
|
||||
}
|
||||
|
||||
// An enumeration failure degrades to [] and a WARN — never a failed report.
|
||||
func TestCollectAddresses_EnumerationErrorDegrades(t *testing.T) {
|
||||
c := &Collector{logger: slog.Default()}
|
||||
c.addrEnum = func() ([]ifaceAddrs, error) { return nil, errors.New("netlink is unhappy") }
|
||||
got := c.collectAddresses()
|
||||
if got == nil {
|
||||
t.Fatal("an enumeration error produced nil, which marshals as null")
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("want [] on error, got %s", flatten(got))
|
||||
}
|
||||
}
|
||||
|
||||
// The production enumerator must return SOMETHING on the machine running the tests, and must not
|
||||
// panic. This is the only test that touches the real network stack; it asserts the contract
|
||||
// (non-nil, no error, loopback correctly flagged) rather than any specific address, because the
|
||||
// test host's addresses are not ours to predict.
|
||||
func TestSystemInterfaces_ProductionEnumeratorWorks(t *testing.T) {
|
||||
ifaces, err := systemInterfaces()
|
||||
if err != nil {
|
||||
t.Fatalf("systemInterfaces: %v", err)
|
||||
}
|
||||
if len(ifaces) == 0 {
|
||||
t.Fatal("no interfaces at all — even a container has lo")
|
||||
}
|
||||
var sawLoopback bool
|
||||
for _, i := range ifaces {
|
||||
if i.Loopback {
|
||||
sawLoopback = true
|
||||
}
|
||||
}
|
||||
if !sawLoopback {
|
||||
t.Error("no interface reported the loopback flag — the flag mapping is wrong")
|
||||
}
|
||||
// And the filter must survive real input without panicking.
|
||||
_ = filterHostAddresses(ifaces)
|
||||
}
|
||||
+45
-7
@@ -57,8 +57,20 @@ type Loop struct {
|
||||
logger *slog.Logger
|
||||
trigger <-chan struct{} // optional: an out-of-band report request (storage watchdog)
|
||||
observer EnvelopeObserver // optional: the slice-10A desired-state sync hook
|
||||
|
||||
// Heartbeat log-pull (v0.83.0): logTailSource yields the debug ring's formatted
|
||||
// lines newest-kept within a byte budget (applog.Ring.Lines). logTailPending is
|
||||
// armed by an envelope's log_tail_requested and drained onto the NEXT report —
|
||||
// the report-channel logtail.go consume-once shape: a failed push leaves the
|
||||
// hub's request pending, so the next successful envelope re-arms it (fail-safe
|
||||
// retry, no duplicate shipping). Loop state is single-goroutine (cycle only).
|
||||
logTailSource func(maxBytes int) []string
|
||||
logTailPending bool
|
||||
}
|
||||
|
||||
// logTailMaxBytes caps the heartbeat log tail (newest lines kept).
|
||||
const logTailMaxBytes = 128 * 1024
|
||||
|
||||
// NewLoop builds the loop. interval is the starting cadence (the hub may override it
|
||||
// per-cycle via the control envelope).
|
||||
func NewLoop(collector collectorIface, client reporter, interval time.Duration, logger *slog.Logger) *Loop {
|
||||
@@ -69,9 +81,10 @@ func NewLoop(collector collectorIface, client reporter, interval time.Duration,
|
||||
}
|
||||
|
||||
// SetTrigger wires an out-of-band report channel. A receive on it runs one extra
|
||||
// collect→report cycle immediately WITHOUT disturbing the regular ticker cadence — used by
|
||||
// the storage watchdog to push a disconnect to the hub in seconds. The watchdog debounces,
|
||||
// so this fires at most once per debounce window.
|
||||
// collect→report cycle immediately WITHOUT disturbing the regular ticker cadence. Two producers
|
||||
// fan into this one channel: the storage watchdog (push a disconnect in seconds) and the
|
||||
// agent-plane poke listener (v0.89.0 immediate-sync). Both debounce, and the channel is cap-1
|
||||
// non-blocking, so a burst coalesces to at most one pending extra cycle.
|
||||
func (l *Loop) SetTrigger(ch <-chan struct{}) { l.trigger = ch }
|
||||
|
||||
// SetEnvelopeObserver wires the slice-10A desired-state sync hook. It is called once per cycle
|
||||
@@ -79,6 +92,10 @@ func (l *Loop) SetTrigger(ch <-chan struct{}) { l.trigger = ch }
|
||||
// desired-state when the generation advances. Optional — unset is a clean no-op.
|
||||
func (l *Loop) SetEnvelopeObserver(o EnvelopeObserver) { l.observer = o }
|
||||
|
||||
// SetLogTailSource wires the debug ring for the heartbeat log-pull (v0.83.0).
|
||||
// Optional — unset means an envelope's log_tail_requested is ignored.
|
||||
func (l *Loop) SetLogTailSource(src func(maxBytes int) []string) { l.logTailSource = src }
|
||||
|
||||
// Run reports immediately, then on each tick, until ctx is cancelled (then nil).
|
||||
func (l *Loop) Run(ctx context.Context) error {
|
||||
interval := l.interval
|
||||
@@ -99,10 +116,10 @@ func (l *Loop) Run(ctx context.Context) error {
|
||||
ticker.Reset(interval)
|
||||
}
|
||||
case <-l.trigger:
|
||||
// Out-of-band report (storage watchdog). Run a cycle now; keep the regular
|
||||
// cadence (do not reset the ticker). The envelope's interval is still adopted
|
||||
// Out-of-band report (storage watchdog OR agent-plane poke). Run a cycle now; keep the
|
||||
// regular cadence (do not reset the ticker). The envelope's interval is still adopted
|
||||
// if it changed, mirroring the normal path.
|
||||
l.logger.Info("hub: out-of-band report triggered (storage watchdog)")
|
||||
l.logger.Info("hub: out-of-band report triggered (watchdog/poke)")
|
||||
next := l.cycle(ctx, interval)
|
||||
if next != interval {
|
||||
l.logger.Info("hub: poll interval changed", "from", interval, "to", next)
|
||||
@@ -116,19 +133,40 @@ func (l *Loop) Run(ctx context.Context) error {
|
||||
// cycle runs one collect→report→adopt. It never returns an error: failures are
|
||||
// logged and the current interval is kept, so the loop keeps running.
|
||||
func (l *Loop) cycle(ctx context.Context, current time.Duration) time.Duration {
|
||||
start := time.Now()
|
||||
report, err := l.collector.Collect(ctx)
|
||||
if err != nil {
|
||||
l.logger.Warn("hub: collect failed; skipping this cycle's report", "err", err)
|
||||
return current
|
||||
}
|
||||
// Fulfill a pending log-pull: attach the ring tail to THIS report and clear the
|
||||
// local pending flag (consume-once). On a failed push the hub's request is still
|
||||
// pending and the next envelope re-arms it — logtail.go's fail-safe retry shape.
|
||||
// The explicit nil first makes this robust to a collector reusing its report struct.
|
||||
report.LogTail = nil
|
||||
if l.logTailPending && l.logTailSource != nil {
|
||||
report.LogTail = &LogTail{
|
||||
CollectedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
Lines: l.logTailSource(logTailMaxBytes),
|
||||
}
|
||||
}
|
||||
l.logTailPending = false
|
||||
env, err := l.client.Report(ctx, report)
|
||||
if err != nil {
|
||||
l.logger.Warn("hub: report failed; keeping current interval", "err", err)
|
||||
return current
|
||||
}
|
||||
if report.LogTail != nil {
|
||||
// Transparency: the pull is visible in the box's own log (and thus in the ring).
|
||||
l.logger.Info("operator log pull served", "component", "agent", "lines", len(report.LogTail.Lines))
|
||||
}
|
||||
l.logger.Debug("hub: report sent",
|
||||
"guests", len(report.Guests),
|
||||
"guests", len(report.Guests), "duration_ms", time.Since(start).Milliseconds(),
|
||||
"blocked", env.Blocked, "desired_generation", env.DesiredGeneration, "has_signed_ops", env.HasSignedOps)
|
||||
if env.LogTailRequested {
|
||||
l.logger.Debug("hub: log tail requested — shipping on the next heartbeat")
|
||||
l.logTailPending = true
|
||||
}
|
||||
|
||||
// Slice 10A: hand the envelope to the desired-state sync hook (fetch desired-state on a
|
||||
// generation advance). Done off the report's critical path semantics — a sync/fetch failure
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// recordingReporter records each pushed report and serves a scripted per-call
|
||||
// (envelope, error) sequence — the S2 heartbeat log-pull harness.
|
||||
type recordingReporter struct {
|
||||
reports []*HostReport
|
||||
script []struct {
|
||||
env *ControlEnvelope
|
||||
err error
|
||||
}
|
||||
}
|
||||
|
||||
func (r *recordingReporter) Report(_ context.Context, rep *HostReport) (*ControlEnvelope, error) {
|
||||
// Copy the LogTail pointer state at push time (the loop reuses collector reports).
|
||||
cp := *rep
|
||||
r.reports = append(r.reports, &cp)
|
||||
i := len(r.reports) - 1
|
||||
if i < len(r.script) {
|
||||
return r.script[i].env, r.script[i].err
|
||||
}
|
||||
return &ControlEnvelope{}, nil
|
||||
}
|
||||
|
||||
func tailLoop(rep *recordingReporter) *Loop {
|
||||
var cn int32
|
||||
l := NewLoop(&fakeCollector{report: &HostReport{}, n: &cn}, rep, time.Hour, quietLogger())
|
||||
l.SetLogTailSource(func(maxBytes int) []string { return []string{"line-a", "line-b"} })
|
||||
return l
|
||||
}
|
||||
|
||||
// S2 (agent half): an envelope's log_tail_requested arms the pull; the NEXT report
|
||||
// carries log_tail; the one after (request cleared hub-side) carries nothing —
|
||||
// consume-once. Companion red-proof: drop the `l.logTailPending = false` drain →
|
||||
// report 3 also carries a tail → the last assertion fails.
|
||||
func TestLoop_LogTailRequestedShipsOnNextReportOnce(t *testing.T) {
|
||||
rep := &recordingReporter{script: []struct {
|
||||
env *ControlEnvelope
|
||||
err error
|
||||
}{
|
||||
{env: &ControlEnvelope{LogTailRequested: true}},
|
||||
{env: &ControlEnvelope{}}, // the tail arrived — hub cleared the request
|
||||
{env: &ControlEnvelope{}},
|
||||
}}
|
||||
l := tailLoop(rep)
|
||||
ctx := context.Background()
|
||||
l.cycle(ctx, time.Hour)
|
||||
l.cycle(ctx, time.Hour)
|
||||
l.cycle(ctx, time.Hour)
|
||||
|
||||
if len(rep.reports) != 3 {
|
||||
t.Fatalf("reports = %d, want 3", len(rep.reports))
|
||||
}
|
||||
if rep.reports[0].LogTail != nil {
|
||||
t.Errorf("report 1 must not carry a tail (the request only arrived in its envelope)")
|
||||
}
|
||||
got := rep.reports[1].LogTail
|
||||
if got == nil || len(got.Lines) != 2 || got.Lines[0] != "line-a" || got.CollectedAt == "" {
|
||||
t.Fatalf("report 2 log_tail = %+v, want the 2 ring lines + collected_at", got)
|
||||
}
|
||||
if rep.reports[2].LogTail != nil {
|
||||
t.Errorf("report 3 carries a tail again — consume-once broken: %+v", rep.reports[2].LogTail)
|
||||
}
|
||||
}
|
||||
|
||||
// S2 companion (fail-safe retry): the push CARRYING the tail fails → the local pending
|
||||
// is spent, but the hub's request is still pending, so the next envelope re-arms it and
|
||||
// the following report fulfills. Asserts the retry ships the tail exactly once more.
|
||||
func TestLoop_FailedTailPushIsReArmedByNextEnvelope(t *testing.T) {
|
||||
rep := &recordingReporter{script: []struct {
|
||||
env *ControlEnvelope
|
||||
err error
|
||||
}{
|
||||
{env: &ControlEnvelope{LogTailRequested: true}}, // arm
|
||||
{err: errors.New("hub 5xx")}, // the carrying push FAILS
|
||||
{env: &ControlEnvelope{LogTailRequested: true}}, // hub still pending → re-arm
|
||||
{env: &ControlEnvelope{}}, // fulfilled
|
||||
}}
|
||||
l := tailLoop(rep)
|
||||
ctx := context.Background()
|
||||
for i := 0; i < 4; i++ {
|
||||
l.cycle(ctx, time.Hour)
|
||||
}
|
||||
if len(rep.reports) != 4 {
|
||||
t.Fatalf("reports = %d, want 4", len(rep.reports))
|
||||
}
|
||||
if rep.reports[1].LogTail == nil {
|
||||
t.Errorf("report 2 (the failed push) should have carried the tail")
|
||||
}
|
||||
if rep.reports[2].LogTail != nil {
|
||||
t.Errorf("report 3 must not carry a tail (pending was spent; envelope re-arms only after it)")
|
||||
}
|
||||
if rep.reports[3].LogTail == nil {
|
||||
t.Errorf("report 4 must fulfill the re-armed request — retry lost")
|
||||
}
|
||||
}
|
||||
|
||||
// No source wired → the request is ignored (clean no-op, no panic).
|
||||
func TestLoop_LogTailRequestIgnoredWithoutSource(t *testing.T) {
|
||||
rep := &recordingReporter{script: []struct {
|
||||
env *ControlEnvelope
|
||||
err error
|
||||
}{
|
||||
{env: &ControlEnvelope{LogTailRequested: true}},
|
||||
{env: &ControlEnvelope{}},
|
||||
}}
|
||||
var cn int32
|
||||
l := NewLoop(&fakeCollector{report: &HostReport{}, n: &cn}, rep, time.Hour, quietLogger())
|
||||
ctx := context.Background()
|
||||
l.cycle(ctx, time.Hour)
|
||||
l.cycle(ctx, time.Hour)
|
||||
if rep.reports[1].LogTail != nil {
|
||||
t.Errorf("tail shipped with no source wired: %+v", rep.reports[1].LogTail)
|
||||
}
|
||||
}
|
||||
+229
-2
@@ -43,6 +43,12 @@ type HostReport struct {
|
||||
// alert. Not a secret (the fp is public; the token is never reported).
|
||||
LeafFingerprint string `json:"leaf_fingerprint"`
|
||||
|
||||
// Addresses are the host's routable addresses, one entry per (interface, address) — the LAN
|
||||
// bridge, the WG tunnel, a tailnet. Added v0.119.0 because the hub could not show a managed
|
||||
// box's IP anywhere: nothing in this report carried one. Non-nil so it marshals as [];
|
||||
// see hostaddr.go for why it is iface+cidr rather than a single lan_ip.
|
||||
Addresses []HostAddress `json:"addresses"`
|
||||
|
||||
// DR recipe — the agent (storage/guest/PBS) half of the secret-free reconstruction recipe
|
||||
// (SPIKE-dr-recipe-2026-06-16). Derived from the facts above; carries ONLY identifiers/intents/
|
||||
// sizes/coordinates, never a secret. The hub assembles it with the controller's app half.
|
||||
@@ -62,10 +68,134 @@ type HostReport struct {
|
||||
// report is stored opaquely hub-side, so these additive fields need no hub-schema change.
|
||||
// Both are `omitempty` (the Wireguard precedent): in the steady state (no update in flight)
|
||||
// they are absent — which keeps the cross-repo host-report golden contract byte-stable without
|
||||
// a hub change. They appear only while an update is pending. The hub reads an absent field as
|
||||
// pending=false, the correct default.
|
||||
// a hub change. They appear only while an update is pending.
|
||||
//
|
||||
// ⚠ CORRECTED 2026-08-08 (R-260). This comment used to end "The hub reads an absent field as
|
||||
// pending=false, the correct default." THE HUB HAS NO FIELD FOR EITHER OF THESE, so it reads
|
||||
// nothing — present or absent — and encoding/json discards them on arrival. The sentence
|
||||
// described an intent, not the code, and it read as settled for long enough that a sweep had to
|
||||
// find it. The emission is correct and stays; the missing consumer is tracked as R-264, and
|
||||
// `felhom.eu/scripts/wire_contract_gate.py` now refuses any NEW field of this shape.
|
||||
SelfUpdatePending bool `json:"selfupdate_pending,omitempty"`
|
||||
SelfUpdatePendingVersion string `json:"selfupdate_pending_version,omitempty"`
|
||||
|
||||
// MgmtPlane is the management-plane health stanza (TASK G1). It reports whether OpenSSH's SHARED
|
||||
// privilege-separation dir (/run/sshd) is present and whether the stock sshd listener answers, plus
|
||||
// whether the agent-independent watchdog auto-healed a missing dir since boot (and when). This is
|
||||
// the visibility half of the break-glass system: the dumb watchdog fixes /run/sshd with no login,
|
||||
// and this stanza surfaces a RECURRING clobber to the operator BEFORE it becomes a lockout —
|
||||
// complementing host_staleness (which only catches a box gone silent). `omitempty` (the
|
||||
// SelfUpdatePending precedent): stored opaquely hub-side, so these additive fields need no
|
||||
// hub-schema change and are absent when the reporter is not wired.
|
||||
MgmtPlane *MgmtPlaneStatus `json:"mgmt_plane,omitempty"`
|
||||
|
||||
// PBSDR is the PBS-DR-tier bridge status stanza (slice 2). Present only when the pbsdr
|
||||
// consumer is wired. `consumed_failed` is the LOUD persistent state: the one-time token
|
||||
// secret was consumed but the apply failed afterwards — the secret is burned, the bridge
|
||||
// will NOT silently retry, the operator must Re-issue on the hub. Stored opaquely hub-side
|
||||
// (the Wireguard precedent) — additive, no hub-schema change; hub rendering joins in slice 3.
|
||||
// Carries NO secret.
|
||||
PBSDR *PBSDRStatus `json:"pbs_dr,omitempty"`
|
||||
|
||||
// GuestNet is the per-guest network-watchdog stanza (R-54). Present only when the guestnet
|
||||
// watchdog is wired. Same additive/opaque contract as PBSDR and Wireguard above — no
|
||||
// hub-schema change, absent when the reporter is not wired. Carries NO secret: addresses,
|
||||
// route/liveness booleans, heal timestamps and counters only.
|
||||
//
|
||||
// NAMING NOTE (deliberate deviation from TASK-D, which called this block `WireGuestNet`):
|
||||
// in this repo `Wire*` types are the DOWN direction (WireDesiredState/WirePBSDR — what the
|
||||
// hub sends the agent), while UP-direction report stanzas are `*Status`. A `WireGuestNet`
|
||||
// on HostReport would have been the only report block named against that convention.
|
||||
GuestNet *GuestNetStatus `json:"guest_net,omitempty"`
|
||||
|
||||
// LogTail is the agent's on-demand debug-ring tail (v0.83.0 observability) — the agent
|
||||
// mirror of the controller's report log_tails channel. Present ONLY on the heartbeat
|
||||
// right after the control envelope requested it (log_tail_requested); consume-once on
|
||||
// both ends (the hub clears its pending request on arrival). Newest lines kept, byte-
|
||||
// capped loop-side. Carries log lines only — the logging conventions forbid secrets in
|
||||
// any log line, and the hub's bundle gate re-checks before storing. `omitempty`: absent
|
||||
// in the steady state, so the cross-repo host-report golden stays byte-stable.
|
||||
LogTail *LogTail `json:"log_tail,omitempty"`
|
||||
|
||||
// OOB is the operator-access health stanza (TASK H1). It answers the operator's question — "can I
|
||||
// get into this box right now, and if not, why" — from the hub: felhom-sshd up + on which port,
|
||||
// locally reachable, the tunnel handshake age (the OOB path rides wg-felhom), whether the operator
|
||||
// peer/key is configured, and whether the config is invalid. The heartbeat reaches the hub over
|
||||
// HTTPS even when felhom-sshd or the tunnel is DOWN (channel independence). `omitempty`: absent
|
||||
// when the feature is not wired (pre-H1) — additive, no hub-schema change.
|
||||
OOB *OOBStatus `json:"oob,omitempty"`
|
||||
}
|
||||
|
||||
// PBSDRStatus is the per-heartbeat PBS-DR-tier bridge state (slice 2). States:
|
||||
// "adopted" (existing entry verified + reconciled, no consume), "applied" (fresh entry created,
|
||||
// K born), "waiting_secret" (verified but no unconsumed secret staged — retrying),
|
||||
// "verify_failed" (fingerprint/reachability pre-consume check failing — retrying, NOTHING
|
||||
// consumed), "consumed_failed" (LOUD: secret burned, apply failed, no auto-retry — operator
|
||||
// re-issue required), "disabled" (descriptor enabled:false). Carries no secret.
|
||||
// GuestNetStatus is the R-54 guest-network watchdog stanza. `guests` carries one entry per owned
|
||||
// RUNNING guest that has been probed at least once; an empty list with a fresh `checked_at` means
|
||||
// the watchdog ran and found nothing to report, which is deliberately distinguishable from the
|
||||
// stanza being absent (= the watchdog is not wired at all).
|
||||
type GuestNetStatus struct {
|
||||
CheckedAt string `json:"checked_at"` // RFC3339, the sweep this snapshot came from
|
||||
Guests []GuestNetGuest `json:"guests,omitempty"`
|
||||
}
|
||||
|
||||
// GuestNetGuest mirrors guestnet.GuestReport on the wire. The two structs are deliberately separate:
|
||||
// internal/hub owns the wire contract and imports no feature package (the consumer-side seam rule).
|
||||
type GuestNetGuest struct {
|
||||
VMID int `json:"vmid"`
|
||||
State string `json:"state"` // healthy | unhealthy | static_fault | unknown
|
||||
Mode string `json:"mode"` // dhcp | static | unknown
|
||||
IP string `json:"ip,omitempty"`
|
||||
HasRoute bool `json:"has_route"`
|
||||
DHClientAlive bool `json:"dhclient_alive"`
|
||||
CheckedAt string `json:"checked_at,omitempty"`
|
||||
Healed bool `json:"healed,omitempty"`
|
||||
HealSucceeded bool `json:"heal_succeeded,omitempty"`
|
||||
LastHealAt string `json:"last_heal_at,omitempty"`
|
||||
HealsLastHour int `json:"heals_last_hour,omitempty"`
|
||||
Damped bool `json:"damped,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type PBSDRStatus struct {
|
||||
State string `json:"state"`
|
||||
StorageID string `json:"storage_id,omitempty"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
ConsumedFailed bool `json:"consumed_failed,omitempty"`
|
||||
AppliedAt string `json:"applied_at,omitempty"` // RFC3339; set on adopted/applied
|
||||
// AuthFailed (R-39, v0.91.0) — the credential this box holds is REJECTED by PBS (401). Set by the
|
||||
// verify loop's ProbeAuth, which before v0.91.0 could not run at all: the loop read the secret
|
||||
// file directly as non-root and always failed with "permission denied", so an applied-and-dead
|
||||
// tier was invisible to both tiers. The hub's pbsdrheal escalates state="auth_failed" to a fresh
|
||||
// mint.
|
||||
AuthFailed bool `json:"auth_failed,omitempty"`
|
||||
}
|
||||
|
||||
// OOBStatus is the per-heartbeat operator-access health (TASK H1). Carries no secret.
|
||||
type OOBStatus struct {
|
||||
FelhomSshdActive bool `json:"felhom_sshd_active"` // the felhom-sshd unit is active
|
||||
FelhomSshdPort int `json:"felhom_sshd_port"` // the claimed OOB port (0 = unclaimed)
|
||||
Reachable bool `json:"reachable"` // a local TCP dial to the OOB port succeeds
|
||||
ConfigInvalid bool `json:"config_invalid"` // `sshd -t` on the OOB config fails
|
||||
OperatorPeerConfigured bool `json:"operator_peer_configured"` // oob_peer_ip present in desired-state
|
||||
OperatorKeyConfigured bool `json:"operator_key_configured"` // operator authorized_key installed
|
||||
WGHandshakeAgeS *int64 `json:"wg_handshake_age_s,omitempty"` // wg-felhom last handshake age (nil = unknown)
|
||||
HealedAt string `json:"healed_at,omitempty"` // last felhom-sshd auto-heal (RFC3339)
|
||||
}
|
||||
|
||||
// MgmtPlaneStatus is the per-heartbeat management-plane health (TASK G1). Carries no secret.
|
||||
// HealedRecently is true while the watchdog's heal-marker exists (a privsep-dir heal happened this
|
||||
// boot); PrivsepHealedAt is that marker's RFC3339 timestamp (absent when no heal has occurred). The
|
||||
// hub raises a warning event on a PrivsepHealedAt it has not alerted on — a recurring auto-heal means
|
||||
// a persistent clobber cause worth investigating before the box locks out.
|
||||
type MgmtPlaneStatus struct {
|
||||
PrivsepDirOK bool `json:"privsep_dir_ok"` // /run/sshd exists (the KEXINIT-reset detector)
|
||||
SshdReachable bool `json:"sshd_reachable"` // the stock sshd listener accepts TCP
|
||||
HealedRecently bool `json:"healed_recently"` // the watchdog heal-marker is present (this boot)
|
||||
PrivsepHealedAt string `json:"privsep_healed_at,omitempty"` // marker timestamp; hub warns on a NEW value
|
||||
}
|
||||
|
||||
// WireguardStatus is the per-heartbeat offsite-tunnel status (S3). LastHandshakeAgeS is nil when
|
||||
@@ -96,6 +226,17 @@ type HostMetrics struct {
|
||||
// the per-disk SmartSummary.TemperatureC. Sourced from sysfs (hwmon / thermal zones).
|
||||
// Cross-repo wire field (slice 9) — the hub's HostMetrics copy + golden carry it too.
|
||||
CPUTempC *int `json:"cpu_temp_c"`
|
||||
// WrapperSHA256 is the sha256 of the installed PBS-DR apply wrapper
|
||||
// (/usr/local/sbin/felhom-pbs-apply), R-50b(a), v0.91.0.
|
||||
//
|
||||
// That wrapper is root-owned 0755 and the pinned sudoers vector for the PBS storage verbs, yet it
|
||||
// is installed from `raw/branch/main` — unversioned, unpinned and absent from the Day-0 artifact
|
||||
// manifest. So "which wrapper is on this host?" had no answer: two hosts installed a week apart
|
||||
// could carry different privileged code while reporting the same agent version. Reporting the hash
|
||||
// does not fix the delivery channel (R-50b(b)/(c)); it makes drift VISIBLE.
|
||||
//
|
||||
// Empty = unreadable/absent, which the hub treats as UNKNOWN, never as drift.
|
||||
WrapperSHA256 string `json:"wrapper_sha256,omitempty"`
|
||||
}
|
||||
|
||||
// Guest is one LXC. The agent reports vmid; the hub derives the guest PK
|
||||
@@ -151,6 +292,30 @@ type StorageTarget struct {
|
||||
|
||||
MountPath string `json:"mount_path"` // host mountpoint (dir/usb); "" for network/lvm
|
||||
BackingDevice string `json:"backing_device"` // resolved block device (e.g. /dev/sdb1); "" for network
|
||||
// ConfigPath is the storage's CONFIGURED path from storage.cfg (proxmox.Storage.Path) — not a
|
||||
// resolved mount. It is the only identity a dir storage keeps when its device is gone: MountPath
|
||||
// and BackingDevice both empty out (observe.go's exactMount block) and DurableID degrades off the
|
||||
// fs-UUID, so the configured path is what still says WHICH drive this row is about (R-116).
|
||||
//
|
||||
// `json:"-"` DELIBERATELY. This struct is a cross-repo contract duplicated in felhom.eu/hub and
|
||||
// pinned by testdata/host-report.golden.json + contract_test.go's key-set comparison; a wire-visible
|
||||
// field here would need a matching change in the other repo to stay non-drifting. Nothing off-box
|
||||
// needs this value — its only consumer is the agent's own /disks construction, in-process.
|
||||
ConfigPath string `json:"-"`
|
||||
// PBSNamespace is the storage's CONFIGURED PBS namespace from storage.cfg (proxmox.Storage.Namespace)
|
||||
// — "" for the root namespace, set for S4 per-customer tenancy. Present ONLY on pbs targets.
|
||||
//
|
||||
// It exists because storage.cfg is the ONE authority on which namespace this box's backups use:
|
||||
// `vzdump --storage <pbs>` makes PVE read this exact field, and the agent's own verify client is
|
||||
// built from it (`cmd/felhom-agent/main.go` → `pbs.Config{Namespace: s.Namespace}`). The DR recipe
|
||||
// therefore resolves the namespace from HERE and not from a listed snapshot — a namespace-scoped
|
||||
// PBS list does not echo `ns` per item, so the snapshot's own field is empty and normalising that
|
||||
// empty to "root" is what made the recipe claim "root" on every per-customer box (R-106).
|
||||
//
|
||||
// `json:"-"` for the SAME reason as ConfigPath above: this struct is a cross-repo contract pinned by
|
||||
// testdata/host-report.golden.json, and nothing off-box reads this value — its only consumer is the
|
||||
// agent's own dr_recipe construction, in-process.
|
||||
PBSNamespace string `json:"-"`
|
||||
// ClassHint is a fast|slow HINT derived from the backing disk's rotational flag — a
|
||||
// hint only; the authoritative class is hub-owned (locked decision). "" when not
|
||||
// derivable (network targets have no local rotational flag).
|
||||
@@ -187,6 +352,11 @@ type ThinPoolFill struct {
|
||||
type SmartSummary struct {
|
||||
Health string `json:"health"`
|
||||
|
||||
// ModelName is smartctl's own device model (v0.95.0), captured from the JSON already parsed, so
|
||||
// the UI can show a human label ("TOSHIBA MQ04ABF100") instead of a raw UUID. omitempty +
|
||||
// pointer: absent on an old agent or a device that reports no model.
|
||||
ModelName *string `json:"model_name,omitempty"`
|
||||
|
||||
TemperatureC *int `json:"temperature_c"`
|
||||
PowerOnHours *int `json:"power_on_hours"`
|
||||
|
||||
@@ -262,6 +432,11 @@ type RestoreTest struct {
|
||||
// (⇒ false) when absent — and false is the SAFE default: the hub then treats it as an
|
||||
// unrecognized warning (louder), so a missing flag can only over-notice, never hide.
|
||||
WarningsRecognized bool `json:"warnings_recognized,omitempty"`
|
||||
// MountParity ("ok"|"mismatch"; omitted on pre-v0.76.0 agents) + MountInventory (the verified
|
||||
// mpN set) carry the GL-5b full-fidelity proof: the restored scratch matched the ARCHIVE's
|
||||
// mount layout, not just booted. Additive — a hub that predates them ignores the unknown keys.
|
||||
MountParity string `json:"mount_parity,omitempty"`
|
||||
MountInventory []string `json:"mount_inventory,omitempty"`
|
||||
}
|
||||
|
||||
// PBSSnapshot is one PBS (offsite) snapshot's inventory + integrity state (doc 03 §8, slice
|
||||
@@ -298,6 +473,16 @@ type ControlEnvelope struct {
|
||||
Blocked bool `json:"blocked"` // reserved — ignored
|
||||
DesiredGeneration int64 `json:"desired_generation"` // slice 10A: the cached-vs-current change signal
|
||||
HasSignedOps bool `json:"has_signed_ops"` // slice 10A: signed-jobs queue non-empty (exec 10B)
|
||||
// LogTailRequested (v0.83.0) — the operator wants this agent's debug-ring tail; the
|
||||
// NEXT heartbeat carries it in log_tail (the report-channel log_tail_requests mirror).
|
||||
// Absent/false on an old hub → nothing happens.
|
||||
LogTailRequested bool `json:"log_tail_requested"`
|
||||
}
|
||||
|
||||
// LogTail is the heartbeat's on-demand agent log tail (see HostReport.LogTail).
|
||||
type LogTail struct {
|
||||
CollectedAt string `json:"collected_at"` // RFC3339
|
||||
Lines []string `json:"lines"`
|
||||
}
|
||||
|
||||
// DesiredStateResponse is GET /hosts/{host_id}/desired-state (slice 10A — the "Down" channel's
|
||||
@@ -325,6 +510,35 @@ type WireDesiredState struct {
|
||||
PBSNamespace string `json:"pbs_namespace,omitempty"`
|
||||
RestoreDirective *WireRestoreDirective `json:"restore_directive,omitempty"` // slice 10D (forward-compat)
|
||||
Wireguard *WireWireguard `json:"wireguard,omitempty"` // S3 (doc 06 §3.2; golden-pinned)
|
||||
PBSDR *WirePBSDR `json:"pbs_dr,omitempty"` // PBS DR tier (slice 2 consumer)
|
||||
}
|
||||
|
||||
// WirePBSDR is the hub's PBS-DR-tier descriptor (PBS DR slice 1, hub/internal/web/pbsdr.go
|
||||
// pbsDRDescriptor — field-exact, cross-repo). NON-SECRET by contract: the token secret NEVER
|
||||
// rides the desired-state; the agent fetches it consume-once via ConsumePBSToken. Absent/nil on
|
||||
// pre-v0.44.0 hubs → the pbsdr consumer no-ops (old-hub compat).
|
||||
type WirePBSDR struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
StorageID string `json:"storage_id,omitempty"`
|
||||
PBSTunnelIP string `json:"pbs_tunnel_ip,omitempty"`
|
||||
Datastore string `json:"datastore,omitempty"`
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
TokenID string `json:"token_id,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
// SecretGeneration (R-39, agent v0.91.0 / hub v0.68.0) is the hub's monotonic per-host counter,
|
||||
// advanced by every fresh secret MINT. It carries no secret material — only the fact that one
|
||||
// rotated.
|
||||
//
|
||||
// THIS FIELD IS THE RE-ARM SIGNAL, and it works only because descriptorHash marshals THIS STRUCT:
|
||||
// an ep0 re-issue re-keys the secret of an existing token, so token_id, fingerprint, datastore and
|
||||
// namespace all come back byte-identical. Without this field the descriptor never moves, the
|
||||
// converged agent short-circuits, the fresh secret is never consumed, and the box serves a revoked
|
||||
// credential while reporting `applied` (the N100, 2026-07-18).
|
||||
//
|
||||
// Corollary worth stating: an agent that does NOT carry this field drops the unknown JSON key and
|
||||
// keeps today's behaviour exactly — inert, not broken. That is why hub v0.68.0 is safe to deploy
|
||||
// ahead of the fleet, and why the re-arm guarantee needs agent >= 0.91.0.
|
||||
SecretGeneration int64 `json:"secret_generation,omitempty"`
|
||||
}
|
||||
|
||||
// WireWireguard is the hub-owned offsite-tunnel assignment (S3) — field-exact with the S2 golden
|
||||
@@ -335,6 +549,19 @@ type WireWireguard struct {
|
||||
Endpoint WireWireguardEndpoint `json:"endpoint"`
|
||||
Pubkey string `json:"pubkey"` // the box's registered pubkey
|
||||
AssignedIP string `json:"assigned_ip"` // e.g. "10.77.0.2/32"
|
||||
|
||||
// OOBPeerIP is the operator OOB peer's tunnel address (bare IPv4, e.g. "10.77.0.250", TASK H1).
|
||||
// When set, the agent appends it as a second /32 to the box's wg-felhom AllowedIPs so packets the
|
||||
// endpoint forwards FROM the operator are accepted — RENDERED into the conf (not a runtime `wg
|
||||
// set`) so it survives self-heal/restart/reboot ([OF-1]: a runtime widening was wiped by the
|
||||
// agent's own self-heal). `omitempty`: absent = OOB off, byte-identical conf for pre-H1 hosts.
|
||||
OOBPeerIP string `json:"oob_peer_ip,omitempty"`
|
||||
|
||||
// OOBOperatorSSHKey is the operator's SSH PUBLIC key (an authorized_keys line, TASK H1). The agent
|
||||
// writes it to felhom-sshd's dedicated AuthorizedKeysFile (/etc/felhom-sshd/authorized_keys/{root,
|
||||
// felhom-op}) — OUTSIDE ~/.ssh, so the customer's sshd never honours it [SF-3]. Hub-driven so it
|
||||
// rotates fleet-wide. `omitempty`: absent = no operator login installed. NOT a secret (public key).
|
||||
OOBOperatorSSHKey string `json:"oob_operator_ssh_key,omitempty"`
|
||||
}
|
||||
|
||||
// WireWireguardEndpoint is the endpoint half of the wireguard block.
|
||||
|
||||
@@ -32,10 +32,11 @@ func TestHostReport_FieldNamesAndEmptyCollections(t *testing.T) {
|
||||
Cloudflared: Cloudflared{Status: "active"},
|
||||
Capabilities: []capability.Status{},
|
||||
LeafFingerprint: "60b5974d586f5f3c8ec41eb998d0f07406178219c36bf6d3ff377570279d8245",
|
||||
Addresses: []HostAddress{},
|
||||
}
|
||||
// dr_recipe is always set on the real path (Collect); set it here too so the "no null" invariant
|
||||
// covers it (empty pbs is omitempty → omitted, never null).
|
||||
r.DRRecipe = BuildDRRecipeHostHalf(r.Guests, r.StorageTargets, r.PBSSnapshots)
|
||||
r.DRRecipe = BuildDRRecipeHostHalf(r.Guests, r.StorageTargets, r.PBSSnapshots, ConfiguredBackupTarget{})
|
||||
b, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -51,6 +52,7 @@ func TestHostReport_FieldNamesAndEmptyCollections(t *testing.T) {
|
||||
// empty collections must be [] not null
|
||||
`"storage_targets":[]`, `"backups":[]`, `"restore_tests":[]`, `"pbs_snapshots":[]`, `"audit_tail":[]`,
|
||||
`"capabilities":[]`,
|
||||
`"addresses":[]`,
|
||||
`"leaf_fingerprint":"60b5974d586f5f3c8ec41eb998d0f07406178219c36bf6d3ff377570279d8245"`,
|
||||
} {
|
||||
if !strings.Contains(got, field) {
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
package hub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// R-189 — a passing restore-test must survive an agent restart and reach the hub.
|
||||
//
|
||||
// THE OBSERVATION THIS EXISTS FOR (2026-08-03, demo-felhom): a real 14.5 GB offsite restore-test
|
||||
// PASSED at 15:25:14; the agent was restarted 2 m 43 s later for a deploy; the hub logged
|
||||
// `0 restore-tests` on the next two host-reports. The in-memory store's own comment said "lost on
|
||||
// restart; the cadence re-populates", which was true under a timer and stopped being true when R-86
|
||||
// made the agent refuse to re-test an archive it has already proven.
|
||||
//
|
||||
// Timestamps here carry JITTER (odd minutes and seconds, not round hours) — yesterday a test was
|
||||
// hollow because a perfectly regular series landed exactly on a threshold and passed under the
|
||||
// mutation it was meant to catch.
|
||||
|
||||
type fakeLatest struct{ tests []RestoreTest }
|
||||
|
||||
func (f *fakeLatest) RestoreTests(context.Context) []RestoreTest { return f.tests }
|
||||
|
||||
type fakeProven struct{ tests []RestoreTest }
|
||||
|
||||
func (f *fakeProven) ProvenRestoreTests(context.Context) []RestoreTest { return f.tests }
|
||||
|
||||
func rt(tier, archive string, pass bool, at time.Time) RestoreTest {
|
||||
return RestoreTest{
|
||||
SourceArchive: archive, SourceTier: tier, Pass: pass,
|
||||
Verified: "boot+running", TestedAt: at.UTC().Format(time.RFC3339),
|
||||
}
|
||||
}
|
||||
|
||||
// mergeCollector builds a Collector with only the two restore-test seams wired — the merge is what
|
||||
// is under test, not the rest of the collection.
|
||||
func mergeCollector(latest, proven []RestoreTest) *Collector {
|
||||
c := &Collector{}
|
||||
if latest != nil {
|
||||
c.restoreTests = &fakeLatest{tests: latest}
|
||||
}
|
||||
if proven != nil {
|
||||
c.provenTests = &fakeProven{tests: proven}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func findTier(got []RestoreTest, tier string) (RestoreTest, int) {
|
||||
var hit RestoreTest
|
||||
n := 0
|
||||
for _, e := range got {
|
||||
if e.SourceTier == tier {
|
||||
hit, n = e, n+1
|
||||
}
|
||||
}
|
||||
return hit, n
|
||||
}
|
||||
|
||||
// ── SCENARIO A — a proof survives a restart and reaches the hub ──────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-03): delete the `c.provenTests` merge from
|
||||
// collectRestoreTests (return the in-memory slice as it used to) →
|
||||
//
|
||||
// --- FAIL: TestMerge_ProofSurvivesARestart
|
||||
// restoretest_merge_test.go: after a restart the persisted proof must be reported; got 0 entr(ies)
|
||||
//
|
||||
// which is exactly the live observation: `0 restore-tests`. Restored.
|
||||
func TestMerge_ProofSurvivesARestart(t *testing.T) {
|
||||
provenAt := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC) // the real run's timestamp
|
||||
// After a restart the in-memory store is EMPTY — this is the whole point.
|
||||
c := mergeCollector([]RestoreTest{}, []RestoreTest{
|
||||
rt("pbs", "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z", true, provenAt),
|
||||
})
|
||||
|
||||
got := c.collectRestoreTests(context.Background())
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("after a restart the persisted proof must be reported; got %d entr(ies): %+v", len(got), got)
|
||||
}
|
||||
e := got[0]
|
||||
if e.SourceArchive != "felhom-pbs:backup/ct/9201/2026-07-28T04:49:43Z" {
|
||||
t.Fatalf("the entry must name the archive that was proven — the hub keys on it; got %q", e.SourceArchive)
|
||||
}
|
||||
if e.SourceTier != "pbs" || !e.Pass {
|
||||
t.Fatalf("the entry must be a PASS on the tier it was proven on; got tier=%q pass=%v", e.SourceTier, e.Pass)
|
||||
}
|
||||
if e.TestedAt != provenAt.Format(time.RFC3339) {
|
||||
t.Fatalf("the entry must carry the ORIGINAL test time, not now(); got %q", e.TestedAt)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO B — the report does not invent a pass ───────────────────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): make the state layer emit an entry for an unproven tier (drop the
|
||||
// `reportable()` filter in ProvenRestoreTests, so a legacy record with no archive is emitted) — the
|
||||
// equivalent at this layer is a proven-source that returns an entry for a tier nothing proved, which
|
||||
// this test injects directly and the assertion below rejects.
|
||||
func TestMerge_NeverInventsAPassForAnUnprovenTier(t *testing.T) {
|
||||
// Nothing proven anywhere: no in-memory result, no persisted proof.
|
||||
c := mergeCollector([]RestoreTest{}, []RestoreTest{})
|
||||
if got := c.collectRestoreTests(context.Background()); len(got) != 0 {
|
||||
t.Fatalf("a tier with no proof must produce NO entry — an unproven tier reading as proven is "+
|
||||
"worse than the defect being fixed; got %+v", got)
|
||||
}
|
||||
|
||||
// And an entry the state layer could not describe (no tier) is never promoted into a proof.
|
||||
c2 := mergeCollector([]RestoreTest{}, []RestoreTest{
|
||||
{SourceArchive: "local:backup/x.tar.zst", SourceTier: "", Pass: true,
|
||||
TestedAt: time.Date(2026, 8, 1, 4, 41, 58, 0, time.UTC).Format(time.RFC3339)},
|
||||
})
|
||||
if got := c2.collectRestoreTests(context.Background()); len(got) != 0 {
|
||||
t.Fatalf("a persisted record with no tier is not a usable proof and must be dropped; got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO C — a fresh in-memory result wins, and never duplicates ─────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-03): remove the de-duplication (append every persisted entry
|
||||
// unconditionally) →
|
||||
//
|
||||
// --- FAIL: TestMerge_NewerWinsAndNeverDuplicatesATier
|
||||
// restoretest_merge_test.go: one entry per tier; got 2 for "pbs" — the hub would read two tests
|
||||
//
|
||||
// Restored.
|
||||
func TestMerge_NewerWinsAndNeverDuplicatesATier(t *testing.T) {
|
||||
lastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC) // jittered, from the real box
|
||||
fiveMinAgo := time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC)
|
||||
|
||||
c := mergeCollector(
|
||||
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)},
|
||||
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)},
|
||||
)
|
||||
got := c.collectRestoreTests(context.Background())
|
||||
e, n := findTier(got, "pbs")
|
||||
if n != 1 {
|
||||
t.Fatalf("one entry per tier; got %d for \"pbs\" — the hub would read two tests: %+v", n, got)
|
||||
}
|
||||
if e.SourceArchive != "felhom-pbs:backup/ct/9201/new" {
|
||||
t.Fatalf("the NEWER result must win; got %q tested %q", e.SourceArchive, e.TestedAt)
|
||||
}
|
||||
|
||||
// ...and the older-in-memory / newer-persisted direction, which is the post-restart case.
|
||||
c2 := mergeCollector(
|
||||
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, lastWeek)},
|
||||
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", true, fiveMinAgo)},
|
||||
)
|
||||
e2, n2 := findTier(c2.collectRestoreTests(context.Background()), "pbs")
|
||||
if n2 != 1 || e2.SourceArchive != "felhom-pbs:backup/ct/9201/new" {
|
||||
t.Fatalf("newest must win regardless of which source it came from; got %d entr(ies), archive %q", n2, e2.SourceArchive)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO D — a failure still reaches the hub ─────────────────────────────────────────────
|
||||
//
|
||||
// The merge must not mask a failure with an older stored success. A failing tier is retried at the
|
||||
// next evaluation and its record lives ONLY in memory, so losing it here would silence the loudest
|
||||
// DR signal this system produces.
|
||||
func TestMerge_AFailureIsStillReported(t *testing.T) {
|
||||
provenLastWeek := time.Date(2026, 7, 27, 19, 55, 41, 0, time.UTC)
|
||||
failedJustNow := time.Date(2026, 8, 3, 13, 41, 7, 0, time.UTC)
|
||||
|
||||
c := mergeCollector(
|
||||
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/new", false, failedJustNow)},
|
||||
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/old", true, provenLastWeek)},
|
||||
)
|
||||
e, n := findTier(c.collectRestoreTests(context.Background()), "pbs")
|
||||
if n != 1 {
|
||||
t.Fatalf("one entry per tier; got %d: %+v", n, c.collectRestoreTests(context.Background()))
|
||||
}
|
||||
if e.Pass {
|
||||
t.Fatalf("a FAILURE newer than the stored proof must be what is reported — masking it would "+
|
||||
"silence the loudest DR signal there is; got pass=%v archive=%q", e.Pass, e.SourceArchive)
|
||||
}
|
||||
}
|
||||
|
||||
// Two different tiers are both reported — the merge is per tier, not a single slot.
|
||||
func TestMerge_BothTiersSurvive(t *testing.T) {
|
||||
c := mergeCollector(
|
||||
[]RestoreTest{rt("local", "felhom-backup:backup/vzdump-lxc-9201-a.tar.zst", true,
|
||||
time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC))},
|
||||
[]RestoreTest{rt("pbs", "felhom-pbs:backup/ct/9201/x", true,
|
||||
time.Date(2026, 8, 2, 5, 12, 33, 0, time.UTC))},
|
||||
)
|
||||
got := c.collectRestoreTests(context.Background())
|
||||
if _, n := findTier(got, "local"); n != 1 {
|
||||
t.Fatalf("the in-memory tier must survive the merge; got %+v", got)
|
||||
}
|
||||
if _, n := findTier(got, "pbs"); n != 1 {
|
||||
t.Fatalf("the persisted tier must survive the merge; got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A malformed timestamp must never displace a good entry — "unparseable" is not "newest".
|
||||
func TestMerge_MalformedTimestampNeverWins(t *testing.T) {
|
||||
good := rt("pbs", "felhom-pbs:backup/ct/9201/good", true, time.Date(2026, 8, 3, 13, 25, 14, 0, time.UTC))
|
||||
bad := RestoreTest{SourceArchive: "felhom-pbs:backup/ct/9201/bad", SourceTier: "pbs", Pass: true, TestedAt: "not-a-time"}
|
||||
|
||||
c := mergeCollector([]RestoreTest{good}, []RestoreTest{bad})
|
||||
e, n := findTier(c.collectRestoreTests(context.Background()), "pbs")
|
||||
if n != 1 || e.SourceArchive != "felhom-pbs:backup/ct/9201/good" {
|
||||
t.Fatalf("an unparseable timestamp must not displace a good entry; got %d entr(ies), archive %q", n, e.SourceArchive)
|
||||
}
|
||||
}
|
||||
|
||||
// A nil proven-source leaves the pre-R-189 behaviour exactly as it was.
|
||||
func TestMerge_NilProvenSourceIsANoOp(t *testing.T) {
|
||||
only := rt("local", "felhom-backup:backup/x.tar.zst", true, time.Date(2026, 8, 3, 4, 44, 50, 0, time.UTC))
|
||||
c := mergeCollector([]RestoreTest{only}, nil)
|
||||
got := c.collectRestoreTests(context.Background())
|
||||
if len(got) != 1 || got[0].SourceArchive != only.SourceArchive {
|
||||
t.Fatalf("a nil durable source must not change anything; got %+v", got)
|
||||
}
|
||||
}
|
||||
+13
-3
@@ -134,6 +134,9 @@
|
||||
"audit_tail": [],
|
||||
"capabilities": [],
|
||||
"leaf_fingerprint": "60b5974d586f5f3c8ec41eb998d0f07406178219c36bf6d3ff377570279d8245",
|
||||
"addresses": [
|
||||
{ "iface": "vmbr0", "cidr": "192.168.0.162/24" }
|
||||
],
|
||||
"dr_recipe": {
|
||||
"recipe_version": 1,
|
||||
"guests": [
|
||||
@@ -141,7 +144,8 @@
|
||||
],
|
||||
"pbs": {
|
||||
"repo_id": "felhom-pbs",
|
||||
"namespace": "root",
|
||||
"namespace": "felhom-spike",
|
||||
"namespace_state": "resolved",
|
||||
"latest_snapshot_id": "9001"
|
||||
},
|
||||
"drives": [
|
||||
@@ -154,7 +158,13 @@
|
||||
],
|
||||
"pve_storage": [
|
||||
{ "name": "local-lvm", "type": "lvmthin", "content": "rootdir,images" },
|
||||
{ "name": "usb-backup", "type": "usb", "content": "backup" }
|
||||
]
|
||||
{ "name": "usb-backup", "type": "usb", "content": "backup" },
|
||||
{ "name": "felhom-pbs", "type": "pbs", "content": "backup" }
|
||||
],
|
||||
"backup_target": {
|
||||
"state": "resolved",
|
||||
"storage_id": "usb-backup",
|
||||
"mount_path": "/mnt/usb-backup"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// R-88 Part 2 — the agent gains a third state.
|
||||
//
|
||||
// `newestArchiveOn` promised in its own doc comment that "errors degrade to unknown, never to
|
||||
// no-backup", while its `(time.Time, bool)` return made that impossible: an error and a genuine
|
||||
// not-found both produced `(zero, false)`, so `/backup/due` answered a POSITIVE
|
||||
// "no successful backup recorded yet" with a nil age. The controller read that as "never backed up"
|
||||
// and fired its window-gate safety valve, quiescing customer app stacks outside the backup window.
|
||||
//
|
||||
// These tests assert the WIRE, because the wire is the contract another component reads.
|
||||
|
||||
// listerBackups is a fakeBackups that also implements BackupArchiveLister, with a controllable outcome.
|
||||
type listerBackups struct {
|
||||
fakeBackups
|
||||
t time.Time
|
||||
found bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (l *listerBackups) NewestArchiveTime(context.Context, int) (time.Time, bool, error) {
|
||||
return l.t, l.found, l.err
|
||||
}
|
||||
|
||||
// dueWithLister builds a server whose single tier's service is the given lister, and returns the
|
||||
// /backup/due response.
|
||||
// NOTE: the server must receive `lb` ITSELF, not its embedded fakeBackups — the tier's Service is
|
||||
// type-asserted to BackupArchiveLister, and the embedded value does not satisfy it. Passing the
|
||||
// inner struct silently routes every case to archiveUnknown, which looks like a code bug and is not.
|
||||
func dueWithLister(t *testing.T, lb *listerBackups, store *fakeStore) BackupDueResponse {
|
||||
t.Helper()
|
||||
srv, err := NewServer(Options{
|
||||
ListenAddr: "127.0.0.1:0",
|
||||
Guests: &fakeGuests{},
|
||||
Backups: lb,
|
||||
Store: store,
|
||||
Storage: fakeStorage{},
|
||||
Tokens: staticTokens{"A": 8200, "B": 9300},
|
||||
BackupCadence: 24 * time.Hour,
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewServer: %v", err)
|
||||
}
|
||||
srv.now = func() time.Time { return testNow }
|
||||
return dueOf(t, srv.Handler())
|
||||
}
|
||||
|
||||
// ── SCENARIO A (agent half) — an unreadable storage is UNKNOWN, not "never" ──────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): make newestArchiveOn return archiveAbsent on the error path (the
|
||||
// pre-fix collapse) and this fails with
|
||||
//
|
||||
// "an unreadable storage must report age_state=unknown, got \"absent\" — that is a POSITIVE claim
|
||||
// of 'never backed up' built out of two absences"
|
||||
//
|
||||
// Restored.
|
||||
func TestAgeState_UnreadableStorageIsUnknown(t *testing.T) {
|
||||
lb := &listerBackups{err: errors.New("proxmox: GET storage content: 500 connection refused")}
|
||||
got := dueWithLister(t, lb, &fakeStore{}) // empty store = cold in-memory record
|
||||
|
||||
if got.AgeState != AgeStateUnknown {
|
||||
t.Fatalf("an unreadable storage must report age_state=%q, got %q — that is a POSITIVE claim of "+
|
||||
"'never backed up' built out of two absences", AgeStateUnknown, got.AgeState)
|
||||
}
|
||||
if !got.Due {
|
||||
t.Fatal("FAIL-SAFE DIRECTION: unknown must still be DUE — an unreadable storage must never suppress a backup")
|
||||
}
|
||||
if got.AgeSecs != nil {
|
||||
t.Fatalf("an unknown age must not invent a number; got %d", *got.AgeSecs)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO B (agent half) — a genuine first-ever backup is ABSENT ──────────────────────────
|
||||
//
|
||||
// B is what makes A safe: an implementation that reported everything as "unknown" would pass A and
|
||||
// silently starve a brand-new box, because the controller only fires the first-backup valve on ABSENT.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): make newestArchiveOn return archiveUnknown when !found and this
|
||||
// fails with
|
||||
//
|
||||
// "a genuine never-backed-up tier must report age_state=\"absent\", got \"unknown\" — the
|
||||
// controller only licenses a first backup outside the window on ABSENT"
|
||||
//
|
||||
// Restored.
|
||||
func TestAgeState_GenuinelyNeverIsAbsent(t *testing.T) {
|
||||
lb := &listerBackups{found: false} // read SUCCEEDED, nothing there
|
||||
got := dueWithLister(t, lb, &fakeStore{})
|
||||
|
||||
if got.AgeState != AgeStateAbsent {
|
||||
t.Fatalf("a genuine never-backed-up tier must report age_state=%q, got %q — the controller only "+
|
||||
"licenses a first backup outside the window on ABSENT", AgeStateAbsent, got.AgeState)
|
||||
}
|
||||
if !got.Due {
|
||||
t.Fatal("a never-backed-up tier must be due")
|
||||
}
|
||||
}
|
||||
|
||||
// A real archive → known, with a real age.
|
||||
func TestAgeState_FoundIsKnown(t *testing.T) {
|
||||
lb := &listerBackups{t: testNow.Add(-2 * time.Hour), found: true}
|
||||
got := dueWithLister(t, lb, &fakeStore{})
|
||||
|
||||
if got.AgeState != AgeStateKnown {
|
||||
t.Fatalf("a readable archive must report age_state=%q, got %q", AgeStateKnown, got.AgeState)
|
||||
}
|
||||
if got.AgeSecs == nil || *got.AgeSecs < 7100 || *got.AgeSecs > 7300 {
|
||||
t.Fatalf("expected ~7200s age, got %v", got.AgeSecs)
|
||||
}
|
||||
if got.Due {
|
||||
t.Fatal("2h old against a 24h cadence is not due")
|
||||
}
|
||||
}
|
||||
|
||||
// An unparseable in-memory timestamp is UNKNOWN too — a backup DID happen, we just cannot date it.
|
||||
// Reporting "absent" there would be the same false-positive claim in a different costume.
|
||||
func TestAgeState_UnparseableTimestampIsUnknown(t *testing.T) {
|
||||
st := &fakeStore{backups: []hub.Backup{{VMID: 8200, Success: true, StartedAt: "not-a-timestamp"}}}
|
||||
lb := &listerBackups{err: errors.New("storage unreadable")}
|
||||
got := dueWithLister(t, lb, st)
|
||||
|
||||
if got.AgeState != AgeStateUnknown {
|
||||
t.Fatalf("an unparseable backup time means we cannot DATE a backup that exists — want %q, got %q",
|
||||
AgeStateUnknown, got.AgeState)
|
||||
}
|
||||
if !got.Due {
|
||||
t.Fatal("still due — fail safe toward taking a backup")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO D (agent half) — additive on the wire ───────────────────────────────────────────
|
||||
//
|
||||
// An OLD controller decodes into a struct without `age_state` and ignores it. What it MUST still see
|
||||
// unchanged is every pre-existing field.
|
||||
func TestAgeState_IsAdditive_PreExistingFieldsUnchanged(t *testing.T) {
|
||||
lb := &listerBackups{t: testNow.Add(-48 * time.Hour), found: true}
|
||||
got := dueWithLister(t, lb, &fakeStore{})
|
||||
|
||||
if !got.Due || got.Reason != "older than cadence" {
|
||||
t.Fatalf("pre-existing due/reason semantics changed: due=%v reason=%q", got.Due, got.Reason)
|
||||
}
|
||||
if got.AgeSecs == nil {
|
||||
t.Fatal("age_seconds must still be present for a known age")
|
||||
}
|
||||
// And the state rides alongside rather than replacing anything.
|
||||
if got.AgeState != AgeStateKnown {
|
||||
t.Fatalf("age_state should be %q, got %q", AgeStateKnown, got.AgeState)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BackupTargetWrapperPath is the pinned sudoers vector (configs/felhom-backup-target-apply). The
|
||||
// agent cannot create a PVE storage or grant an ACL itself — Datastore.Allocate at /storage and
|
||||
// Permissions.Modify are deliberately outside its role — so the privileged half runs here.
|
||||
const BackupTargetWrapperPath = "/usr/local/sbin/felhom-backup-target-apply"
|
||||
|
||||
// backupTargetRequest is POST /backup/target: move the PRIMARY whole-guest backup tier onto the
|
||||
// drive mounted at Where, creating the storage if needed.
|
||||
type backupTargetRequest struct {
|
||||
VMID int `json:"vmid"`
|
||||
Where string `json:"where"` // the drive's OWN host mountpoint (F-1)
|
||||
ID string `json:"id,omitempty"` // storage id; default backupTargetStorageID
|
||||
}
|
||||
|
||||
// backupTargetStorageID is the conventional id, matching what E-1 created by hand on both demo boxes.
|
||||
// Keeping the name identical is what makes this endpoint IDEMPOTENT on an already-migrated box: the
|
||||
// wrapper accepts an existing entry with the same path and changes nothing.
|
||||
const backupTargetStorageID = "felhom-backup"
|
||||
|
||||
// handleSetBackupTarget performs the whole move as one ordered operation: create the storage, grant
|
||||
// the agent access, repoint the primary tier in agent.json, and hand back what the caller must do to
|
||||
// make it take effect.
|
||||
//
|
||||
// THE ORDER IS THE DESIGN, and each step is a precondition for the next:
|
||||
//
|
||||
// create → grant → config
|
||||
//
|
||||
// Reversed, a config pointing at a storage that does not exist would make the tier DEFER (harmless
|
||||
// but silent), and a config pointing at an ungranted storage would make every backup 403 on its
|
||||
// first run — which is exactly what E-1 hit when the grant was forgotten (finding F-3). Creating and
|
||||
// granting BEFORE the config means the worst interruption leaves an unused storage, never a broken
|
||||
// tier.
|
||||
//
|
||||
// IT DOES NOT RESTART THE AGENT. That is deliberate and it is the E-1 lesson encoded: the backup
|
||||
// tiers are built once at daemon start, so the move needs a restart to take effect — but restarting
|
||||
// while a backup or restore-test is in flight cancels the wait and records a SPURIOUS tier failure
|
||||
// for a backup that actually succeeded (E-1 did exactly this to a felhom-pbs run). A restart that
|
||||
// this handler fires itself could never be re-checked against in-flight work by the caller, so the
|
||||
// response reports `restart_required` and the caller performs it behind its own immediate
|
||||
// in-flight check.
|
||||
func (s *Server) handleSetBackupTarget(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.privileged == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "privileged runner not configured on this host")
|
||||
return
|
||||
}
|
||||
var req backupTargetRequest
|
||||
if !decodeBody(w, r, &req) {
|
||||
return
|
||||
}
|
||||
if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
|
||||
return
|
||||
}
|
||||
where := strings.TrimSpace(req.Where)
|
||||
if where == "" {
|
||||
writeErr(w, http.StatusBadRequest, "where (the drive's own mountpoint) is required")
|
||||
return
|
||||
}
|
||||
id := strings.TrimSpace(req.ID)
|
||||
if id == "" {
|
||||
id = backupTargetStorageID
|
||||
}
|
||||
|
||||
// AGENT-SIDE VALIDATION FIRST, from the agent's own storage view — never the caller's claim.
|
||||
// The wrapper re-checks everything as root (it is the security boundary), but refusing here gives
|
||||
// the customer a reason instead of a shell error, and keeps a bad request from reaching sudo at all.
|
||||
if err := s.validateBackupTargetMount(r.Context(), where); err != nil {
|
||||
s.logger.Warn("local-api: backup-target move refused", "vmid", vmid, "where", where, "err", err)
|
||||
writeErr(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if _, errOut, err := s.privileged.Run(r.Context(), BackupTargetWrapperPath, "create", id, where); err != nil {
|
||||
s.logger.Error("local-api: backup-target create failed", "id", id, "where", where, "err", err, "stderr", string(errOut))
|
||||
writeErr(w, http.StatusBadGateway, "could not create the backup storage: "+wrapperReason(errOut, err))
|
||||
return
|
||||
}
|
||||
if _, errOut, err := s.privileged.Run(r.Context(), BackupTargetWrapperPath, "grant", id); err != nil {
|
||||
// The storage exists but the agent cannot write to it. Say so precisely: this is the exact
|
||||
// state that produced E-1's "403 permission denied at /storage/felhom-backup" on first backup.
|
||||
s.logger.Error("local-api: backup-target grant failed", "id", id, "err", err, "stderr", string(errOut))
|
||||
writeErr(w, http.StatusBadGateway, "storage created but the access grant failed — backups would 403: "+wrapperReason(errOut, err))
|
||||
return
|
||||
}
|
||||
if err := s.setConfiguredBackupTarget(id); err != nil {
|
||||
s.logger.Error("local-api: backup-target config write failed", "id", id, "err", err)
|
||||
writeErr(w, http.StatusInternalServerError, "storage is ready but the config could not be updated: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
s.logger.Info("local-api: backup target moved — RESTART REQUIRED for it to take effect",
|
||||
"vmid", vmid, "target", id, "where", where)
|
||||
writeOK(w, map[string]any{
|
||||
"vmid": vmid, "target": id, "where": where,
|
||||
// The caller must restart the agent BEHIND ITS OWN in-flight check — see the doc comment.
|
||||
"restart_required": true,
|
||||
})
|
||||
}
|
||||
|
||||
// validateBackupTargetMount refuses a mount that cannot be a real backup target, from the agent's own
|
||||
// storage view + mount table. Mirrors the wrapper's laws so the customer gets a reason, not a shell error.
|
||||
func (s *Server) validateBackupTargetMount(ctx context.Context, where string) error {
|
||||
if s.storage == nil {
|
||||
return fmt.Errorf("storage view unavailable")
|
||||
}
|
||||
// It must currently BE a mountpoint (F-1/F-2). Resolved from the mount table, which is the same
|
||||
// source the wrapper's `mountpoint -q` consults.
|
||||
mounts, err := s.hostReader().Mounts()
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not read the mount table")
|
||||
}
|
||||
var dev string
|
||||
for _, m := range mounts {
|
||||
if m.MountPoint == where {
|
||||
dev = m.Device
|
||||
break
|
||||
}
|
||||
}
|
||||
if dev == "" {
|
||||
return fmt.Errorf("%s is not a mountpoint — the backup target must be the drive's own mountpoint", where)
|
||||
}
|
||||
// Never the system disk: a target there protects against corruption only, never drive loss.
|
||||
for _, m := range mounts {
|
||||
if m.MountPoint == "/" && m.Device == dev {
|
||||
return fmt.Errorf("%s is on the system disk — a backup target there cannot survive a drive failure", where)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// setConfiguredBackupTarget rewrites backup.local_backup_target in agent.json.
|
||||
//
|
||||
// Read-modify-write over map[string]json.RawMessage so UNKNOWN KEYS ARE PRESERVED VERBATIM — the
|
||||
// same discipline as pbsdr.seedEscrowStorageID, and the property that made E-1's hand edit safe to
|
||||
// begin with. A typed round-trip would silently drop any key this build does not know about.
|
||||
//
|
||||
// Written IN PLACE (O_TRUNC), not tmp+rename: /etc/felhom-agent is root-owned while agent.json is
|
||||
// agent-owned 0600, so the non-root agent cannot rename into that directory. A recovery copy is
|
||||
// parked first, so a torn write is recoverable.
|
||||
func (s *Server) setConfiguredBackupTarget(id string) error {
|
||||
path := s.configPath
|
||||
if path == "" {
|
||||
return fmt.Errorf("no config path known to this agent (env-only config)")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var doc map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return fmt.Errorf("parse %s: %w", path, err)
|
||||
}
|
||||
var bk map[string]json.RawMessage
|
||||
if cur, ok := doc["backup"]; ok {
|
||||
if err := json.Unmarshal(cur, &bk); err != nil {
|
||||
return fmt.Errorf("parse backup section: %w", err)
|
||||
}
|
||||
} else {
|
||||
bk = map[string]json.RawMessage{}
|
||||
}
|
||||
idJSON, _ := json.Marshal(id)
|
||||
bk["local_backup_target"] = idJSON
|
||||
bkJSON, err := json.Marshal(bk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
doc["backup"] = bkJSON
|
||||
out, err := json.MarshalIndent(doc, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s.stateDir != "" {
|
||||
if err := os.MkdirAll(s.stateDir, 0o700); err == nil {
|
||||
_ = os.WriteFile(filepath.Join(s.stateDir, "agent.json.pre-backup-target"), raw, 0o600)
|
||||
}
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, st.Mode().Perm())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write(out); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// wrapperReason surfaces the wrapper's own REFUSED line when it produced one — it explains WHY in
|
||||
// terms the customer can act on ("not a mountpoint", "already exists at …") — falling back to the
|
||||
// exec error only when stderr said nothing useful.
|
||||
func wrapperReason(errOut []byte, err error) string {
|
||||
for _, line := range strings.Split(string(errOut), "\n") {
|
||||
if strings.Contains(line, "REFUSED:") {
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// backupTargetServer builds a server whose PRIMARY tier is `felhom-backup`, mounted at /mnt/nvme-1tb
|
||||
// on its own non-system device — i.e. the exact live shape E-1 created on demo-hp and demo-felhom:
|
||||
// the drive is simultaneously the enrolled user-data drive AND the whole-guest vzdump target.
|
||||
func backupTargetServer(t *testing.T) http.Handler {
|
||||
t.Helper()
|
||||
sv := fakeStorage{targets: []hub.StorageTarget{
|
||||
{
|
||||
Name: "felhom-backup", Type: hub.StorageTypeLocalDir, State: hub.StorageStateAttached,
|
||||
Reachable: true, MountPath: "/mnt/nvme-1tb", BackingDevice: "/dev/nvme0n1",
|
||||
Content: "backup",
|
||||
},
|
||||
{
|
||||
Name: "spare-drive", Type: hub.StorageTypeLocalDir, State: hub.StorageStateAttached,
|
||||
Reachable: true, MountPath: "/mnt/spare", BackingDevice: "/dev/sdz1",
|
||||
Content: "backup",
|
||||
},
|
||||
}}
|
||||
srv, err := NewServer(Options{
|
||||
ListenAddr: "127.0.0.1:0",
|
||||
Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{}, Storage: sv,
|
||||
Tokens: staticTokens{"A": 8200},
|
||||
Disks: &fakeDiskOps{}, DiskGate: &fakeGate{}, HostReader: sysOnSDA(),
|
||||
// Service is load-bearing: normalizeBackupTiers DROPS any tier with a nil Service and falls
|
||||
// back to the legacy single tier with an empty TargetID — which silently made an earlier
|
||||
// version of this test exercise nothing.
|
||||
BackupTiers: []BackupTier{
|
||||
{TargetID: "felhom-backup", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
|
||||
{TargetID: "felhom-pbs", Cadence: 168 * time.Hour, Service: &fakeBackups{}},
|
||||
},
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("new server: %v", err)
|
||||
}
|
||||
srv.baseCtx = context.Background()
|
||||
srv.now = func() time.Time { return testNow }
|
||||
return srv.Handler()
|
||||
}
|
||||
|
||||
// E-2c — ejecting the drive that holds the only local whole-guest backup must be refused.
|
||||
//
|
||||
// This is a REGRESSION GUARD on a live configuration, not a hypothetical. E-1 (2026-07-28) moved the
|
||||
// vzdump target onto each demo box's secondary drive, and `RoleForStorage` types a local-dir on a
|
||||
// non-system device as user-data — so the pre-existing role gate PASSES it and the customer could
|
||||
// self-serve eject the drive holding their backups. It would have succeeded silently.
|
||||
//
|
||||
// The assertion is on the CONSEQUENCE (the request is refused) plus the remedy being named, because a
|
||||
// refusal the customer cannot act on just moves the failure.
|
||||
func TestEjectRefusedOnTheBackupTargetDrive(t *testing.T) {
|
||||
h := backupTargetServer(t)
|
||||
rr := do(t, h, http.MethodPost, "/disks/eject", "A", `{"vmid":8200,"where":"/mnt/nvme-1tb"}`)
|
||||
if rr.Code == http.StatusOK {
|
||||
t.Fatalf("eject of the backup-target drive SUCCEEDED (%d) — the box would silently lose its "+
|
||||
"local drive-loss protection with nothing alarming", rr.Code)
|
||||
}
|
||||
body := rr.Body.String()
|
||||
if !strings.Contains(body, "felhom-backup") {
|
||||
t.Errorf("refusal must NAME the backup target so the customer knows which role blocks it; got: %s", body)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(body), "reassign") {
|
||||
t.Errorf("refusal must name the REMEDY (reassign the target first), else it is a dead end; got: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Decommission strands the target just as thoroughly as eject — it migrates data off and retires the
|
||||
// drive. Same gate, asserted separately because it is a different handler and a different caller.
|
||||
func TestDecommissionRefusedOnTheBackupTargetDrive(t *testing.T) {
|
||||
h := backupTargetServer(t)
|
||||
rr := do(t, h, http.MethodPost, "/disks/decommission", "A", `{"vmid":8200,"where":"/mnt/nvme-1tb"}`)
|
||||
if rr.Code == http.StatusOK {
|
||||
t.Fatalf("decommission of the backup-target drive SUCCEEDED (%d)", rr.Code)
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "felhom-backup") {
|
||||
t.Errorf("refusal must name the backup target; got: %s", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// THE OVER-CORRECTION GUARD, and the reason this is a narrow gate instead of a role reclassification.
|
||||
//
|
||||
// The tempting fix — make RoleForStorage return RoleBackup for the target — would also refuse every
|
||||
// OTHER user-data drive op on a box, and on the demo boxes it would refuse the customer's own data
|
||||
// drive, because that drive IS the target. This pins that a non-target drive stays ejectable: the new
|
||||
// gate must block exactly one drive, not harden the whole eject path.
|
||||
//
|
||||
// It asserts "not blocked BY THIS GATE" rather than "succeeds", because eject has other legitimate
|
||||
// failure modes in a fake harness; what must never appear is this gate's message.
|
||||
func TestEjectStillAllowedOnANonTargetDrive(t *testing.T) {
|
||||
h := backupTargetServer(t)
|
||||
rr := do(t, h, http.MethodPost, "/disks/eject", "A", `{"vmid":8200,"where":"/mnt/spare"}`)
|
||||
if strings.Contains(rr.Body.String(), "whole-guest backup target") {
|
||||
t.Fatalf("the backup-target gate blocked a NON-target drive (/mnt/spare) — over-correction: "+
|
||||
"it must block exactly the target, not harden the whole eject path; got: %s", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// E-2 — GET /disks must FLAG the backup-target drive, because the controller cannot work it out.
|
||||
//
|
||||
// The controller's own StoragePath.BackupTarget is customer INTENT, and on a box migrated by hand
|
||||
// (E-1, both demo boxes) nobody ever assigned it — intent is empty while the drive really is the
|
||||
// target. Without this flag the absent-target alarm could not name the drive on exactly the boxes
|
||||
// that have one, which is the only place it currently matters.
|
||||
func TestDisksFlagsTheBackupTargetDrive(t *testing.T) {
|
||||
h := backupTargetServer(t)
|
||||
rr := do(t, h, http.MethodGet, "/disks", "A", "")
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("GET /disks = %d, body %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
body := rr.Body.String()
|
||||
// The target must be flagged and the non-target must not be — asserted as a pair, since a
|
||||
// blanket true would satisfy a naive "is it flagged?" check.
|
||||
var got struct {
|
||||
Data struct {
|
||||
Disks []struct {
|
||||
Name string `json:"name"`
|
||||
BackupTarget bool `json:"backup_target"`
|
||||
} `json:"disks"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &got); err != nil {
|
||||
t.Fatalf("decode: %v (body %s)", err, body)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, d := range got.Data.Disks {
|
||||
seen[d.Name] = d.BackupTarget
|
||||
}
|
||||
if !seen["felhom-backup"] {
|
||||
t.Errorf("felhom-backup is the primary tier's storage but backup_target is false; body: %s", body)
|
||||
}
|
||||
if seen["spare-drive"] {
|
||||
t.Errorf("spare-drive is NOT the target but was flagged — a blanket true is not a signal; body: %s", body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
|
||||
// wrapperRunner captures the wrapper vector without ever running sudo. The wrapper IS the security
|
||||
// boundary, so tests substitute it rather than bypassing it — what is asserted here is the ORDER and
|
||||
// the ARGUMENTS the agent sends, which is the agent's half of the contract.
|
||||
type wrapperRunner struct {
|
||||
calls [][]string
|
||||
failOn string // verb to fail, "" = all succeed
|
||||
}
|
||||
|
||||
func (r *wrapperRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
r.calls = append(r.calls, append([]string{name}, args...))
|
||||
if len(args) > 0 && args[0] == r.failOn {
|
||||
return nil, []byte("felhom-backup-target-apply: REFUSED: synthetic " + r.failOn + " failure\n"),
|
||||
io.ErrUnexpectedEOF
|
||||
}
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func (r *wrapperRunner) verbs() []string {
|
||||
var out []string
|
||||
for _, c := range r.calls {
|
||||
if len(c) > 1 {
|
||||
out = append(out, c[1])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moveServer builds a server with /mnt/data mounted on its own device and / on another, plus a
|
||||
// throwaway agent.json the move can rewrite.
|
||||
func moveServer(t *testing.T, run *wrapperRunner) (http.Handler, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
cfgPath := filepath.Join(dir, "agent.json")
|
||||
// An UNKNOWN key is deliberately present: the rewrite must preserve it verbatim.
|
||||
seed := `{"backup":{"local_backup_target":"local","local_backup_retention":3},"some_future_key":{"keep":"me"}}`
|
||||
if err := os.WriteFile(cfgPath, []byte(seed), 0o600); err != nil {
|
||||
t.Fatalf("seed config: %v", err)
|
||||
}
|
||||
hr := fakeHostReader{mounts: []storage.Mount{
|
||||
{Device: "/dev/sda1", MountPoint: "/"},
|
||||
{Device: "/dev/sdb1", MountPoint: "/mnt/data"},
|
||||
}}
|
||||
srv, err := NewServer(Options{
|
||||
ListenAddr: "127.0.0.1:0",
|
||||
Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{}, Storage: fakeStorage{},
|
||||
Tokens: staticTokens{"A": 8200}, HostReader: hr,
|
||||
Privileged: run, ConfigPath: cfgPath, StateDir: dir,
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("new server: %v", err)
|
||||
}
|
||||
srv.baseCtx = context.Background()
|
||||
return srv.Handler(), cfgPath
|
||||
}
|
||||
|
||||
// THE ORDER IS THE CONTRACT: create → grant → config. Reversed, a config pointing at an ungranted
|
||||
// storage makes every backup 403 on first run, which is precisely what E-1 hit (finding F-3).
|
||||
func TestBackupTargetMoveOrdersCreateThenGrantThenConfig(t *testing.T) {
|
||||
run := &wrapperRunner{}
|
||||
h, cfgPath := moveServer(t, run)
|
||||
|
||||
rr := do(t, h, http.MethodPost, "/backup/target", "A", `{"vmid":8200,"where":"/mnt/data"}`)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("move = %d, body %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
got := strings.Join(run.verbs(), ",")
|
||||
if got != "create,grant" {
|
||||
t.Fatalf("wrapper verbs = %q, want create,grant (in that order)", got)
|
||||
}
|
||||
// The config must have been written only AFTER both wrapper calls succeeded.
|
||||
raw, _ := os.ReadFile(cfgPath)
|
||||
var doc map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
t.Fatalf("config unreadable after move: %v", err)
|
||||
}
|
||||
var bk map[string]any
|
||||
_ = json.Unmarshal(doc["backup"], &bk)
|
||||
if bk["local_backup_target"] != "felhom-backup" {
|
||||
t.Errorf("local_backup_target = %v, want felhom-backup", bk["local_backup_target"])
|
||||
}
|
||||
// Unknown keys preserved verbatim — the property that made E-1's hand edit safe.
|
||||
if _, ok := doc["some_future_key"]; !ok {
|
||||
t.Error("the rewrite DROPPED an unknown top-level key — a typed round-trip would do this " +
|
||||
"and silently discard config this build does not know about")
|
||||
}
|
||||
// Sibling keys inside `backup` survive too.
|
||||
if bk["local_backup_retention"] == nil {
|
||||
t.Error("the rewrite dropped local_backup_retention from the backup section")
|
||||
}
|
||||
}
|
||||
|
||||
// A FAILED GRANT MUST NOT LEAVE THE CONFIG POINTING AT THE NEW STORAGE. That state is exactly E-1's
|
||||
// 403-on-every-backup: the tier looks configured and cannot write.
|
||||
func TestBackupTargetMoveDoesNotRepointWhenTheGrantFails(t *testing.T) {
|
||||
run := &wrapperRunner{failOn: "grant"}
|
||||
h, cfgPath := moveServer(t, run)
|
||||
|
||||
rr := do(t, h, http.MethodPost, "/backup/target", "A", `{"vmid":8200,"where":"/mnt/data"}`)
|
||||
if rr.Code == http.StatusOK {
|
||||
t.Fatalf("move SUCCEEDED despite a failed grant (%d)", rr.Code)
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "403") {
|
||||
t.Errorf("the error should name the consequence (backups would 403); got %s", rr.Body.String())
|
||||
}
|
||||
raw, _ := os.ReadFile(cfgPath)
|
||||
if strings.Contains(string(raw), "felhom-backup") {
|
||||
t.Fatal("the config was repointed at a storage the agent cannot write to — every backup " +
|
||||
"would 403 while the tier reported as configured")
|
||||
}
|
||||
}
|
||||
|
||||
// It must NOT restart the agent itself. Restarting with a backup in flight cancels the wait and
|
||||
// records a spurious tier failure for a backup that actually succeeded — E-1 did exactly that to a
|
||||
// felhom-pbs run. Only the caller can re-check in-flight work immediately before restarting.
|
||||
func TestBackupTargetMoveReportsRestartRequiredRatherThanRestarting(t *testing.T) {
|
||||
run := &wrapperRunner{}
|
||||
h, _ := moveServer(t, run)
|
||||
rr := do(t, h, http.MethodPost, "/backup/target", "A", `{"vmid":8200,"where":"/mnt/data"}`)
|
||||
if !strings.Contains(rr.Body.String(), `"restart_required":true`) {
|
||||
t.Errorf("response must tell the caller a restart is required; got %s", rr.Body.String())
|
||||
}
|
||||
for _, c := range run.calls {
|
||||
joined := strings.Join(c, " ")
|
||||
if strings.Contains(joined, "systemctl") || strings.Contains(joined, "restart") {
|
||||
t.Fatalf("the handler restarted the agent itself: %q — the caller must do it behind its "+
|
||||
"own in-flight check", joined)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A path that is not a mountpoint is refused BEFORE sudo is reached (F-1): a subdirectory target
|
||||
// reports disconnected forever, and an unmounted path silently retargets onto the system drive.
|
||||
func TestBackupTargetMoveRefusesANonMountpoint(t *testing.T) {
|
||||
run := &wrapperRunner{}
|
||||
h, _ := moveServer(t, run)
|
||||
rr := do(t, h, http.MethodPost, "/backup/target", "A", `{"vmid":8200,"where":"/mnt/data/sub"}`)
|
||||
if rr.Code == http.StatusOK {
|
||||
t.Fatal("a non-mountpoint was accepted as the backup target")
|
||||
}
|
||||
if len(run.calls) != 0 {
|
||||
t.Errorf("a refused request still reached the privileged wrapper: %v", run.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// The system disk is refused: a target there protects against corruption only, never drive loss —
|
||||
// which is the entire point of the move.
|
||||
func TestBackupTargetMoveRefusesTheSystemDisk(t *testing.T) {
|
||||
run := &wrapperRunner{}
|
||||
h, _ := moveServer(t, run)
|
||||
rr := do(t, h, http.MethodPost, "/backup/target", "A", `{"vmid":8200,"where":"/"}`)
|
||||
if rr.Code == http.StatusOK {
|
||||
t.Fatal("the system disk was accepted as the backup target")
|
||||
}
|
||||
if len(run.calls) != 0 {
|
||||
t.Errorf("a refused request still reached the privileged wrapper: %v", run.calls)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user