Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1693da4d1a | |||
| 79ba884859 | |||
| f17e1979bd | |||
| 0963bae3c3 | |||
| 8561dc8a5c | |||
| f7b7308c4d | |||
| 27bbdaaa22 | |||
| e25454ccc3 | |||
| 65c1994d3b | |||
| 6545cccff7 | |||
| eb38d78945 | |||
| 4a1c07b288 | |||
| 5c3b218822 | |||
| e15d1d30ea | |||
| 344c4f9030 | |||
| c52d591851 | |||
| 37b99d73b3 | |||
| dd8b2ace02 | |||
| 383522450a | |||
| 1a8bed11e7 | |||
| 98f3cdd76a | |||
| 888a98571b | |||
| 912f7c127e | |||
| e569ec6c5f | |||
| 52ed52c131 |
@@ -16,7 +16,16 @@
|
|||||||
# A failing run must reach a person — a detector nobody hears is the defect R-29 filed, rebuilt one
|
# A failing run must reach a person — a detector nobody hears is the defect R-29 filed, rebuilt one
|
||||||
# layer up. That is the last step, and it runs ONLY on failure.
|
# layer up. That is the last step, and it runs ONLY on failure.
|
||||||
name: gates
|
name: gates
|
||||||
on: [push]
|
# R-1129: `workflow_dispatch` is the manual PROOF that the failure mail can be sent. A push run is
|
||||||
|
# unchanged by it: the proof step below is skipped on a push, and `force_alarm` is then empty.
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
force_alarm:
|
||||||
|
description: "Fail this run on purpose so the alarm step sends its real mail (a proof, not a gate result)"
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
gates:
|
gates:
|
||||||
@@ -57,6 +66,22 @@ jobs:
|
|||||||
GITEA_BASE: http://gitea.gitea-system.svc.cluster.local:3000
|
GITEA_BASE: http://gitea.gitea-system.svc.cluster.local:3000
|
||||||
run: cd ws/felhom-agent && python3 scripts/agent_gates.py
|
run: cd ws/felhom-agent && python3 scripts/agent_gates.py
|
||||||
|
|
||||||
|
- name: Manual proof - alarm key present, optional forced failure (dispatch only, R-1129)
|
||||||
|
# Runs ONLY on a manual dispatch, never on a push. It prints the key's LENGTH, never the key,
|
||||||
|
# so "CI can see the secret" is an observable rather than an assumption. With
|
||||||
|
# force_alarm=true it then fails on purpose so the alarm step below sends its real mail;
|
||||||
|
# the commit itself stays green, because its push run skips this step.
|
||||||
|
if: github.event_name == 'workflow_dispatch'
|
||||||
|
env:
|
||||||
|
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
||||||
|
FORCE_ALARM: ${{ github.event.inputs.force_alarm }}
|
||||||
|
run: |
|
||||||
|
python3 -c 'import os;print("RESEND_API_KEY length: %d" % len(os.environ.get("RESEND_API_KEY","")))'
|
||||||
|
if [ "$FORCE_ALARM" = "true" ]; then
|
||||||
|
echo "force_alarm=true - failing on purpose so the alarm step runs (R-1129 proof)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Alarm on failure
|
- name: Alarm on failure
|
||||||
# THE POINT OF THE WHOLE THING. Probe P5 measured that a failed run produces NO mail, NO
|
# THE POINT OF THE WHOLE THING. Probe P5 measured that a failed run produces NO mail, NO
|
||||||
# notification row and NO log line from Gitea itself — a red tick in a web UI nobody watches
|
# notification row and NO log line from Gitea itself — a red tick in a web UI nobody watches
|
||||||
@@ -70,6 +95,8 @@ jobs:
|
|||||||
if: failure()
|
if: failure()
|
||||||
env:
|
env:
|
||||||
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
||||||
|
# Empty on a push, so a push run's mail is exactly what it was before R-1129.
|
||||||
|
FORCE_ALARM: ${{ github.event.inputs.force_alarm }}
|
||||||
run: |
|
run: |
|
||||||
python3 - <<'PY'
|
python3 - <<'PY'
|
||||||
import json, os, sys, urllib.request, urllib.error
|
import json, os, sys, urllib.request, urllib.error
|
||||||
@@ -81,13 +108,23 @@ jobs:
|
|||||||
|
|
||||||
repo = os.environ.get("GITHUB_REPOSITORY", "?")
|
repo = os.environ.get("GITHUB_REPOSITORY", "?")
|
||||||
sha = os.environ.get("GITHUB_SHA", "?")
|
sha = os.environ.get("GITHUB_SHA", "?")
|
||||||
run = os.environ.get("GITHUB_RUN_NUMBER", "?")
|
# R-1135: the link is for a PERSON, so it is the public address and the run ID.
|
||||||
srv = os.environ.get("GITHUB_SERVER_URL", "https://gitea.dooplex.hu")
|
# NOT GITHUB_SERVER_URL: the runner registers at the in-cluster address, so that variable
|
||||||
|
# reads http://gitea.gitea-system.svc.cluster.local:3000, which nobody outside the cluster
|
||||||
|
# can open (forced-test mail, job 1997). NOT GITHUB_RUN_NUMBER: Gitea 1.26 reads
|
||||||
|
# /actions/runs/<n> as a run id first, so a run number that is also an older run's id
|
||||||
|
# opens THAT run (measured 2026-10-18: felhom.eu /actions/runs/1116 opened the run whose
|
||||||
|
# id is 1116 = run number 613). The link is printed too, so the job log carries it.
|
||||||
|
run = os.environ.get("GITHUB_RUN_ID", "?")
|
||||||
|
srv = "https://gitea.dooplex.hu"
|
||||||
|
print("ALARM-LINK %s/%s/actions/runs/%s" % (srv, repo, run))
|
||||||
|
|
||||||
body = json.dumps({
|
body = json.dumps({
|
||||||
"from": "Felhom CI <monitoring@felhom.eu>",
|
"from": "Felhom CI <monitoring@felhom.eu>",
|
||||||
"to": ["admin@felhom.eu"],
|
"to": ["admin@felhom.eu"],
|
||||||
"subject": "[felhom CI] gates FAILED in %s" % repo,
|
"subject": ("[felhom CI] FORCED TEST, not a real failure - gates FAILED in %s"
|
||||||
|
if os.environ.get("FORCE_ALARM") == "true"
|
||||||
|
else "[felhom CI] gates FAILED in %s") % repo,
|
||||||
"text": (
|
"text": (
|
||||||
"The gate entry point exited non-zero.\n\n"
|
"The gate entry point exited non-zero.\n\n"
|
||||||
"Repository : %s\n"
|
"Repository : %s\n"
|
||||||
|
|||||||
+224
@@ -1,3 +1,227 @@
|
|||||||
|
## Unreleased (2026-10-18) — CI workflow: the failure mail links the public run page, by run id (R-1135); no code change
|
||||||
|
|
||||||
|
`.gitea/workflows/gates.yml`, the alarm step: the mail's run link is now `https://gitea.dooplex.hu/<repo>/actions/runs/<run id>`.
|
||||||
|
It used `GITHUB_SERVER_URL`, which on this runner is the in-cluster address (`http://gitea.gitea-system.svc.cluster.local:3000`,
|
||||||
|
the address the runner registers at), so nobody outside the cluster could open it. It also used the run NUMBER, and
|
||||||
|
Gitea 1.26 reads `/actions/runs/<n>` as a run id first: a number that is also an older run's id opened that other run
|
||||||
|
(measured 2026-10-18: `felhom.eu/actions/runs/1116` opened run id 1116, run number 613). It now uses `GITHUB_RUN_ID` and
|
||||||
|
prints the link (`ALARM-LINK …`) into the job log. Same change in all four repos' workflows.
|
||||||
|
|
||||||
|
## Unreleased — three fixes from the 2026-10-12 security review and the storage build (R-1093, R-978, R-973) (2026-10-18)
|
||||||
|
|
||||||
|
Committed, not released — the release and its vouch are the operator's.
|
||||||
|
|
||||||
|
- **R-1093 — an EMPTY partition no longer asks the typed „data will be erased" confirmation.** `lsblk` prints the
|
||||||
|
parent disk's table type (PTTYPE) on every partition, so `SudoHostOps.InspectDevice` read a blank partition as
|
||||||
|
„device has a partition table". Now lsblk's PTTYPE is set aside only when `blkid -p` on the partition itself proved
|
||||||
|
it read the device and that it is a partition of a table of exactly that type (`PART_ENTRY_NUMBER` +
|
||||||
|
`PART_ENTRY_SCHEME` equal to lsblk's PTTYPE). The partition's own signatures still come from `blkid -p`: a
|
||||||
|
filesystem or a nested table inside it stays data-bearing; a blkid that printed nothing, a whole disk, or a scheme
|
||||||
|
that differs keeps the old, safe reading. **No new command, no sudoers or config-bundle change** (the two probes are
|
||||||
|
the same allow-listed argv). Test `TestInspect_R1093_PartitionVerdicts` (8 rows), red-proved: with the old
|
||||||
|
`if pttype != ""` both empty-partition rows fail `DataBearing=true reason="device has a partition table"`.
|
||||||
|
**Hardened after the same night's review (fail-closed on every path):** `PART_ENTRY_*` is read from the PARENT's
|
||||||
|
table, so lsblk's PTTYPE is set aside only when blkid also **exited 0** — exit 2, an I/O error or a missing binary
|
||||||
|
keep „device has a partition table"; and an `lsblk` that exits 0 with output that does not parse into a device (or
|
||||||
|
lists none) now leaves the device UNPROBED („could not be reliably inspected") instead of „blank" — that hole
|
||||||
|
predates R-1093. Six more rows in the same test, each red-proved (`blkidErr` check removed → „device is blank";
|
||||||
|
`&& parsed` removed → `Probed=true … „device is blank"`).
|
||||||
|
- **R-978 — an abandoned second escrow ceremony no longer makes the customer's recovery code useless.** The root
|
||||||
|
ceremony (the FELHOM_ESCROW argv, unchanged: `--upload --output=json`) used to upload the new package before the new
|
||||||
|
code was claimed, and the hub overwrites the current package when the sealed secrets are unchanged — so a customer
|
||||||
|
who started the ceremony again and walked away (or an agent restart in the 10-minute window) was left with a
|
||||||
|
written-down code that opened nothing. Now json mode hands the package back (`CeremonyOutput.PendingUpload`, output
|
||||||
|
**version 2**; text mode still uploads at once) and the daemon uploads it at `POST /escrow/ceremony/claim`, **before**
|
||||||
|
the code is handed out (`EscrowCeremonyConfig.Upload` → `putEscrowBody`, the same PUT and per-host key). A failed
|
||||||
|
upload answers 502 and keeps the code claimable; the TTL and a new start wait while the claim's upload is in flight;
|
||||||
|
no uploader → 503; an output that uploaded by itself or carries no package fails the ceremony. Status `uploaded`
|
||||||
|
now reads false until the claim. The controller is unchanged (it never read `uploaded`; its claim already assumed
|
||||||
|
the package is on the hub when the claim returns). **No sudoers or config-bundle change.** Tests (localapi)
|
||||||
|
`TestEscrowCeremony_R978_*` (abandon by TTL / by a new start / by a restart → the hub's package still opens with
|
||||||
|
code 1; claim stores before handing out; upload failure; in-flight TTL; no uploader; no package) and (cmd)
|
||||||
|
`TestEscrowOptsFromCLI_JSONModeDefersUpload`, `TestDeliverEscrowPackage_JSONModeSendsNothing`,
|
||||||
|
`TestR978_CeremonyUploadIsWiredIntoTheDaemon`. Red-proved: upload-at-create restored → „code 1 no longer opens it
|
||||||
|
(uploads=2)"; the in-flight TTL guard removed → „the TTL voided the code while its package was uploading";
|
||||||
|
`deferUpload: false` → „json mode must defer the upload"; the `Upload:` line removed → „has no Upload". **Residual:**
|
||||||
|
the customer's typed-back check after the claim is browser-only, so „confirmed" here means „claimed" — a customer
|
||||||
|
who opens the code and closes the page without writing it down has replaced the package.
|
||||||
|
- **R-973 — the local-API whole-guest backup route has a cadence cap.** Every successful run prunes its tier, so a
|
||||||
|
compromised controller calling `POST /backup` in a loop pushed out every pre-compromise restore point (local
|
||||||
|
keep-last 3; off-site via ep0's keep-last 2). Now (`internal/localapi/backup_cap.go`): a tier whose cadence is ≤ 24 h
|
||||||
|
allows **3 successful runs per rolling 24 h** (the `07` §6 day: the night run, a „Mentés most" press, and the next
|
||||||
|
night, which since R-899 still runs and can land inside the same 24 h — so a press never costs the night); a longer
|
||||||
|
tier allows **1 per max(cadence − 24 h, cadence/2)** — 6 days for the 7-day off-site tier, which the controller only
|
||||||
|
runs when due and the button never targets. Only successes count (a failed run prunes nothing). A refusal answers
|
||||||
|
**429** with the next allowed time, is logged at WARN, is saved, and for 24 h raises the CRITICAL host-report status
|
||||||
|
`localapi:backup-cadence-cap` — the hub's existing operator-only `agent_capability_degraded` event, **no hub change**.
|
||||||
|
**The cap survives an agent restart:** `BackupSuccessState` (the R-894 file) now also keeps the newest 8 success
|
||||||
|
times and the last refusal per tier and guest (additive JSON fields; a pre-R-973 file counts its one known
|
||||||
|
success). Tests `TestBackupCap_*` (an off-site flood keeps a pre-compromise copy for 5+ days; a local flood runs 3
|
||||||
|
a day; the scheduled day is never refused; restart; failures; the operator signal) + `TestBackupCadenceCap`,
|
||||||
|
`TestBackupSuccessState_R973_*`, `TestBackupCapStatus_*`, `TestR973_CapRefusalIsWiredIntoTheReport`. Red-proved:
|
||||||
|
the cap disabled → „the off-site tier lost every pre-compromise copy: 10 runs"; the disk history ignored → „after a
|
||||||
|
restart the 4th run in a day was allowed"; the wiring line removed → „never binds backupCapRefusal". **Limits, stated:**
|
||||||
|
with keep-last 3 the local tier's three points can still be replaced within one day (3 per day is the schedule's own
|
||||||
|
pace plus one press); the hub's event text reads „agent privileged capability degraded" (the type is generic). No
|
||||||
|
sudoers or bundle change.
|
||||||
|
|
||||||
|
## v0.159.2 — the orphan thin check runs only after a failed backup (R-1012 follow-up) (2026-10-18)
|
||||||
|
|
||||||
|
**Released 2026-10-18 (committed 2026-10-17): sha256 `4d98868e922d6c10665a6c0da9323e8627ea9c589094e81d27d741e625d9d8f9`, bundle `477704ea2980948433ffc34bb59f008cc11410fd9683d386bfd3e2f085d55bc9`; the vouch is the operator's.** No `configs/` change since v0.159.1. The independent review of 2026-10-18 found nothing in it. v0.159.0/v0.159.1 ran the R-1012 orphan check
|
||||||
|
before EVERY snapshot-mode whole-guest backup, and that check takes a short metadata snapshot of the live thin pool.
|
||||||
|
Now `BackupRunner` runs it only when the snapshot step has failed (the vzdump start or its task) — the moment the
|
||||||
|
answer is needed. The finding is still appended in plain words to the failed backup's error, exactly as before; a
|
||||||
|
successful backup never runs the check and keeps a clean record; stop mode and an unwired runner are unchanged. The
|
||||||
|
`thin-orphan-check` capability text says „after a failed whole-guest backup". Tests
|
||||||
|
`TestBackup_SnapshotFailure_NamesTheOrphan` (now: exactly 1 call, after the failure) and
|
||||||
|
`TestBackup_ThinCheck_SilentWhenNotApplicable` (now: 0 calls on a successful backup), red-proved against the v0.159.1
|
||||||
|
`runner.go`: failure legs „thin check ran 2 times, want 1", success leg `calls=1`. No root file, sudoers or bundle
|
||||||
|
change. (Installer side, felhom.eu, same night: `felhom-host-install.sh --rotate-recovery-only`, R-1141.)
|
||||||
|
**Proven live 2026-10-18 (demo-hp):** a green snapshot backup with 0 `felhom-thin-check` calls (the 0.159.1 night backup
|
||||||
|
had 1). The failed leg stays covered by the fixture test.
|
||||||
|
|
||||||
|
## v0.159.1 — the signed decommission prunes the guest-bind record (found by v0.159.0's own live proof) (2026-10-17)
|
||||||
|
|
||||||
|
**v0.159.0 is superseded — do not vouch it.** Everything of v0.159.0 below, plus: `signedjobs.DecommissionExecutor`
|
||||||
|
drops the drive from every guest's bind record (`SetGuestBinds`, wired to the agent's `GuestBindStore`), as the
|
||||||
|
self-serve decommission always did. Without it a drive decommissioned by an operator-signed job and then forgotten
|
||||||
|
(R-1124) left a record the 20-second guest-bind re-assert warned about every tick („reconcile: skipping non-enrolled
|
||||||
|
drive") — seen on demo-hp minutes after the R-1124 proof. Test `TestDecommissionExecutor_PrunesTheGuestBindRecord`,
|
||||||
|
red-proved. Released: tag `v0.159.1`, sha256 `de8f6ad9…aff9`, bundle `2af3d27d…7cd6` (the same root files as v0.159.0).
|
||||||
|
**Not vouched.** Delivery to a box on the v0.158.1 bundle: `agent_update 0.159.1` → `agent_config_update 0.159.0-step1`
|
||||||
|
(sha `1d65edb3…d6cf`, published) → `agent_config_update 0.159.1` (the bundle adds `felhom-thin-check`, R16).
|
||||||
|
|
||||||
|
## v0.159.0 — forget a decommissioned drive that is gone (R-1124); name an orphan thin device in a backup failure (R-1012); Go 1.27.2 (R-972) (2026-10-17)
|
||||||
|
|
||||||
|
**Superseded by v0.159.1 — do not vouch.** Task: „agent release, the catalog allow-list, break-glass rotation, off-site
|
||||||
|
history check" (felhom.eu `REPORT-agent-allowlist-rotation-2026-10-17.md`).
|
||||||
|
|
||||||
|
- **Go 1.27.2 + x/crypto v0.58.0** (R-972, `37b99d7`): the first release that carries them (v0.158.1 was built with
|
||||||
|
go1.26.0).
|
||||||
|
- **R-1124 — a decommissioned drive that is gone is forgotten.** `SudoHostOps.ForgetRetiredDrives` (agent start and the
|
||||||
|
20-second storage tick): for a Felhom drive unit whose intent is `decommissioned`, whose `/dev/disk/by-uuid` link is
|
||||||
|
gone and which is not mounted, `systemctl disable` + the new root verb `felhom-priv-apply unit-forget <unit>` +
|
||||||
|
`daemon-reload`. The unit is gone, so a later boot no longer waits for the drive and the agent stops listing it (no
|
||||||
|
„absent by UUID" line every 20 s). A decommissioned drive still plugged in KEEPS its unit and raw mount, so the
|
||||||
|
one-click re-enroll (H3) still works. `unit-forget` re-checks as root: a felhom by-UUID drive unit (not a network
|
||||||
|
unit), disabled, its `Where` not mounted, its by-UUID link absent — otherwise refused. Sudoers `FELHOM_MOUNT` gains the
|
||||||
|
one verb. Tests `TestForgetRetiredDrives_*` and `configs/test_felhom_priv_apply.py` `UnitForget`, red-proved.
|
||||||
|
**Live on demo-hp:** a loop-backed test drive enrolled through the local API, decommissioned by a signed job (kept
|
||||||
|
its unit while present), unplugged → 8 s later the unit and its wants link were removed and `GET /disks` dropped it.
|
||||||
|
- **R-1012 — an orphan thin device is NAMED in the backup failure, never removed.** New READ-ONLY root helper
|
||||||
|
`felhom-thin-check <vg> <pool>` (config bundle): LVM's thin ids before and after, the pool's ids through a metadata
|
||||||
|
snapshot it reserves and releases (refuses — exit 3 — when someone else holds one; releases on every exit; sends no
|
||||||
|
other pool message). `internal/thincheck`: an orphan = in the pool and in neither LVM list, on two reads 3 s apart;
|
||||||
|
fails open. `BackupRunner.SetThinCheck`: before a snapshot-mode backup (a WARN when found) and again after a failure —
|
||||||
|
the finding is appended in plain words to the error the controller mails to the operator, pointing at the new
|
||||||
|
runbook `whole-guest-backup-orphan-thin.md`. Sudoers `FELHOM_THINCHECK`, capability `thin-orphan-check`. Tests
|
||||||
|
`internal/thincheck`, `TestBackup_SnapshotFailure_NamesTheOrphan`, `configs/test_felhom_thin_check.py`, red-proved.
|
||||||
|
**Live on demo-hp:** on a scratch thin pool (a loop file on `/mnt/hdd_1`, never `pve/data`) the helper listed a made
|
||||||
|
orphan (device 7) and refused a snapshot held by someone else; on `pve/data` it read 1, 2 = LVM's and released; a
|
||||||
|
normal whole-guest backup of 9201 ran the check first and finished green.
|
||||||
|
- Released: tag `v0.159.0`, sha256 `3cfd0a25…edfed5`, bundle `40663c4f…6d681`; step bundle `0.159.0-step1`.
|
||||||
|
|
||||||
|
## CI only — a manual proof that the failure mail can be sent (R-1129, 2026-10-17)
|
||||||
|
|
||||||
|
No agent release; no binary changes.
|
||||||
|
|
||||||
|
`.gitea/workflows/gates.yml` gains a manual `workflow_dispatch` trigger with one input, `force_alarm` (default false), and
|
||||||
|
one step that runs ONLY on a dispatch: it prints the length of `RESEND_API_KEY` (never the key) and, with
|
||||||
|
`force_alarm=true`, fails on purpose so the existing alarm step sends its real mail; the mail's subject then starts
|
||||||
|
„FORCED TEST, not a real failure". A push run is unchanged: the new step is skipped and the subject is the old one. Same
|
||||||
|
change in all four repos' workflows. Purpose: prove the failure mail now goes out after the operator set the key
|
||||||
|
(every failed run until today ended „ALARM FAILED: RESEND_API_KEY is empty").
|
||||||
|
|
||||||
|
## scripts only — the decoy suite is a gate of the full run (R-1116, 2026-10-17)
|
||||||
|
|
||||||
|
No agent release; no binary changes. `scripts/test_gate_decoys.py` is now the `gate-decoys` row of `agent_gates.py`:
|
||||||
|
the full run (which CI runs on every push), not `--fast` — it takes ~20 s in the workspace (measured: 30 cases; ~5 s in a
|
||||||
|
CI-shaped shallow clone, also green). It was in no runner, the shape that let the catalog's R-1115 fail unseen for a
|
||||||
|
night. It plants only in scratch clones and asks a fake Gitea on 127.0.0.1; it does not call the runner (no
|
||||||
|
recursion). Declared in its own `COVERS`; `test_agent_gates.py` pins it (registered once, full run only, named by
|
||||||
|
`--fast`'s SKIPPED line). Red-proved: a reuse-refs FACT case flipped to expect rc 0 turned the full run red
|
||||||
|
(`CONVICTED: gate-decoys`); restored, green. CLAUDE.md's `--fast` sentence names it.
|
||||||
|
|
||||||
|
## unreleased — Go 1.27.2 and golang.org/x/crypto v0.58.0 (R-972, 2026-10-16)
|
||||||
|
|
||||||
|
**Not released, not deployed** — agent releases wait for the operator's vouch; the next vouched release carries this.
|
||||||
|
`go.mod`: `go 1.27.2` (was 1.25.0; the shipped 0.158.1 binary was built with go1.26.0), x/crypto v0.58.0 (was v0.52.0),
|
||||||
|
x/sys v0.49.0. `go build`, `go vet`, the full `go test ./...` and `agent_gates.py --fast` green on Go 1.27.2.
|
||||||
|
`release-agent.sh` builds with the toolchain `go.mod` names (GOTOOLCHAIN=auto), so no script change.
|
||||||
|
|
||||||
|
## scripts only — `release-agent.sh` refuses when the registry cannot say the version is new (R-1098, 2026-10-16)
|
||||||
|
|
||||||
|
No agent release; no binary changes. `scripts/release-agent.sh` already refused a version whose tag exists or whose
|
||||||
|
package answers 200. It let everything else through — a registry it could not reach (curl prints `000`) or a 500 read
|
||||||
|
as „new". Now only a clean `404` proves the version is new; anything else refuses with the code it got. New
|
||||||
|
`scripts/test_release_push_once.sh` (the real script, a fake `curl` and `go` on PATH; published / absent / unreachable /
|
||||||
|
server error / tag exists), red-proved: with the 404 rule removed, the unreachable and server-error cases fail. Same rule
|
||||||
|
as the controller `build.sh` and, since today, felhom.eu `scripts/build-hub.sh`.
|
||||||
|
|
||||||
|
## v0.158.1 — security review of v0.158.0: the mkfs wrapper trusts no sibling claim; the internal-place helper fails closed (2026-10-16)
|
||||||
|
|
||||||
|
**v0.158.0 is superseded — do not vouch it.** A security review of `912f7c1` found two defects; both fixed here, each test
|
||||||
|
seen failing against the 0.158.0 script:
|
||||||
|
- **`felhom-mkfs-guarded`**: 0.158.0 let a format through when a SIBLING part was mounted at /mnt/<name> AND under
|
||||||
|
/mnt/felhom-drives („Felhom's own drive"). That corroboration is a mount the agent can make itself, so a compromised agent
|
||||||
|
could unlock a host owner's disk — the exact case this root wrapper exists for. Removed: any mount of any node outside
|
||||||
|
/mnt/felhom-drives refuses again (now read from the kernel's mount table for every node; an unlistable disk refuses).
|
||||||
|
Consequence: the free part next to an in-use Felhom drive cannot be formatted — `CandidatePartition.Formattable` is false
|
||||||
|
for every part of a disk with a mounted node, and the controller does not offer it (a part with a filesystem can still be
|
||||||
|
USED). Pinned by configs/test_felhom_mkfs_guarded.py + partitions_test.go.
|
||||||
|
- **`felhom-internal-place`**: a failed `lvs` listing read as „nothing promised" (a too-large place) and a `blkid` error read
|
||||||
|
as „no signature" (the only gate before mkfs). Now: a failed listing, a listing without the pool, or any blkid exit other
|
||||||
|
than 0/2 REFUSES. Three new tests, all failing against 0.158.0.
|
||||||
|
- Released: tag `v0.158.1` (code `98f3cdd`), sha256 `20264df6…84b76a1`, bundle `c23c3d35…cb1c41b`. **Not vouched.**
|
||||||
|
|
||||||
|
## v0.158.0 — the internal data place; partitions as places; add_internal_place (2026-10-16)
|
||||||
|
|
||||||
|
Design: felhom.eu `architecture/16-storage-physical-identity.md` §5, §6, §7, §9 (operator rulings D3–D5; the D3 sizes
|
||||||
|
ruled 2026-10-16 after the space check, felhom.eu `audits/storage-identity-buildB-2026-10-15/SPACE-CHECK.md`).
|
||||||
|
|
||||||
|
- **The internal data place („Belső tárhely")** (`internal/localapi/internalplace.go`): when the record
|
||||||
|
`/var/lib/felhom-agent/internal-place.json` names a filesystem, the agent mounts it at `/mnt/belso` by fs-UUID like an
|
||||||
|
enrolled drive (startup + every 20 s tick) and records it `enrolled`; `GET /disks` reports it with kind `internal`.
|
||||||
|
It never takes `/mnt/belso` from a household drive, and eject/decommission/format of it are refused (device-mapper on
|
||||||
|
the system disk). Its identity is the system disk's, so build A never counts a copy to `sys_drive` as a second copy.
|
||||||
|
- **Root helper `configs/felhom-internal-place`** (new in the bundle): `plan|create <planned GiB>` — the ruled size rule
|
||||||
|
(reserve max(16 GiB, 10 %) ≤ 48 GiB, made only when ≥ 20 GiB, nothing promised beyond pool − reserve), the only
|
||||||
|
lvcreate/mkfs of a system-disk volume; formats only a `felhom-belso` with no signature. Run by the installer (1.33.0)
|
||||||
|
and by the new signed op.
|
||||||
|
- **Signed op `add_internal_place`** (old one-disk boxes, `16` §9): no parameters (the size is the rule's), destructive
|
||||||
|
class (operator key). Runs the helper with `create 0`, writes the record, mounts.
|
||||||
|
- **Partitions as places** (`internal/storage/partitions.go`): `GET /disks/candidates` lists each partition of a free
|
||||||
|
disk as empty / filesystem / in_use. A data-bearing part's wipe id binds disk id + part number + fs-UUID
|
||||||
|
(`byid:<…-partN>+fsuuid:<uuid>`); both must name the same device at format time.
|
||||||
|
- **`felhom-mkfs-guarded`** (REVERTED in v0.158.1): a SIBLING part that is Felhom's own drive no longer blocked formatting
|
||||||
|
the free part next to it. First tests of the wrapper: `configs/test_felhom_mkfs_guarded.py`.
|
||||||
|
- `configs/` changed (sudoers `FELHOM_INTERNALPLACE`, the helper, the wrapper, the bundle list) — **a config-bundle job
|
||||||
|
is needed** with the agent update. **Not vouched** (the operator's act).
|
||||||
|
- Released: tag `v0.158.0` (code `912f7c1`), sha256 `a3589d1d…2caba`, bundle `17066273…fb5c76`. **Superseded by v0.158.1 —
|
||||||
|
do not vouch** (its mkfs wrapper trusted a sibling claim the agent can fake; its helper failed open).
|
||||||
|
|
||||||
|
## v0.157.0 — every storage place knows its physical disk(s); POST /disks/same-disk (2026-10-15)
|
||||||
|
|
||||||
|
Design: felhom.eu `architecture/16-storage-physical-identity.md` §3, §4, §10 (operator rulings D1–D6, 2026-10-15). Spike:
|
||||||
|
felhom.eu `audits/storage-identity-2026-10-15/`.
|
||||||
|
|
||||||
|
- **Identity of a storage place** (`internal/storage/identity.go`): the physical disk(s) under it, from `/sys`, `/run/udev/data`
|
||||||
|
and `/proc/mounts` only — partition → disk, device-mapper / LVM thin → slaves, loop → backing file → its disk. Keys:
|
||||||
|
`eui:` (NVMe), `serial:<bus>:<model>:<serial>`, `wwn:` only without a serial and only with a vendor (OUI ≠ 000000).
|
||||||
|
`proven`, or `unknown` with one reason: `vm` (systemd-detect-virt), `network`, `usb_bridge` (a placeholder serial — the rule
|
||||||
|
is the design's, UNPROVEN on a real enclosure, R-1077), `weak_wwn`, `no_serial`. No block I/O.
|
||||||
|
- **Reported:** `GET /disks` rows and the hub report's `storage_targets` gain `kind`, `physical_disks`, `identity`,
|
||||||
|
`identity_reason` (additive; the golden and the hub's copy change together). A dir storage is judged only on its OWN mount — an
|
||||||
|
absent drive is "not computed", never the root disk.
|
||||||
|
- **`POST /disks/same-disk`** `{a, b}` (guest paths) → `{different, proven, reason, a, b}`. Maps `/mnt/felhom-drives/<name>` to
|
||||||
|
the host mount at exactly that path, and `/mnt/sys_drive`, `/var/lib/felhom`, `/var/lib/docker` to the guest's data volume's
|
||||||
|
storage. Anything else, or an absent drive → `unmapped`. A shared disk key is "same, proven" even when the disk is unknown.
|
||||||
|
Scoped like every `/disks` call.
|
||||||
|
- Tests: fixture `/sys` + udev trees for every spike case (one disk LVM thin, two disks, two partitions, a loop on a file, NVMe eui,
|
||||||
|
vendor-less WWN, a VM, a USB placeholder serial, a network share) and the same-disk call (equal, different, VM, NAS, absent
|
||||||
|
drive, scope); red-proofed (the LVM "diamond" bug was seen failing first; the VM, WWN, own-mount and sys_drive rules broken on purpose).
|
||||||
|
- `configs/` unchanged — no config-bundle job needed. **Not vouched** (the operator's act).
|
||||||
|
|
||||||
## Unreleased (2026-10-15, afternoon) — the shared rule file; no code change
|
## Unreleased (2026-10-15, afternoon) — the shared rule file; no code change
|
||||||
|
|
||||||
- `.claude/rules/unprompted-work.md` §3 item 6: **a controller release needs no golden** — the floor carries it after the
|
- `.claude/rules/unprompted-work.md` §3 item 6: **a controller release needs no golden** — the floor carries it after the
|
||||||
|
|||||||
@@ -56,7 +56,8 @@ This is in the core because breaching it is how this component stops being audit
|
|||||||
gate in its `GATES` table (that table is the list); the shared ones — `reuse_refs_check`,
|
gate in its `GATES` table (that table is the list); the shared ones — `reuse_refs_check`,
|
||||||
`instructions_gate`, `observations_gate` — are the copies in `felhom.eu/scripts/`, never copied into
|
`instructions_gate`, `observations_gate` — are the copies in `felhom.eu/scripts/`, never copied into
|
||||||
this repo (a copy recreates the drift they detect; an absent sibling clone FAILS). `--fast` selects the
|
this repo (a copy recreates the drift they detect; an absent sibling clone FAILS). `--fast` selects the
|
||||||
gates touching no network and no container runtime, and skips `published` (network), naming it. **A missing gate is a FAILURE, never a skip.**
|
gates touching no network and no container runtime, and skips `published` (network) and `gate-decoys` (the decoy
|
||||||
|
suite, ~20 s; R-1116), naming them — CI runs the full set. **A missing gate is a FAILURE, never a skip.**
|
||||||
|
|
||||||
**The pre-push hook** (`.githooks/pre-push`) runs it with `--fast` and refuses a failing push. It is
|
**The pre-push hook** (`.githooks/pre-push`) runs it with `--fast` and refuses a failing push. It is
|
||||||
**per-clone** — switch it on once with `git config core.hooksPath .githooks`, and a manual run WARNS
|
**per-clone** — switch it on once with `git config core.hooksPath .githooks`, and a manual run WARNS
|
||||||
|
|||||||
+29
@@ -1,5 +1,34 @@
|
|||||||
# CONTEXT — felhom-agent working state
|
# CONTEXT — felhom-agent working state
|
||||||
|
|
||||||
|
> **2026-10-18 (night) — UNRELEASED on main: R-1093, R-978, R-973** (release + vouch: the operator's). R-1093 an empty
|
||||||
|
> partition is blank (lsblk's PTTYPE on a partition is the parent's; set aside only on a clean blkid exit 0 with a
|
||||||
|
> matching `PART_ENTRY_SCHEME`; an unparsable lsblk is unprobed). R-978 the escrow package reaches the hub only at the
|
||||||
|
> claim (ceremony output v2, `PendingUpload`; `EscrowCeremonyConfig.Upload`). R-973 the `POST /backup` cadence cap.
|
||||||
|
> **Decision (decided by CC unattended — operator may reverse): the cap's numbers.** Q: how many whole-guest runs may
|
||||||
|
> the guest ask for? Options: (a) 2 per 24 h on the daily tier — keeps one local point older than a day, but refuses
|
||||||
|
> the NIGHT after a press (R-899's night can fall inside the same 24 h); (b) 3 per 24 h + 1 per 6 days on the weekly
|
||||||
|
> tier — the schedule never refused, a second press in a day refused, the local tier still replaceable in a day;
|
||||||
|
> (c) none — status quo. Chose (b): the off-site tier is the one `07` §8 row 10 relies on and it is fully protected
|
||||||
|
> for ~6 days; the local half's real fix is a time-bucketed prune (keep-daily), which is a retention change for the
|
||||||
|
> operator. No sudoers or bundle change in any of the three.
|
||||||
|
|
||||||
|
> **2026-10-17 — v0.159.1 RELEASED (0.159.0 superseded, do not vouch).** Go 1.27.2; R-1124 a decommissioned drive that is
|
||||||
|
> gone is forgotten (`ForgetRetiredDrives`, root verb `unit-forget`); R-1012 the read-only `felhom-thin-check` names an
|
||||||
|
> orphan thin device in a failed backup's text; the signed decommission prunes the guest-bind record. Bundle adds a path →
|
||||||
|
> step bundle `0.159.0-step1`. Proven on demo-hp; delivered by signed jobs on the operator's yes. Vouch: the operator's.
|
||||||
|
|
||||||
|
> **2026-10-16 — v0.158.1 RELEASED and VOUCHED (storage identity build B; 0.158.0 superseded, do not vouch).** The internal
|
||||||
|
> data place (root helper `felhom-internal-place`, the record in the state dir, mounted at /mnt/belso like a drive), the
|
||||||
|
> signed `add_internal_place`, partitions in `/disks/candidates` (`formattable` only while nothing on the disk is mounted),
|
||||||
|
> the partition wipe id `byid:<…-partN>+fsuuid:<uuid>`, the strict mkfs wrapper. Delivered through step bundle
|
||||||
|
> `0.158.1-step1` (a bundle that adds a path, R16). Next: R-1093 (an empty part's probe reads the parent's partition table).
|
||||||
|
|
||||||
|
> **2026-10-15 (night) — v0.157.0 RELEASED, NOT VOUCHED (storage identity build A, felhom.eu `16` §13).** Every storage
|
||||||
|
> place carries `kind`, `physical_disks`, `identity` (+ reason) in `GET /disks` and the hub report; `POST /disks/same-disk`
|
||||||
|
> answers the controller (v0.317.0, MinAgent 0.157.0). Computed on every read, never cached. Delivered by signed jobs to
|
||||||
|
> Tester 1, demo-hp, N100 only. Next: the operator vouches 0.157.0 → the global floor 0.317.0 / MinAgent 0.157.0. Build B
|
||||||
|
> (partition candidates, the internal place) waits for a task file (R-1079).
|
||||||
|
|
||||||
> **2026-10-08 (day) — UNRELEASED on main, ships with tomorrow's release (decision 178).** R-899: `POST /backup?trigger=manual`
|
> **2026-10-08 (day) — UNRELEASED on main, ships with tomorrow's release (decision 178).** R-899: `POST /backup?trigger=manual`
|
||||||
> runs no OS leg (`localapi/server.go`); after-boot kernel reports carry the saved block's ring (`kernelReportRing`). R-304:
|
> runs no OS leg (`localapi/server.go`); after-boot kernel reports carry the saved block's ring (`kernelReportRing`). R-304:
|
||||||
> `escrow.ErrRetainedUnchecked` → HTTP 424 `older_unchecked` when not every earlier package was tried (withheld, caps,
|
> `escrow.ErrRetainedUnchecked` → HTTP 424 `older_unchecked` when not every earlier package was tried (withheld, caps,
|
||||||
|
|||||||
@@ -1,38 +1,40 @@
|
|||||||
# REPORT — v0.156.1: after a power cut the box clears up (R-1006, R-1008) (2026-10-14)
|
# REPORT — three agent fixes, committed and NOT released (2026-10-18 evening)
|
||||||
|
|
||||||
Evidence: felhom.eu `documentation/audits/power-cut-fixes-2026-10-14/` — the spike (C1 the night's leftover, C2 a killed
|
Task: the evening brief of 2026-10-18, Part 4 (felhom.eu `REPORT-evening-2026-10-18.md`). Baseline: `27bbdaa` (v0.159.2
|
||||||
worker, C3 a real power cut, C4 a full disk), the 17 red-proofs.
|
released). Commits: `f7b7308` + `0963bae` (R-1093), `8561dc8` (R-978), `f17e197` (R-973). CHANGELOG under „Unreleased";
|
||||||
|
the release and its vouch are the operator's. Gates green per commit (`go build/vet/test ./...`, `agent_gates.py --fast`);
|
||||||
|
CI job 2062 success for `f17e197`.
|
||||||
|
|
||||||
## What the spike showed
|
## R-1093 — an empty partition is blank
|
||||||
|
|
||||||
- A power cut mid-backup leaves `vzdump-lxc-<vmid>-<date>.tar.dat` + `.tmp/` (root, in `/var/lib/vz/dump`); Proxmox never
|
lsblk reports the PARENT disk's PTTYPE for a partition. `InspectDevice` now ignores that PTTYPE only when `blkid -p` on the
|
||||||
lists it as a backup; nothing removes it; every later backup is skipped „not enough space".
|
partition exited 0, printed `PART_ENTRY_NUMBER`, and its `PART_ENTRY_SCHEME` equals lsblk's PTTYPE. Every error path
|
||||||
- The dead vzdump task stays in Proxmox's active list ~30–60 s after the boot — long enough for the agent's start-up pass
|
stays data-bearing: blkid exit 2 / I/O error / missing binary keep „device has a partition table"; an lsblk answer that
|
||||||
to call it „genuinely in flight" (R-1008). Short, so R-1008's harm is small.
|
exits 0 but does not parse or names no device is „could not be reliably inspected" (an older fail-open, closed here after
|
||||||
- A killed worker leaves `tar`/`zstd` writing into the `.tar.dat` for minutes, none named „vzdump" — the reason the root
|
a background security review flagged the probe). No new sudo command, no config bundle.
|
||||||
helper checks open files, not only process names.
|
Test `TestInspect_R1093_PartitionVerdicts` (14 rows); three red-proofs. **Live on demo-hp** (a 1 GB loop file in `/root`,
|
||||||
- On „No space left" vzdump removes its own partial (both paths): the preflight's sizing stays. But a full host root breaks
|
removed after; felhom.eu `audits/agent-fixes-2026-10-18/r1093-loop-check.txt`): the empty GPT partition → lsblk
|
||||||
LVM's own metadata writes (a half-removed snapshot; the janitor cannot clear it — new row).
|
`pttype "gpt"` (the bug's input), blkid rc 0 with only `PART_ENTRY_*` (the fix's condition holds); ext4 → `TYPE=ext4`; a
|
||||||
|
dos table inside the partition → `PTTYPE=dos`. Not walked: the whole `/api/storage/init` call (the agent refuses loop
|
||||||
|
devices; it needs a VM with a spare disk running a build of this commit).
|
||||||
|
|
||||||
## 0.156.0 is superseded — do not vouch it
|
## R-978 — the new escrow package reaches the hub only at the claim
|
||||||
|
|
||||||
A background security review of the pushed 0.156.0 flagged „fail-open state drift": its in-flight rule compared a task's
|
A ceremony started again no longer replaces the hub's package at create time. The package travels with the pending
|
||||||
wall-clock start with the kernel's `btime`, which moves when the clock is corrected after the boot (an NTP step after a
|
ceremony and is uploaded when the code is claimed, before the code is handed out; an upload failure is 502 and the code
|
||||||
power cut) — a live backup could then look „from before the boot" and its lock be cleared. 0.156.1 uses Proxmox's own
|
stays claimable; a ceremony abandoned by TTL, a new start or a restart leaves the first code opening the hub's package.
|
||||||
liveness rule instead (the UPID's pid + pstart against `/proc/<pid>/stat` start tick and state; no clock), and fails safe
|
Ceremony output is version 2 (`PendingUpload`); the sudo argv is unchanged. Tests `TestEscrowCeremony_R978_*` and three
|
||||||
when this agent cannot see root's processes (PID 1 not visible).
|
`cmd` tests; four red-proofs. **Residual (filed):** the typed-back check is the browser's, so a household that opens the
|
||||||
|
new code and closes the page without writing it down has still replaced the package.
|
||||||
|
|
||||||
## What changed
|
## R-973 — a cadence cap on `POST /backup`
|
||||||
|
|
||||||
See `CHANGELOG.md` v0.156.0: `vzdumpInFlight` behind the one `BackupRunning`; `internal/vzpartial` at start-up and before
|
Per guest and tier, successful runs only: the daily (local) tier at most **3 per rolling 24 h** (the night run, one
|
||||||
each backup; the root helper `felhom-vzdump-partial-clean` + one sudoers line + the bundle entry.
|
„Mentés most", the next night — `07` §6; since R-899 a press never costs the night); the longer tier at most one per
|
||||||
|
max(cadence − 24 h, cadence / 2) = **6 days** for the weekly off-site tier. A refusal is 429 with the next allowed time,
|
||||||
|
WARN-logged, kept in `backup-success-state.json` (survives a restart), and for 24 h the host report carries the CRITICAL
|
||||||
|
degraded status `localapi:backup-cadence-cap` (the hub's operator-only `agent_capability_degraded`). Tests `TestBackupCap_*`
|
||||||
|
and others; three red-proofs. **Limits (filed):** keep-last 3 lets three local points be replaced within one day; the
|
||||||
|
off-site namespace's keep-last 2 is a count — a time-bucketed prune on ep0 is the operator's.
|
||||||
|
|
||||||
## Gates
|
*Decided by CC unattended — operator may reverse:* the numbers 3 / 24 h and 6 days, and counting successes only.
|
||||||
|
|
||||||
`go build ./... && go vet` clean; `go test ./... -count=1` rc 0; `agent_gates.py` all OK; `visudo -cf` parsed OK; the
|
|
||||||
config bundle tests (55) and the helper's tests (21) pass. 17 mutations, 17 red.
|
|
||||||
|
|
||||||
## Next
|
|
||||||
|
|
||||||
Released: tag `v0.156.0`, binary sha256 `8db22788…`, bundle `a2160521…`, step bundle `0.156.0-step1` `8be2bb8f…`. Then the operator's vouch + floor, then the live proof on demo-hp (a cut mid-backup) and
|
|
||||||
delivery to Tester 1 and the N100.
|
|
||||||
|
|||||||
@@ -15,13 +15,14 @@
|
|||||||
| `SudoHostOps.run` | internal/storage/hostops.go | `run(ctx, name, args...) error` | allowlisted exec with stderr-wrapped error | Every arg pre-validated via validate.go before this is called |
|
| `SudoHostOps.run` | internal/storage/hostops.go | `run(ctx, name, args...) error` | allowlisted exec with stderr-wrapped error | Every arg pre-validated via validate.go before this is called |
|
||||||
| `Prober.Probe` | internal/capability/probe.go | `Probe(ctx) []Status` | live sudo-policy capability check (`sudo -n -l --`) | Needs a DIRECT runner (never the sudo-prefixing one — double-sudo); never executes probed cmds. v0.86.0: config-gated caps (`Capability.GatedBy` + `Prober.GateActive`) report `inactive`/"disabled by configuration" ONLY when healthy — broken plumbing stays degraded; the pbsdr-* gate answers from `pbsdr.Manager.DRConfigured` (marker-backed across restarts) |
|
| `Prober.Probe` | internal/capability/probe.go | `Probe(ctx) []Status` | live sudo-policy capability check (`sudo -n -l --`) | Needs a DIRECT runner (never the sudo-prefixing one — double-sudo); never executes probed cmds. v0.86.0: config-gated caps (`Capability.GatedBy` + `Prober.GateActive`) report `inactive`/"disabled by configuration" ONLY when healthy — broken plumbing stays degraded; the pbsdr-* gate answers from `pbsdr.Manager.DRConfigured` (marker-backed across restarts) |
|
||||||
| ~~`stageTemp`~~ (REMOVED v0.146.0, R-861) | — | — | — | Nothing the agent writes is `install`ed where root reads it any more: use `felhom-priv-apply` (below) or ship a fixed file in the bundle |
|
| ~~`stageTemp`~~ (REMOVED v0.146.0, R-861) | — | — | — | Nothing the agent writes is `install`ed where root reads it any more: use `felhom-priv-apply` (below) or ship a fixed file in the bundle |
|
||||||
| `felhom-priv-apply` (v0.146.0, R-861) | configs/felhom-priv-apply | `felhom-priv-apply unit <name> \| dnsmasq <tmp> <name> \| wg \| sshd-config \| sshd-key \| controller-image <vmid>` (the last reads the ref on stdin, R-861 (a) A1) | ANY agent-rendered file a root program reads (systemd unit, dnsmasq drop-in, wg-quick conf, OOB sshd) — fixed source + destination, CONTENT checked against the agent's own renderers | A new renderer needs a verb + a contract test (`internal/privapplytest.Check`) feeding its REAL output; never a new `install` sudoers line |
|
| `felhom-priv-apply` (v0.146.0, R-861) | configs/felhom-priv-apply | `felhom-priv-apply unit <name> \| unit-forget <name> (v0.159.0, R-1124) \| dnsmasq <tmp> <name> \| wg \| sshd-config \| sshd-key \| controller-image <vmid>` (the last reads the ref on stdin, R-861 (a) A1) | ANY agent-rendered file a root program reads (systemd unit, dnsmasq drop-in, wg-quick conf, OOB sshd) — fixed source + destination, CONTENT checked against the agent's own renderers | A new renderer needs a verb + a contract test (`internal/privapplytest.Check`) feeding its REAL output; never a new `install` sudoers line |
|
||||||
| `privapplytest.Check` | internal/privapplytest/check.go | `Check(t, verb, name, content) string` | the Go↔root-checker contract: a renderer's real output must read `OK` | Skips without python3; one call per rendered shape + one refused control |
|
| `privapplytest.Check` | internal/privapplytest/check.go | `Check(t, verb, name, content) string` | the Go↔root-checker contract: a renderer's real output must read `OK` | Skips without python3; one call per rendered shape + one refused control |
|
||||||
| `BUNDLE_FILES` + `Bundle` (mode `bundle`, `--install-bundle`; mode `agent_update` v0.146.0, R-861) | configs/felhom-os-apply | the ONE table of root-owned paths + the installer of them | ANY new root-owned file the installer writes (sudoers line, wrapper, unit) — add it to the table, never a new installer fetch (R-840) | The builder (`scripts/build-config-bundle.py`) and the installer read the same table; `test_every_root_file_the_installer_writes_is_in_the_bundle` fails on a path the bundle lacks. Trust files (`/etc/felhom/os-trust.json`, `operator-signers`) are NEVER bundle paths (R17) |
|
| `BUNDLE_FILES` + `Bundle` (mode `bundle`, `--install-bundle`; mode `agent_update` v0.146.0, R-861) | configs/felhom-os-apply | the ONE table of root-owned paths + the installer of them | ANY new root-owned file the installer writes (sudoers line, wrapper, unit) — add it to the table, never a new installer fetch (R-840) | The builder (`scripts/build-config-bundle.py`) and the installer read the same table; `test_every_root_file_the_installer_writes_is_in_the_bundle` fails on a path the bundle lacks. Trust files (`/etc/felhom/os-trust.json`, `operator-signers`) are NEVER bundle paths (R17) |
|
||||||
| `osupdate.ConfigUpdateExecutor` | internal/osupdate/bundle.go | signed op `agent_config_update` {agent_version, bundle_sha256} | delivering the bundle to an installed box | a courier only: the root wrapper re-verifies signature, host, nonce and sha itself |
|
| `osupdate.ConfigUpdateExecutor` | internal/osupdate/bundle.go | signed op `agent_config_update` {agent_version, bundle_sha256} | delivering the bundle to an installed box | a courier only: the root wrapper re-verifies signature, host, nonce and sha itself |
|
||||||
| `osupdate.Leg.SendUnsent` / `lockPass` (v0.144.0, R-868) | internal/osupdate/unsent.go | `(ctx) int` | an OS-pass report the agent never sent (killed mid-pass): the wrapper keeps `report-<run>-<layer>-apply.json` beside the plan; the agent deletes it once the hub has it | any new caller that runs an apply pass must hold `lockPass` (flock, across processes) — the sender must never take a running pass's copy |
|
| `osupdate.Leg.SendUnsent` / `lockPass` (v0.144.0, R-868) | internal/osupdate/unsent.go | `(ctx) int` | an OS-pass report the agent never sent (killed mid-pass): the wrapper keeps `report-<run>-<layer>-apply.json` beside the plan; the agent deletes it once the hub has it | any new caller that runs an apply pass must hold `lockPass` (flock, across processes) — the sender must never take a running pass's copy |
|
||||||
| `osupdate.LoadSavedBlock` (v0.144.0, R-866) | internal/osupdate/leg.go | `(planDir) (block, savedAt, ok)` | the hub's newest os_update block as the daemon last received it (`os-update-block.json`) | the debug pass uses it ONLY when the hub cannot be reached, and says so in its header; no saved block → no pass |
|
| `osupdate.LoadSavedBlock` (v0.144.0, R-866) | internal/osupdate/leg.go | `(planDir) (block, savedAt, ok)` | the hub's newest os_update block as the daemon last received it (`os-update-block.json`) | the debug pass uses it ONLY when the hub cannot be reached, and says so in its header; no saved block → no pass |
|
||||||
| `dpkg_state()` / `DPKG_STATE_SCRIPT` (v0.145.0, R-876) | configs/felhom-os-apply | `audit, journal = self.dpkg_state()` | dpkg's state in ONE call: `--audit` AND the update journal | never gate a repair on `--audit` alone — a crash leaves only the journal (measured); keep it one call (R-845) |
|
| `dpkg_state()` / `DPKG_STATE_SCRIPT` (v0.145.0, R-876) | configs/felhom-os-apply | `audit, journal = self.dpkg_state()` | dpkg's state in ONE call: `--audit` AND the update journal | never gate a repair on `--audit` alone — a crash leaves only the journal (measured); keep it one call (R-845) |
|
||||||
|
| `thincheck.Checker.Check` + `Describe` (v0.159.0, R-1012) | internal/thincheck/thincheck.go | `Check(ctx, vmid) []Finding` | name an ORPHAN thin device (pool holds it, LVM does not) in a whole-guest backup failure — `BackupRunner.SetThinCheck` | READ-ONLY by design: the root helper `felhom-thin-check <vg> <pool>` sends only reserve/release_metadata_snap; an orphan needs two reads; fails open (never stops a backup). The repair stays human |
|
||||||
| `guesthook.SnippetReady` / `Register` (v0.146.0, R-861) | internal/guesthook/install.go | `SnippetReady(path) error` | is the pre-start hook (a FIXED file from the bundle) in place — register only then | The agent never installs the hook (Proxmox runs it as root); a missing hookscript stops a guest start, so never `Register` without `SnippetReady` |
|
| `guesthook.SnippetReady` / `Register` (v0.146.0, R-861) | internal/guesthook/install.go | `SnippetReady(path) error` | is the pre-start hook (a FIXED file from the bundle) in place — register only then | The agent never installs the hook (Proxmox runs it as root); a missing hookscript stops a guest start, so never `Register` without `SnippetReady` |
|
||||||
|
|
||||||
### Disk / format safety (role gates, durable IDs, format guards)
|
### Disk / format safety (role gates, durable IDs, format guards)
|
||||||
@@ -42,6 +43,10 @@
|
|||||||
| `deriveDurableID` | internal/storage/durableid.go | `deriveDurableID(typ, s, backingDevice, uuid) string` | storage-target durable id (DR re-attach key) | Deterministic per type; `uuid:` for usb/local-dir; PBS id includes `#<fingerprint>` |
|
| `deriveDurableID` | internal/storage/durableid.go | `deriveDurableID(typ, s, backingDevice, uuid) string` | storage-target durable id (DR re-attach key) | Deterministic per type; `uuid:` for usb/local-dir; PBS id includes `#<fingerprint>` |
|
||||||
| `SystemDisks` / `isSystemBacked` / `RoleForStorage` / `RoleForRawDevice` | internal/storage/role.go | `RoleForRawDevice(device, sysDisks, sysKnown) DeviceRole` | protection-tier classification | Fails safe to `system` (most protected) on any ambiguity; role is AGENT-derived, never caller-supplied |
|
| `SystemDisks` / `isSystemBacked` / `RoleForStorage` / `RoleForRawDevice` | internal/storage/role.go | `RoleForRawDevice(device, sysDisks, sysKnown) DeviceRole` | protection-tier classification | Fails safe to `system` (most protected) on any ambiguity; role is AGENT-derived, never caller-supplied |
|
||||||
| `storage.IdentityResolver` / `Relation` / `MountFor` | internal/storage/identity.go | `HostPath(path) PlaceIdentity`, `Device(dev) PlaceIdentity`, `Relation(a, b) (different, proven, reason)` | physical-disk identity of a storage place (felhom.eu `16-storage-physical-identity.md` §3, v0.157.0) | Reads /sys + /run/udev/data + /proc/mounts only (no block I/O). Judge a drive ONLY on its own mount (`localapi.ownMountIdentity`) — never the containing fs, or an absent drive reads as the root disk. A shared disk key is "same, proven" even when unknown. Pinned by identity_test.go (fixture trees) + samedisk_test.go |
|
| `storage.IdentityResolver` / `Relation` / `MountFor` | internal/storage/identity.go | `HostPath(path) PlaceIdentity`, `Device(dev) PlaceIdentity`, `Relation(a, b) (different, proven, reason)` | physical-disk identity of a storage place (felhom.eu `16-storage-physical-identity.md` §3, v0.157.0) | Reads /sys + /run/udev/data + /proc/mounts only (no block I/O). Judge a drive ONLY on its own mount (`localapi.ownMountIdentity`) — never the containing fs, or an absent drive reads as the root disk. A shared disk key is "same, proven" even when unknown. Pinned by identity_test.go (fixture trees) + samedisk_test.go |
|
||||||
|
| `localapi.Server.EnsureInternalPlace` / `InternalPlaceRecord` | internal/localapi/internalplace.go | `EnsureInternalPlace(ctx)` (startup + 20 s tick); `ReadInternalPlace` / `WriteInternalPlace` | the one-disk box's internal data place (felhom.eu `16` §5, v0.158.0): mounts it at /mnt/belso by fs-UUID like an enrolled drive, records it `enrolled`; the controller binds + registers it | Never takes /mnt/belso from a household drive (a foreign mount, or a unit naming another UUID, waits). Never formats/ejects (dm on the system disk → role system). The ONLY maker is the root helper `configs/felhom-internal-place` (installer + signed `add_internal_place`). Pinned by internalplace_test.go |
|
||||||
|
| `configs/felhom-internal-place` | configs/ (bundle → /usr/local/sbin) | `plan\|create <planned GiB>` | the ruled size rule (reserve max(16, 10 %) ≤ 48 GiB; ≥ 20 GiB or nothing) + the only lvcreate/mkfs of a system-disk volume | Formats only a felhom-belso with NO signature; refuses a foreign one; never resizes/removes. Sudoers: `^(plan\|create) 0$` only. Pinned by configs/test_felhom_internal_place.py |
|
||||||
|
| `storage.candidatePartitions` / `CandidatePartition` | internal/storage/partitions.go | `CandidateDisk.Partitions` | partitions as places (`16` §6): empty / filesystem / in_use per part of a free disk; `Formattable` only while no node of the disk is mounted | A part with children or a member type is never listed; a raw /mnt mount counts as Felhom's only when corroborated (felhomOwnedMounts) — for LISTING only: the root mkfs wrapper trusts no such claim (v0.158.1). Pinned by partitions_test.go |
|
||||||
|
| partition wipe id `byid:<…-partN>+fsuuid:<uuid>` | internal/storage/durable_device.go | `DeviceDurableID` / `ResolveDurableDevice` | a data-bearing PART binds disk id + part number + filesystem (`16` §6) | Both halves must resolve to the SAME device or the wipe refuses. Whole disks keep the plain id. Pinned by partition_durable_test.go |
|
||||||
| `ValidateUUID/MountPath/BlockDevice/FSType/SMARTDevice/LVMName`, `UnitNameForMount` | internal/storage/validate.go | `Validate*(v) error` | EVERY arg that reaches a root shell-out | The security boundary; strict whitelists (no by-* symlinks, no dm, no traversal); `systemdEscapePath` computed in-process |
|
| `ValidateUUID/MountPath/BlockDevice/FSType/SMARTDevice/LVMName`, `UnitNameForMount` | internal/storage/validate.go | `Validate*(v) error` | EVERY arg that reaches a root shell-out | The security boundary; strict whitelists (no by-* symlinks, no dm, no traversal); `systemdEscapePath` computed in-process |
|
||||||
| `ValidateNetworkMountSpec` | internal/storage/netmount.go | `ValidateNetworkMountSpec(spec) error` | NAS mount input boundary | Same discipline as validate.go; SMB requires a creds ref; mountpoint confined under `NetworkMountRoot` |
|
| `ValidateNetworkMountSpec` | internal/storage/netmount.go | `ValidateNetworkMountSpec(spec) error` | NAS mount input boundary | Same discipline as validate.go; SMB requires a creds ref; mountpoint confined under `NetworkMountRoot` |
|
||||||
|
|
||||||
@@ -52,6 +57,7 @@
|
|||||||
| `SudoHostOps.EnsureMount` | internal/storage/hostops.go | `EnsureMount(ctx, MountSpec) error` | persistent by-UUID systemd .mount | Validate→render→stage→`install`→`enable --now`; idempotent |
|
| `SudoHostOps.EnsureMount` | internal/storage/hostops.go | `EnsureMount(ctx, MountSpec) error` | persistent by-UUID systemd .mount | Validate→render→stage→`install`→`enable --now`; idempotent |
|
||||||
| `SudoHostOps.Unmount` | internal/storage/hostops.go | `Unmount(ctx, where) error` | detach a mount unit | DESTRUCTIVE — caller MUST have gated it; does not self-authorize |
|
| `SudoHostOps.Unmount` | internal/storage/hostops.go | `Unmount(ctx, where) error` | detach a mount unit | DESTRUCTIVE — caller MUST have gated it; does not self-authorize |
|
||||||
| `SudoHostOps.ReassertEnrolledMounts` | internal/storage/hostops.go | `ReassertEnrolledMounts(ctx)` | reboot remount (re-resolve by UUID) | Re-asserts unless mounted AND enabled (`shouldReassertMount`); skips absent UUIDs |
|
| `SudoHostOps.ReassertEnrolledMounts` | internal/storage/hostops.go | `ReassertEnrolledMounts(ctx)` | reboot remount (re-resolve by UUID) | Re-asserts unless mounted AND enabled (`shouldReassertMount`); skips absent UUIDs |
|
||||||
|
| `SudoHostOps.ForgetRetiredDrives` (v0.159.0, R-1124) | internal/storage/forget_retired.go | `ForgetRetiredDrives(ctx, IntentReader) int` | drop the host mount unit of a DECOMMISSIONED drive that is GONE (start-up + the 20 s tick) | Only decommissioned + absent + unmounted; a present decommissioned drive keeps its unit (H3 re-enroll). The file goes only through `felhom-priv-apply unit-forget`, which re-checks all three as root |
|
||||||
| `GuestBinder.AttachDrive` / `DetachDrive` | internal/localapi/intermediary.go | `AttachDrive(ctx, vmid, where) (guestPath, err)` | live drive hot-swap under `/mnt/felhom-drives` | Normalizes to EXACTLY ONE bind via `countHostMounts` (converges double-binds); force re-bind when guest can't see it |
|
| `GuestBinder.AttachDrive` / `DetachDrive` | internal/localapi/intermediary.go | `AttachDrive(ctx, vmid, where) (guestPath, err)` | live drive hot-swap under `/mnt/felhom-drives` | Normalizes to EXACTLY ONE bind via `countHostMounts` (converges double-binds); force re-bind when guest can't see it |
|
||||||
| `GuestBinder.EnsureSharedParent` | internal/localapi/intermediary.go | `EnsureSharedParent(ctx) error` | shared-parent bind + boot unit | make-private+make-shared ONLY on first bind — re-running orphans the guest's slave; F2-a: compares script AND unit for staleness |
|
| `GuestBinder.EnsureSharedParent` | internal/localapi/intermediary.go | `EnsureSharedParent(ctx) error` | shared-parent bind + boot unit | make-private+make-shared ONLY on first bind — re-running orphans the guest's slave; F2-a: compares script AND unit for staleness |
|
||||||
| `StablePathForRaw` / `DriveNameFromRaw` | internal/localapi/intermediary.go | `StablePathForRaw("/mnt/<n>") string` | raw host mount → stable guest path | Single source of truth both repos derive from |
|
| `StablePathForRaw` / `DriveNameFromRaw` | internal/localapi/intermediary.go | `StablePathForRaw("/mnt/<n>") string` | raw host mount → stable guest path | Single source of truth both repos derive from |
|
||||||
@@ -108,8 +114,8 @@
|
|||||||
| `Engine.RunSignedJob` | internal/reconcile/job.go | `RunSignedJob(ctx, intent, signed, exec) JobResult` | executing a gated destructive job | Idempotency by nonce; journaled |
|
| `Engine.RunSignedJob` | internal/reconcile/job.go | `RunSignedJob(ctx, intent, signed, exec) JobResult` | executing a gated destructive job | Idempotency by nonce; journaled |
|
||||||
| `escrow.Create` | internal/escrow/escrow.go | `Create(ctx, CreateOptions) (CreateResult, R, error)` | PBS-key escrow (zero-knowledge) | Recovery code returned SEPARATELY from the result (anti-log); self-verifies recoverability |
|
| `escrow.Create` | internal/escrow/escrow.go | `Create(ctx, CreateOptions) (CreateResult, R, error)` | PBS-key escrow (zero-knowledge) | Recovery code returned SEPARATELY from the result (anti-log); self-verifies recoverability |
|
||||||
| `escrow.GenerateRecoveryCode` / `joinSafe` / `RecoveryCodeSep` | internal/escrow/wordlist.go | `GenerateRecoveryCode() (string, error)` | minting the customer recovery code R | Draws from the EFF large list **filtered of every word containing `RecoveryCodeSep`** (4 entries: drop-down, felt-tip, t-shirt, yo-yo) so a code always segments back into exactly 10 words — a hyphenated word made codes ambiguous to transcribe AND flaked the test ~1/5 (v0.93.0). Generation-only: **already-issued codes stay valid**, R is verified as a whole passphrase and never re-split. Never count words by splitting the joined string — count what the generator drew |
|
| `escrow.GenerateRecoveryCode` / `joinSafe` / `RecoveryCodeSep` | internal/escrow/wordlist.go | `GenerateRecoveryCode() (string, error)` | minting the customer recovery code R | Draws from the EFF large list **filtered of every word containing `RecoveryCodeSep`** (4 entries: drop-down, felt-tip, t-shirt, yo-yo) so a code always segments back into exactly 10 words — a hyphenated word made codes ambiguous to transcribe AND flaked the test ~1/5 (v0.93.0). Generation-only: **already-issued codes stay valid**, R is verified as a whole passphrase and never re-split. Never count words by splitting the joined string — count what the generator drew |
|
||||||
| `escrow.CeremonyBinary` / `CeremonyArgs()` / `CeremonyOutput` | internal/escrow/ceremony.go | the ONE fixed sudo self-invocation argv + the `--output=json` wire object (v1) | controller-driven ceremony (v0.88.0) | SINGLE SOURCE shared by the localapi exec, the capability manifest entry, and (byte-identically) the FELHOM_ESCROW sudoers line — `TestEscrowCeremonyArgvPinned` + `TestManifestCoveredBySudoers` lock all three. Never flag-helpers, never `--`→`-` (spike §2.2) |
|
| `escrow.CeremonyBinary` / `CeremonyArgs()` / `CeremonyOutput` | internal/escrow/ceremony.go | the ONE fixed sudo self-invocation argv + the `--output=json` wire object (v2, R-978: json mode returns the package in `PendingUpload` instead of uploading) | controller-driven ceremony (v0.88.0) | SINGLE SOURCE shared by the localapi exec, the capability manifest entry, and (byte-identically) the FELHOM_ESCROW sudoers line — `TestEscrowCeremonyArgvPinned` + `TestManifestCoveredBySudoers` lock all three. Never flag-helpers, never `--`→`-` (spike §2.2) |
|
||||||
| localapi escrow ceremony job | internal/localapi/escrow_ceremony.go | `POST /escrow/ceremony` + status + ONE-SHOT claim + preflight | the wizard's agent half | R lives ONLY in `Server.escrowR` (NEVER the job struct — snapshots must be structurally R-free); zeroed on claim/supersede/10-min TTL (`unclaimed_void`); in-memory BY DESIGN (restart loses R safely; re-run supersedes); subprocess stdout is SECRET-BEARING → parsed then zeroed, never logged |
|
| localapi escrow ceremony job | internal/localapi/escrow_ceremony.go | `POST /escrow/ceremony` + status + ONE-SHOT claim + preflight | the wizard's agent half | R lives ONLY in `Server.escrowR` (NEVER the job struct — snapshots must be structurally R-free); zeroed on claim/supersede/10-min TTL (`unclaimed_void`); in-memory BY DESIGN (restart loses R safely; re-run supersedes); **R-978: the new package reaches the hub only at the claim, before R is handed out (`EscrowCeremonyConfig.Upload`); TTL and a new start wait while that upload is in flight — an abandoned ceremony leaves the previous package current**; subprocess stdout is SECRET-BEARING → parsed then zeroed, never logged |
|
||||||
| `poke.Listener` + `poke.Port` | internal/poke/poke.go | `NewListener(resolve, trigger, port, logger)`; `poke.Port = 51822` | agent-plane immediate-sync (Direction-2a, v0.89.0) | Binds a contentless UDP socket EXCLUSIVELY to the box's WG /32 (`wgtunnel.LoadAssignedAddr`), fires the hub-loop out-of-band trigger. **Port 51822 is a SHARED cross-repo contract** — the hub poke sender + the ep0 `felhom-poke` forced-command target the SAME number; change one → change all three. Contentless (payload ignored), leading-edge debounced (`DebounceWindow`), WG-confined (kernel EKEYREJECTED refuses non-peer /32s). Wired only when `wg_tunnel.enabled` |
|
| `poke.Listener` + `poke.Port` | internal/poke/poke.go | `NewListener(resolve, trigger, port, logger)`; `poke.Port = 51822` | agent-plane immediate-sync (Direction-2a, v0.89.0) | Binds a contentless UDP socket EXCLUSIVELY to the box's WG /32 (`wgtunnel.LoadAssignedAddr`), fires the hub-loop out-of-band trigger. **Port 51822 is a SHARED cross-repo contract** — the hub poke sender + the ep0 `felhom-poke` forced-command target the SAME number; change one → change all three. Contentless (payload ignored), leading-edge debounced (`DebounceWindow`), WG-confined (kernel EKEYREJECTED refuses non-peer /32s). Wired only when `wg_tunnel.enabled` |
|
||||||
| `wgtunnel.LoadAssignedAddr` | internal/wgtunnel/manager.go | `LoadAssignedAddr(stateDir) (netip.Addr, bool)` | the box's own WG /32 without a Manager | Reads `registered.json`; ok=false until registered; strips the /32 → bare addr (the poke bind target) |
|
| `wgtunnel.LoadAssignedAddr` | internal/wgtunnel/manager.go | `LoadAssignedAddr(stateDir) (netip.Addr, bool)` | the box's own WG /32 without a Manager | Reads `registered.json`; ok=false until registered; strips the /32 → bare addr (the poke bind target) |
|
||||||
| `fasttick.Loop` + `fasttick.SourceFunc` | internal/fasttick/fasttick.go | `New(out chan<- struct{}, interval, logger, sources...)`; `Source.Unconverged() (bool, reason)` | agent-plane immediacy SECONDARY (v0.90.0, R-28): pulse the SAME out-of-band trigger every 30 s while ANY source is unconverged, self-disarm on convergence | STATE-BASED (no timer, nothing to journal). Every source MUST be a CACHED read (no exec/network per tick) — `desiredProvider.Generation()`, `reconcile.Engine.LastResult()` (Planned−Pending>0), `pbsdrLoop.PBSDRStatus().State=="waiting_secret"` ONLY, `wgtunnel.Manager.TunnelConvergence()`. The LOUD pbsdr states + destructive `pending_signature` are DELIBERATELY EXCLUDED (a stuck-loud box must not hammer). Pulses the cap-1 channel non-blocking (coalesces with poke/watchdog); NEVER touch `MinPollSeconds`/`clampInterval`/the ticker |
|
| `fasttick.Loop` + `fasttick.SourceFunc` | internal/fasttick/fasttick.go | `New(out chan<- struct{}, interval, logger, sources...)`; `Source.Unconverged() (bool, reason)` | agent-plane immediacy SECONDARY (v0.90.0, R-28): pulse the SAME out-of-band trigger every 30 s while ANY source is unconverged, self-disarm on convergence | STATE-BASED (no timer, nothing to journal). Every source MUST be a CACHED read (no exec/network per tick) — `desiredProvider.Generation()`, `reconcile.Engine.LastResult()` (Planned−Pending>0), `pbsdrLoop.PBSDRStatus().State=="waiting_secret"` ONLY, `wgtunnel.Manager.TunnelConvergence()`. The LOUD pbsdr states + destructive `pending_signature` are DELIBERATELY EXCLUDED (a stuck-loud box must not hammer). Pulses the cap-1 channel non-blocking (coalesces with poke/watchdog); NEVER touch `MinPollSeconds`/`clampInterval`/the ticker |
|
||||||
@@ -168,6 +174,7 @@
|
|||||||
| `capability` store-grant probe (`storeGrantStatuses` / `storeGrantVerdict` / `Client.Permissions`) | cmd/felhom-agent/main.go, internal/proxmox/query.go | *"may the agent READ this backup tier?"*, one `capability.Status` per configured tier | R-185. **Never infer permission from an empty content listing** — `{"data":[]}` is what a FORBIDDEN tier and a NEWBORN tier both return, and that ambiguity hid an unreadable host tier on both demo boxes. Ask `/access/permissions` **as the agent's own token** (root always says yes). **The ungranted answer is not empty and not a 403** — it carries the privileges inherited from the box-wide `/` grant, so test for **`Datastore.AllocateSpace`** specifically; path-presence or `Datastore.Audit` reports a blinded storage healthy. Probed set comes from `BackupTiers()`, never a fixed list. Critical except the `local` fallback. Composes AROUND the sudo prober (the `poolReadStatus` precedent); `Status`'s wire shape is untouched so the hub alert is free. Unreachable PVE ⇒ degraded, never ok. |
|
| `capability` store-grant probe (`storeGrantStatuses` / `storeGrantVerdict` / `Client.Permissions`) | cmd/felhom-agent/main.go, internal/proxmox/query.go | *"may the agent READ this backup tier?"*, one `capability.Status` per configured tier | R-185. **Never infer permission from an empty content listing** — `{"data":[]}` is what a FORBIDDEN tier and a NEWBORN tier both return, and that ambiguity hid an unreadable host tier on both demo boxes. Ask `/access/permissions` **as the agent's own token** (root always says yes). **The ungranted answer is not empty and not a 403** — it carries the privileges inherited from the box-wide `/` grant, so test for **`Datastore.AllocateSpace`** specifically; path-presence or `Datastore.Audit` reports a blinded storage healthy. Probed set comes from `BackupTiers()`, never a fixed list. Critical except the `local` fallback. Composes AROUND the sudo prober (the `poolReadStatus` precedent); `Status`'s wire shape is untouched so the hub alert is free. Unreachable PVE ⇒ degraded, never ok. |
|
||||||
| `backup.RestoreTestState` | internal/backup/restoretest_state.go | `RecordSuccess(target,archive,tier,verified,t)` / `ProvenArchive(target)` / `ProvenRestoreTests(ctx)` / `LastSuccess(target)` / `OldestFirst(targets)` | Per-tier restore-test PROOF state, persisted (atomic tmp+rename) — **which archive** was proven, and when (R-86) | **Credit ONLY on success** — a permanently failing tier must keep sorting first, or it looks freshly proven and stops being retried. Ties break on target id: without it, two tiers proven in the same second rotate by Go's randomised map order. **This one NEEDS persistence unlike R-84** — R-84 had ground truth to consult (the archive is still on the storage); a restore-test destroys its scratch and leaves no artifact. **R-86: the ARCHIVE is the state, the time is metadata** — a time alone cannot answer "have we proven THIS archive", which is the due-check's whole question. A pre-R-86 file (bare RFC3339 per target) keeps its time and yields NO proven archive, so each tier is due once after the upgrade; reading a legacy time as proof of the current archive would invent a guarantee. **R-189: it is also the REPORTABLE half of the restore-test signal.** The in-memory `backup.Store` holds only this process's latest run, and under per-archive due-ness the agent will not re-test a proven archive — so a proof lost to a restart is not repeated for a whole archive generation (observed live: a passing 14.5 GB offsite restore reached no host-report). `ProvenRestoreTests` renders the stored proofs as `hub.RestoreTest` entries and the collector merges them; a record missing the archive or the tier is NOT emitted, because an unproven tier reading as proven is worse than the defect. **Only successes are stored, deliberately:** a success suppresses future work, a failure causes it. |
|
| `backup.RestoreTestState` | internal/backup/restoretest_state.go | `RecordSuccess(target,archive,tier,verified,t)` / `ProvenArchive(target)` / `ProvenRestoreTests(ctx)` / `LastSuccess(target)` / `OldestFirst(targets)` | Per-tier restore-test PROOF state, persisted (atomic tmp+rename) — **which archive** was proven, and when (R-86) | **Credit ONLY on success** — a permanently failing tier must keep sorting first, or it looks freshly proven and stops being retried. Ties break on target id: without it, two tiers proven in the same second rotate by Go's randomised map order. **This one NEEDS persistence unlike R-84** — R-84 had ground truth to consult (the archive is still on the storage); a restore-test destroys its scratch and leaves no artifact. **R-86: the ARCHIVE is the state, the time is metadata** — a time alone cannot answer "have we proven THIS archive", which is the due-check's whole question. A pre-R-86 file (bare RFC3339 per target) keeps its time and yields NO proven archive, so each tier is due once after the upgrade; reading a legacy time as proof of the current archive would invent a guarantee. **R-189: it is also the REPORTABLE half of the restore-test signal.** The in-memory `backup.Store` holds only this process's latest run, and under per-archive due-ness the agent will not re-test a proven archive — so a proof lost to a restart is not repeated for a whole archive generation (observed live: a passing 14.5 GB offsite restore reached no host-report). `ProvenRestoreTests` renders the stored proofs as `hub.RestoreTest` entries and the collector merges them; a record missing the archive or the tier is NOT emitted, because an unproven tier reading as proven is worse than the defect. **Only successes are stored, deliberately:** a success suppresses future work, a failure causes it. |
|
||||||
| `backup.BackupSuccessState` | internal/backup/backup_state.go | `RecordBackupSuccess(target, b)` / `LastKnownSuccess(target, vmid)` | Newest SUCCESSFUL backup per tier+guest, persisted (atomic tmp+rename) — the due-check's fallback when the tier's storage cannot be read after a restart (R-894) | **Read ONLY when the storage cannot be read** — a storage that answers is the ground truth (R-84), and an archive absent there must make the tier due even when this file remembers one. A saved copy older than the cadence still reads due. Only successes are written. |
|
| `backup.BackupSuccessState` | internal/backup/backup_state.go | `RecordBackupSuccess(target, b)` / `LastKnownSuccess(target, vmid)` | Newest SUCCESSFUL backup per tier+guest, persisted (atomic tmp+rename) — the due-check's fallback when the tier's storage cannot be read after a restart (R-894) | **Read ONLY when the storage cannot be read** — a storage that answers is the ground truth (R-84), and an archive absent there must make the tier due even when this file remembers one. A saved copy older than the cadence still reads due. Only successes are written. |
|
||||||
|
| localapi backup cadence cap (R-973) | internal/localapi/backup_cap.go | `backupCadenceCap(cadence)`, `Server.capRefuses`, `Server.BackupCapRefusal()` | POST /backup refuses (429) a run the `07` §6 schedule would never ask for: ≤24 h tier → 3 successes per rolling 24 h; longer tier → 1 per max(cadence−24 h, cadence/2) | Counts SUCCESSES only, from this process AND `BackupSuccessState.RecentBackupSuccesses` (so a restart does not reset it); refusals saved (`RecordBackupRefusal`) and raised for 24 h as the CRITICAL host-report status `localapi:backup-cadence-cap` (`backupCapStatus` in main) → the hub's existing `agent_capability_degraded`. Tests `TestBackupCap_*`. |
|
||||||
| `hub.ProvenRestoreTestReporter` + `Collector.SetProvenRestoreTests` | internal/hub/collect.go | the DURABLE restore-test source, merged with the in-memory one | R-189. Merge rule: **one entry per tier, newest by `TestedAt` wins** — a fresh failure beats a stored success (the failure is the news, and it lives nowhere else), a stored success beats a stale in-memory entry after a restart, and a tier never appears twice (the hub would read two tests). An unparseable timestamp counts as OLDER, so a malformed entry cannot displace a good one. **The wiring is pinned by an AST test** — the method this replaced (`RestoreTestState.Snapshot`) carried a doc comment naming a host-report gauge and had no caller for weeks. |
|
| `hub.ProvenRestoreTestReporter` + `Collector.SetProvenRestoreTests` | internal/hub/collect.go | the DURABLE restore-test source, merged with the in-memory one | R-189. Merge rule: **one entry per tier, newest by `TestedAt` wins** — a fresh failure beats a stored success (the failure is the news, and it lives nowhere else), a stored success beats a stale in-memory entry after a restart, and a tier never appears twice (the hub would read two tests). An unparseable timestamp counts as OLDER, so a malformed entry cannot displace a good one. **The wiring is pinned by an AST test** — the method this replaced (`RestoreTestState.Snapshot`) carried a doc comment naming a host-report gauge and had no caller for weeks. |
|
||||||
| `backup.SpecBuilder` / `backup.TierPicker` / `(*BackupRunner).PickSettledRestoreCandidateOn` | internal/backup/schedule.go, runner.go | `func(ctx,archive) RestoreTestSpec`; `func(ctx,target,notAfter) (archive,landed,error)` | The per-run restore-test spec + per-tier **settled** candidate lookup (R-85, widened by R-86) | The spec is built **PER RUN**, never frozen at construction — the pre-R-85 immediately-invoked value made the offsite tier unschedulable AND went stale on any config change. `SourceTier` comes from **the archive**, never the configured target (the v0.100.0 rule). A tier with no archive returns `("", zero, nil)` — **`""` is NOT an error**, or every fresh box looks broken for its first week. **R-86: `notAfter` is the settle cutoff** (zero = no cutoff, which is what keeps `PickRestoreCandidateOn` a one-line call into it), and the picker now skips entries failing `archivePlausiblyComplete` — under per-archive due-ness an incomplete phantom would be picked forever, fail forever, never earn proof, and make the tier due at EVERY evaluation. |
|
| `backup.SpecBuilder` / `backup.TierPicker` / `(*BackupRunner).PickSettledRestoreCandidateOn` | internal/backup/schedule.go, runner.go | `func(ctx,archive) RestoreTestSpec`; `func(ctx,target,notAfter) (archive,landed,error)` | The per-run restore-test spec + per-tier **settled** candidate lookup (R-85, widened by R-86) | The spec is built **PER RUN**, never frozen at construction — the pre-R-85 immediately-invoked value made the offsite tier unschedulable AND went stale on any config change. `SourceTier` comes from **the archive**, never the configured target (the v0.100.0 rule). A tier with no archive returns `("", zero, nil)` — **`""` is NOT an error**, or every fresh box looks broken for its first week. **R-86: `notAfter` is the settle cutoff** (zero = no cutoff, which is what keeps `PickRestoreCandidateOn` a one-line call into it), and the picker now skips entries failing `archivePlausiblyComplete` — under per-archive due-ness an incomplete phantom would be picked forever, fail forever, never earn proof, and make the tier due at EVERY evaluation. |
|
||||||
| `localapi.BackupTier` + `normalizeBackupTiers` / `config.BackupConfig.BackupTiers` | internal/localapi/backup_tiers.go, internal/config/config.go | `normalizeBackupTiers(tiers, legacy, cadence) []BackupTier`; `BackupTiers() ([]BackupTier, []string)` | THE R-82 multi-tier resolution — one runner per tier, primary first | **The untargeted local-API contract is FROZEN**: no `?target=` ⇒ primary tier ⇒ pre-R-82 response BYTES (Target is `omitempty` and stays empty). Never default a missing cadence — reject it and log the warning at ERROR. Never share one retention knob between tiers. Jobs are keyed by (vmid,target). |
|
| `localapi.BackupTier` + `normalizeBackupTiers` / `config.BackupConfig.BackupTiers` | internal/localapi/backup_tiers.go, internal/config/config.go | `normalizeBackupTiers(tiers, legacy, cadence) []BackupTier`; `BackupTiers() ([]BackupTier, []string)` | THE R-82 multi-tier resolution — one runner per tier, primary first | **The untargeted local-API contract is FROZEN**: no `?target=` ⇒ primary tier ⇒ pre-R-82 response BYTES (Target is `omitempty` and stays empty). Never default a missing cadence — reject it and log the warning at ERROR. Never share one retention knob between tiers. Jobs are keyed by (vmid,target). |
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go/ast"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
|
||||||
|
)
|
||||||
|
|
||||||
|
// R-973: a cadence-cap refusal becomes a CRITICAL degraded status — what the hub's existing
|
||||||
|
// agent_capability_degraded event (operator-only) keys on — and clears when the refusal ages out.
|
||||||
|
func TestBackupCapStatus_RefusalIsCriticalDegraded(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||||
|
s := backupCapStatus(func() (string, int, time.Time, bool) { return "felhom-pbs", 9201, at, true })
|
||||||
|
if !s.Critical || s.Status != capability.StatusDegraded || !strings.Contains(s.Reason, "9201") || !strings.Contains(s.Reason, "felhom-pbs") {
|
||||||
|
t.Fatalf("a refusal must raise a critical degraded status naming the guest and the tier: %+v", s)
|
||||||
|
}
|
||||||
|
if s := backupCapStatus(func() (string, int, time.Time, bool) { return "", 0, time.Time{}, false }); s.Status != capability.StatusOK || !s.Critical {
|
||||||
|
t.Fatalf("no refusal → ok (and still critical, so a later refusal alerts): %+v", s)
|
||||||
|
}
|
||||||
|
if s := backupCapStatus(nil); s.Status != capability.StatusOK {
|
||||||
|
t.Fatalf("no local API → ok: %+v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wiring (the built-but-never-wired class): runDaemon's capability probe appends backupCapStatus, and the
|
||||||
|
// late-bound source is set from the local API server.
|
||||||
|
//
|
||||||
|
// COMPANION RED-PROOF (observed 2026-10-18): delete `backupCapRefusal = localSrv.BackupCapRefusal` →
|
||||||
|
// "runDaemon never binds backupCapRefusal to the local API server". Restored.
|
||||||
|
func TestR973_CapRefusalIsWiredIntoTheReport(t *testing.T) {
|
||||||
|
_, f := parseMain(t)
|
||||||
|
var appended, bound bool
|
||||||
|
for _, d := range f.Decls {
|
||||||
|
fd, ok := d.(*ast.FuncDecl)
|
||||||
|
if !ok || fd.Name == nil || fd.Name.Name != "runDaemon" || fd.Body == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||||
|
switch x := n.(type) {
|
||||||
|
case *ast.CallExpr:
|
||||||
|
if id, ok := x.Fun.(*ast.Ident); ok && id.Name == "backupCapStatus" && len(x.Args) == 1 {
|
||||||
|
if a, ok := x.Args[0].(*ast.Ident); ok && a.Name == "backupCapRefusal" {
|
||||||
|
appended = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case *ast.AssignStmt:
|
||||||
|
if len(x.Lhs) == 1 && len(x.Rhs) == 1 {
|
||||||
|
if l, ok := x.Lhs[0].(*ast.Ident); ok && l.Name == "backupCapRefusal" {
|
||||||
|
if sel, ok := x.Rhs[0].(*ast.SelectorExpr); ok && sel.Sel.Name == "BackupCapRefusal" {
|
||||||
|
bound = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if !appended {
|
||||||
|
t.Fatal("runDaemon's capability probe does not call backupCapStatus(backupCapRefusal)")
|
||||||
|
}
|
||||||
|
if !bound {
|
||||||
|
t.Fatal("runDaemon never binds backupCapRefusal to the local API server")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"go/ast"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/config"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
||||||
|
)
|
||||||
|
|
||||||
|
// R-978, the root half: the controller-driven ceremony (json mode, the FELHOM_ESCROW argv with
|
||||||
|
// `--upload`) must hand its package back instead of sending it — the daemon sends it at the claim.
|
||||||
|
//
|
||||||
|
// COMPANION RED-PROOF (observed 2026-10-18): set `deferUpload: false` in escrowOptsFromCLI → this fails
|
||||||
|
// with "json mode must defer the upload (R-978): got deferUpload=false", and the delivery test's json leg
|
||||||
|
// then sends the package at create time ("put called 1 times"). Restored.
|
||||||
|
func TestEscrowOptsFromCLI_JSONModeDefersUpload(t *testing.T) {
|
||||||
|
// The FELHOM_ESCROW argv: --selftest=escrow-create --upload --output=json.
|
||||||
|
if o := escrowOptsFromCLI("", false, false, true, "", true); !o.deferUpload || !o.upload {
|
||||||
|
t.Fatalf("json mode must defer the upload (R-978): got deferUpload=%v upload=%v", o.deferUpload, o.upload)
|
||||||
|
}
|
||||||
|
// Text mode (an operator at a terminal) keeps the immediate upload; no --upload means no upload at all.
|
||||||
|
if o := escrowOptsFromCLI("", false, false, true, "", false); o.deferUpload {
|
||||||
|
t.Fatal("text mode must upload at once")
|
||||||
|
}
|
||||||
|
if o := escrowOptsFromCLI("", false, false, false, "", true); o.deferUpload {
|
||||||
|
t.Fatal("without --upload there is nothing to defer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeliverEscrowPackage_JSONModeSendsNothing(t *testing.T) {
|
||||||
|
cfg := config.Config{}
|
||||||
|
cfg.Hub.URL, cfg.Hub.HostID, cfg.Hub.APIKey = "https://hub.invalid", "host-1", "k"
|
||||||
|
res := escrow.CreateResult{Blob: []byte("ciphertext"), KeyFingerprint: "f2:87", Posture: escrow.PostureZeroKnowledge, IdentityBlob: []byte("id")}
|
||||||
|
|
||||||
|
for _, jsonMode := range []bool{true, false} {
|
||||||
|
calls := 0
|
||||||
|
var sent []byte
|
||||||
|
put := func(_ context.Context, _ config.Config, body []byte) error { calls++; sent = body; return nil }
|
||||||
|
o := escrowOptsFromCLI("", false, false, true, "", jsonMode)
|
||||||
|
pending, uploaded, err := deliverEscrowPackage(context.Background(), cfg, res, "abc", o.deferUpload, put)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if jsonMode {
|
||||||
|
if calls != 0 || uploaded {
|
||||||
|
t.Fatalf("json mode: put called %d times (uploaded=%v), want 0 — the package must wait for the claim", calls, uploaded)
|
||||||
|
}
|
||||||
|
var m map[string]any
|
||||||
|
if json.Unmarshal(pending, &m) != nil || m["blob_b64"] == "" || m["identity_blob_b64"] == "" || m["restic_pw_sha256"] != "abc" {
|
||||||
|
t.Fatalf("json mode: the pending package is not the hub upload body: %s", pending)
|
||||||
|
}
|
||||||
|
} else if calls != 1 || !uploaded || pending != nil || len(sent) == 0 {
|
||||||
|
t.Fatalf("text mode: put called %d times, uploaded=%v, pending=%q — want one immediate upload", calls, uploaded, pending)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A hub that is not configured fails in both modes (as the pre-R-978 upload did).
|
||||||
|
if _, _, err := deliverEscrowPackage(context.Background(), config.Config{}, res, "", true, nil); err == nil {
|
||||||
|
t.Fatal("an unconfigured hub must fail the ceremony even when the upload is deferred")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// R-978 wiring (the built-but-never-wired class): runDaemon's localapi.EscrowCeremonyConfig carries an
|
||||||
|
// Upload that calls putEscrowBody. Without it every claim of a new package answers 503 and no customer
|
||||||
|
// can finish a ceremony.
|
||||||
|
//
|
||||||
|
// COMPANION RED-PROOF (observed 2026-10-18): delete the `Upload:` line in runDaemon → this fails with
|
||||||
|
// "runDaemon's EscrowCeremonyConfig has no Upload calling putEscrowBody". Restored.
|
||||||
|
func TestR978_CeremonyUploadIsWiredIntoTheDaemon(t *testing.T) {
|
||||||
|
_, f := parseMain(t)
|
||||||
|
if !callsWithin(f, "main")["runDaemon"] {
|
||||||
|
t.Fatal("main → runDaemon is broken — the path this test asserts is not the live one")
|
||||||
|
}
|
||||||
|
wired := false
|
||||||
|
for _, d := range f.Decls {
|
||||||
|
fd, ok := d.(*ast.FuncDecl)
|
||||||
|
if !ok || fd.Name == nil || fd.Name.Name != "runDaemon" || fd.Body == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||||
|
cl, ok := n.(*ast.CompositeLit)
|
||||||
|
if !ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
sel, ok := cl.Type.(*ast.SelectorExpr)
|
||||||
|
if !ok || sel.Sel.Name != "EscrowCeremonyConfig" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, el := range cl.Elts {
|
||||||
|
if kv, ok := el.(*ast.KeyValueExpr); ok {
|
||||||
|
if k, ok := kv.Key.(*ast.Ident); ok && k.Name == "Upload" {
|
||||||
|
ast.Inspect(kv.Value, func(n ast.Node) bool {
|
||||||
|
if ce, ok := n.(*ast.CallExpr); ok {
|
||||||
|
if id, ok := ce.Fun.(*ast.Ident); ok && id.Name == "putEscrowBody" {
|
||||||
|
wired = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if !wired {
|
||||||
|
t.Fatal("runDaemon's EscrowCeremonyConfig has no Upload calling putEscrowBody")
|
||||||
|
}
|
||||||
|
}
|
||||||
+126
-23
@@ -59,6 +59,7 @@ import (
|
|||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/thincheck"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/vzpartial"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/vzpartial"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/wgtunnel"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/wgtunnel"
|
||||||
)
|
)
|
||||||
@@ -193,7 +194,7 @@ func main() {
|
|||||||
flag.StringVar(&pbsStorage, "storage", "", "for --selftest=escrow-create: the pbs storage whose key to escrow (default: escrow.pbs_storage_id)")
|
flag.StringVar(&pbsStorage, "storage", "", "for --selftest=escrow-create: the pbs storage whose key to escrow (default: escrow.pbs_storage_id)")
|
||||||
flag.BoolVar(&paperkey, "paperkey", false, "for --selftest=escrow-create: ALSO emit the raw-key paperkey (opt-in (a); single-factor, unrevocable)")
|
flag.BoolVar(&paperkey, "paperkey", false, "for --selftest=escrow-create: ALSO emit the raw-key paperkey (opt-in (a); single-factor, unrevocable)")
|
||||||
flag.BoolVar(&offline, "offline", false, "for --selftest=escrow-create: ALSO emit the R-wrapped offline copy to print (opt-in (b))")
|
flag.BoolVar(&offline, "offline", false, "for --selftest=escrow-create: ALSO emit the R-wrapped offline copy to print (opt-in (b))")
|
||||||
flag.BoolVar(&upload, "upload", false, "for --selftest=escrow-create: upload the opaque blob to the hub")
|
flag.BoolVar(&upload, "upload", false, "for --selftest=escrow-create: upload the opaque blob to the hub (with --output=json: hand the package back in pending_upload — the agent daemon uploads it when the code is claimed, R-978)")
|
||||||
flag.StringVar(&blobPath, "blob", "", "for --selftest=escrow-consume: path to the R-wrapped escrow blob file")
|
flag.StringVar(&blobPath, "blob", "", "for --selftest=escrow-consume: path to the R-wrapped escrow blob file")
|
||||||
flag.StringVar(&expectedFP, "fingerprint", "", "for --selftest=escrow-consume: the EXPECTED key fingerprint (the gate target)")
|
flag.StringVar(&expectedFP, "fingerprint", "", "for --selftest=escrow-consume: the EXPECTED key fingerprint (the gate target)")
|
||||||
flag.StringVar(&keyDest, "keydest", "", "for --selftest=escrow-consume: where to install the recovered key (0600)")
|
flag.StringVar(&keyDest, "keydest", "", "for --selftest=escrow-consume: where to install the recovered key (0600)")
|
||||||
@@ -403,6 +404,31 @@ func (r *gateRemounter) Remount(ctx context.Context, t storage.KnownTarget) {
|
|||||||
// poolReadStatus probes the felhom-pool membership read (GET /pools/felhom) the stale-lock reaper's
|
// poolReadStatus probes the felhom-pool membership read (GET /pools/felhom) the stale-lock reaper's
|
||||||
// ownership scoping depends on (v0.62.0, audit A1). Needs `Pool.Audit` at `/pool/felhom` — granted
|
// ownership scoping depends on (v0.62.0, audit A1). Needs `Pool.Audit` at `/pool/felhom` — granted
|
||||||
// by host-install v1.9.0; on an older ACL this reports degraded and the reaper fail-safes (skips).
|
// by host-install v1.9.0; on an older ACL this reports degraded and the reaper fail-safes (skips).
|
||||||
|
// backupCapStatusName is the host-report status the R-973 cadence cap raises.
|
||||||
|
const backupCapStatusName = "localapi:backup-cadence-cap"
|
||||||
|
|
||||||
|
// backupCapStatus turns the local-API backup cadence cap's newest refusal (R-973) into a CRITICAL
|
||||||
|
// capability status: degraded for 24 h after a refusal, so the hub raises its operator-only
|
||||||
|
// agent_capability_degraded event (no hub change). The Reason names the guest, the tier and the time, and
|
||||||
|
// says what to do. nil source (local API off) → ok.
|
||||||
|
func backupCapStatus(refusal func() (string, int, time.Time, bool)) capability.Status {
|
||||||
|
s := capability.Status{
|
||||||
|
Name: backupCapStatusName,
|
||||||
|
Feature: "whole-guest restore points: a guest asked for more whole-guest backups than its schedule allows, and the extra ones were refused (R-973)",
|
||||||
|
Critical: true,
|
||||||
|
Status: capability.StatusOK,
|
||||||
|
}
|
||||||
|
if refusal == nil {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
if target, vmid, at, ok := refusal(); ok {
|
||||||
|
s.Status = capability.StatusDegraded
|
||||||
|
s.Reason = fmt.Sprintf("refused a whole-guest backup of guest %d on target %q at %s — more than the schedule allows; the guest's controller may be misbehaving or compromised: check the guest",
|
||||||
|
vmid, target, at.UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
func poolReadStatus(ctx context.Context, px *proxmox.Client) capability.Status {
|
func poolReadStatus(ctx context.Context, px *proxmox.Client) capability.Status {
|
||||||
s := capability.Status{
|
s := capability.Status{
|
||||||
Name: "pve:pool-read",
|
Name: "pve:pool-read",
|
||||||
@@ -829,8 +855,13 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
run: (&proxmox.ExecRunner{Mode: proxmox.RunnerMode(cfg.Privileged.Mode)}).Run,
|
run: (&proxmox.ExecRunner{Mode: proxmox.RunnerMode(cfg.Privileged.Mode)}).Run,
|
||||||
log: logger,
|
log: logger,
|
||||||
}
|
}
|
||||||
|
// R-973: the local-API backup cadence cap's refusals ride the report as a CRITICAL status, so the hub's
|
||||||
|
// existing agent_capability_degraded event (operator-only) tells the operator. Late-bound like
|
||||||
|
// drConfigured: the local API server is built further down, before the loop starts.
|
||||||
|
var backupCapRefusal func() (string, int, time.Time, bool)
|
||||||
probeAll := func(ctx context.Context) []capability.Status {
|
probeAll := func(ctx context.Context) []capability.Status {
|
||||||
out := append(capProber.Probe(ctx), poolReadStatus(ctx, px))
|
out := append(capProber.Probe(ctx), poolReadStatus(ctx, px))
|
||||||
|
out = append(out, backupCapStatus(backupCapRefusal))
|
||||||
return append(out, storeGrantStatuses(ctx, px, cfg, grantRepairer)...)
|
return append(out, storeGrantStatuses(ctx, px, cfg, grantRepairer)...)
|
||||||
}
|
}
|
||||||
// (The startup self-check log runs AFTER the pbsdr manager is wired below, so its snapshot
|
// (The startup self-check log runs AFTER the pbsdr manager is wired below, so its snapshot
|
||||||
@@ -959,10 +990,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
// state. Gates the watchdog's self-heal re-mount to ENROLLED drives, and the local API records
|
// state. Gates the watchdog's self-heal re-mount to ENROLLED drives, and the local API records
|
||||||
// enroll/eject into it. Open failure → ungated legacy remount (logged) so the daemon still runs.
|
// enroll/eject into it. Open failure → ungated legacy remount (logged) so the daemon still runs.
|
||||||
var intentReader storage.IntentReader
|
var intentReader storage.IntentReader
|
||||||
intentStateDir := "/var/lib/felhom-agent"
|
intentStateDir := agentStateDir(cfg)
|
||||||
if cfg.Authz.NonceStorePath != "" {
|
|
||||||
intentStateDir = filepath.Dir(cfg.Authz.NonceStorePath) // share the agent's durable state dir
|
|
||||||
}
|
|
||||||
intentStore, ierr := storage.OpenIntentStore(filepath.Join(intentStateDir, "drive-intents.json"))
|
intentStore, ierr := storage.OpenIntentStore(filepath.Join(intentStateDir, "drive-intents.json"))
|
||||||
if ierr != nil {
|
if ierr != nil {
|
||||||
logger.Warn("storage: intent store unavailable — self-heal runs UNGATED (legacy remount-any)", "err", ierr)
|
logger.Warn("storage: intent store unavailable — self-heal runs UNGATED (legacy remount-any)", "err", ierr)
|
||||||
@@ -1094,6 +1122,9 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
decommIntent = intentStore
|
decommIntent = intentStore
|
||||||
}
|
}
|
||||||
decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)
|
decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)
|
||||||
|
if guestBindStore != nil { // typed-nil guard, as above
|
||||||
|
decommExec.SetGuestBinds(guestBindStore) // v0.159.1: prune the bind record like the self-serve decommission
|
||||||
|
}
|
||||||
|
|
||||||
// Agent self-update (TASK D1): the agent_update executor downloads the operator-signed binary,
|
// Agent self-update (TASK D1): the agent_update executor downloads the operator-signed binary,
|
||||||
// verifies it against the SIGNED sha256, and hands it to the root guarded wrapper (A/B flip +
|
// verifies it against the SIGNED sha256, and hands it to the root guarded wrapper (A/B flip +
|
||||||
@@ -1168,7 +1199,11 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
}
|
}
|
||||||
logger.Warn("osupdate: capability probe after the config bundle", "ok", ok, "total", total, "degraded", strings.Join(names, ","))
|
logger.Warn("osupdate: capability probe after the config bundle", "ok", ok, "total", total, "degraded", strings.Join(names, ","))
|
||||||
}}
|
}}
|
||||||
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec, dockerExec, pveExec, kernelExec, bundleExec}, cfg.Hub.HostID, logger)
|
// v0.158.0 (felhom.eu `16` §9): the internal data place on an old box — the root helper, then the mount (late-bound
|
||||||
|
// to the local API server built below).
|
||||||
|
internalPlaceExec := signedjobs.NewInternalPlaceExecutor(suRunner,
|
||||||
|
localapi.InternalPlaceFileIn(intentStateDir), logger)
|
||||||
|
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec, dockerExec, pveExec, kernelExec, bundleExec, internalPlaceExec}, cfg.Hub.HostID, logger)
|
||||||
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
||||||
|
|
||||||
// Controller-driven escrow ceremony (v0.88.0): static config facts + the LATE-BOUND DR gate —
|
// Controller-driven escrow ceremony (v0.88.0): static config facts + the LATE-BOUND DR gate —
|
||||||
@@ -1192,6 +1227,9 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
return cfg.Escrow.PBSStorageID
|
return cfg.Escrow.PBSStorageID
|
||||||
},
|
},
|
||||||
HubConfigured: cfg.Hub.URL != "" && cfg.Hub.HostID != "" && cfg.Hub.APIKey != "",
|
HubConfigured: cfg.Hub.URL != "" && cfg.Hub.HostID != "" && cfg.Hub.APIKey != "",
|
||||||
|
// R-978: the ceremony hands its package back; the claim uploads it with the same PUT the root
|
||||||
|
// subprocess used, under the same per-host key, before the code is handed out.
|
||||||
|
Upload: func(ctx context.Context, pkg []byte) error { return putEscrowBody(ctx, cfg, pkg) },
|
||||||
DRConfigured: func() bool {
|
DRConfigured: func() bool {
|
||||||
if drConfigured != nil {
|
if drConfigured != nil {
|
||||||
return drConfigured()
|
return drConfigured()
|
||||||
@@ -1201,6 +1239,8 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
}
|
}
|
||||||
localSrv := buildLocalAPIServer(cfg, px, backupStore, heavyOps, observer, driveKnown, hostOps, gate, collector, client, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens)
|
localSrv := buildLocalAPIServer(cfg, px, backupStore, heavyOps, observer, driveKnown, hostOps, gate, collector, client, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens)
|
||||||
if localSrv != nil {
|
if localSrv != nil {
|
||||||
|
backupCapRefusal = localSrv.BackupCapRefusal // R-973
|
||||||
|
internalPlaceExec.SetEnsure(localSrv.EnsureInternalPlace)
|
||||||
// R-998 (doc 14): POST /host/poweroff — guest shutdown, then host power-off; guards in hostpoweroff.go.
|
// R-998 (doc 14): POST /host/poweroff — guest shutdown, then host power-off; guards in hostpoweroff.go.
|
||||||
pwMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
pwMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
||||||
if pwMode == "" {
|
if pwMode == "" {
|
||||||
@@ -1419,7 +1459,13 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
// filesystem UUID and re-mounts (idempotent `enable --now`) so a letter reshuffle is a no-op.
|
// filesystem UUID and re-mounts (idempotent `enable --now`) so a letter reshuffle is a no-op.
|
||||||
// Type-asserted (the concrete op exposes it; the interface stays lean).
|
// Type-asserted (the concrete op exposes it; the interface stays lean).
|
||||||
mountReasserter := hostOps // concrete *storage.SudoHostOps — exposes ReassertEnrolledMounts
|
mountReasserter := hostOps // concrete *storage.SudoHostOps — exposes ReassertEnrolledMounts
|
||||||
|
// v0.158.0 (felhom.eu `16` §5): the internal data place is mounted like an enrolled drive — first, so the
|
||||||
|
// re-assert below sees its unit.
|
||||||
|
localSrv.EnsureInternalPlace(ctx)
|
||||||
if mountReasserter != nil {
|
if mountReasserter != nil {
|
||||||
|
// R-1124: a decommissioned drive that is gone loses its host mount unit first (start-up and every tick), so
|
||||||
|
// no later boot waits for it and the re-assert below no longer lists it.
|
||||||
|
mountReasserter.ForgetRetiredDrives(ctx, intentReader)
|
||||||
mountReasserter.ReassertEnrolledMounts(ctx)
|
mountReasserter.ReassertEnrolledMounts(ctx)
|
||||||
}
|
}
|
||||||
// F9: on startup (the host's bring-up/reconcile trigger), re-assert any enrolled guest data-drive
|
// F9: on startup (the host's bring-up/reconcile trigger), re-assert any enrolled guest data-drive
|
||||||
@@ -1456,7 +1502,9 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
case <-t.C:
|
case <-t.C:
|
||||||
// Re-assert mounts first (handles a USB that enumerated late after a host reboot, or a
|
// Re-assert mounts first (handles a USB that enumerated late after a host reboot, or a
|
||||||
// unit re-disabled at runtime), then re-assert the guest binds against the now-live mounts.
|
// unit re-disabled at runtime), then re-assert the guest binds against the now-live mounts.
|
||||||
|
localSrv.EnsureInternalPlace(ctx) // `16` §5: a record written since the last tick (the signed job)
|
||||||
if mountReasserter != nil {
|
if mountReasserter != nil {
|
||||||
|
mountReasserter.ForgetRetiredDrives(ctx, intentReader) // R-1124: decommissioned + gone → unit removed
|
||||||
mountReasserter.ReassertEnrolledMounts(ctx)
|
mountReasserter.ReassertEnrolledMounts(ctx)
|
||||||
}
|
}
|
||||||
localSrv.ReassertGuestBinds(ctx)
|
localSrv.ReassertGuestBinds(ctx)
|
||||||
@@ -2023,10 +2071,12 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
|
|||||||
// F2-b: recover a guest left with a stale vzdump lock by a reboot-during-backup. Reads + start
|
// F2-b: recover a guest left with a stale vzdump lock by a reboot-during-backup. Reads + start
|
||||||
// go through the API client; the `pct unlock` is the one fenced root-CLI op (no API equivalent).
|
// go through the API client; the `pct unlock` is the one fenced root-CLI op (no API equivalent).
|
||||||
// A1 (v0.62.0): the scan is restricted to felhom-pool members (ownership proven, not assumed).
|
// A1 (v0.62.0): the scan is restricted to felhom-pool members (ownership proven, not assumed).
|
||||||
StaleLock: localapi.NewStaleLockController(px, &proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, reconcile.DefaultPool, logger),
|
StaleLock: localapi.NewStaleLockController(px, &proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, reconcile.DefaultPool, logger),
|
||||||
Intent: intent, // slice 10 P3: record enroll/eject intent for self-heal
|
Intent: intent, // slice 10 P3: record enroll/eject intent for self-heal
|
||||||
GuestBinds: guestBinds, // F9: per-guest bind record for the startup re-assert
|
// v0.158.0 (felhom.eu `16` §5): the internal data place record, beside the intent store.
|
||||||
FormatJobs: formatJobs, // F20-BUG3: detached-format job record + restart recovery
|
InternalPlaceFile: localapi.InternalPlaceFileIn(agentStateDir(cfg)),
|
||||||
|
GuestBinds: guestBinds, // F9: per-guest bind record for the startup re-assert
|
||||||
|
FormatJobs: formatJobs, // F20-BUG3: detached-format job record + restart recovery
|
||||||
|
|
||||||
// Host metrics (slice 9): the shared collector serves GET /host/metrics — a fresh host +
|
// Host metrics (slice 9): the shared collector serves GET /host/metrics — a fresh host +
|
||||||
// per-storage view to the customer's monitoring page (reuses the slice-4 collector).
|
// per-storage view to the customer's monitoring page (reuses the slice-4 collector).
|
||||||
@@ -2053,8 +2103,12 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
|
|||||||
},
|
},
|
||||||
Logger: logger,
|
Logger: logger,
|
||||||
}
|
}
|
||||||
|
// R-1012: an orphan thin device (a full host root during a snapshot) is NAMED in the backup failure the controller
|
||||||
|
// mails to the operator, through the read-only root helper. Never removed — the repair stays a human step.
|
||||||
|
thinChecker := &thincheck.Checker{API: px, Runner: &proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, Logger: logger}
|
||||||
for _, r := range tierRunners {
|
for _, r := range tierRunners {
|
||||||
r.SetBeforeBackup(func(ctx context.Context, storage string, vmid int) { cleaner.Clean(ctx, storage, vmid) })
|
r.SetBeforeBackup(func(ctx context.Context, storage string, vmid int) { cleaner.Clean(ctx, storage, vmid) })
|
||||||
|
r.SetThinCheck(func(ctx context.Context, vmid int) string { return thincheck.Describe(thinChecker.Check(ctx, vmid)) })
|
||||||
}
|
}
|
||||||
srv.SetPartialSweep(func(ctx context.Context, vmid int) {
|
srv.SetPartialSweep(func(ctx context.Context, vmid int) {
|
||||||
for _, t := range tierTargets {
|
for _, t := range tierTargets {
|
||||||
@@ -2806,6 +2860,20 @@ type escrowCeremonyOpts struct {
|
|||||||
offline bool
|
offline bool
|
||||||
upload bool
|
upload bool
|
||||||
identityBundlePath string
|
identityBundlePath string
|
||||||
|
// deferUpload (R-978): with upload, build the hub upload body and hand it back in
|
||||||
|
// CeremonyOutput.PendingUpload instead of sending it. Set for json mode — the controller-driven
|
||||||
|
// ceremony, whose daemon uploads the package only when the customer claims the new code.
|
||||||
|
deferUpload bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// escrowOptsFromCLI maps the escrow-create flags onto the ceremony options. json mode defers the upload
|
||||||
|
// (R-978, pinned by TestEscrowOptsFromCLI_JSONModeDefersUpload); text mode — an operator at a terminal,
|
||||||
|
// who sees R at once — keeps the immediate upload.
|
||||||
|
func escrowOptsFromCLI(storage string, paperkey, offline, upload bool, identityBundlePath string, jsonMode bool) escrowCeremonyOpts {
|
||||||
|
return escrowCeremonyOpts{
|
||||||
|
storage: storage, paperkey: paperkey, offline: offline, upload: upload,
|
||||||
|
identityBundlePath: identityBundlePath, deferUpload: upload && jsonMode,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// escrowCeremonyOutcome is the shared core's result. R is the ONLY secret; Sum mirrors the
|
// escrowCeremonyOutcome is the shared core's result. R is the ONLY secret; Sum mirrors the
|
||||||
@@ -2956,16 +3024,36 @@ func escrowCeremony(ctx context.Context, cfg config.Config, logger *slog.Logger,
|
|||||||
ResticPwSealed: resticStaged,
|
ResticPwSealed: resticStaged,
|
||||||
}
|
}
|
||||||
if opts.upload {
|
if opts.upload {
|
||||||
if err := uploadEscrowBlob(ctx, cfg, res, resticPwSHA256); err != nil {
|
pending, uploaded, err := deliverEscrowPackage(ctx, cfg, res, resticPwSHA256, opts.deferUpload, putEscrowBody)
|
||||||
|
if err != nil {
|
||||||
// R is minted and the blob self-verified — only the hub leg failed. kind "upload" lets
|
// R is minted and the blob self-verified — only the hub leg failed. kind "upload" lets
|
||||||
// the text shell keep the pre-extraction order (R surfaced, THEN the failure).
|
// the text shell keep the pre-extraction order (R surfaced, THEN the failure).
|
||||||
return out, &escrowCeremonyErr{kind: "upload", err: err}
|
return out, &escrowCeremonyErr{kind: "upload", err: err}
|
||||||
}
|
}
|
||||||
out.Sum.Uploaded = true
|
out.Sum.PendingUpload, out.Sum.Uploaded = pending, uploaded
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// deliverEscrowPackage builds the hub upload body and either sends it now (text mode) or returns it for
|
||||||
|
// the caller to send later (deferUpload — json mode, R-978: the daemon sends it when the customer claims
|
||||||
|
// the code). A hub that is not configured fails in both modes, as before: a package that can never be
|
||||||
|
// delivered is not a ceremony. put is the sender seam (putEscrowBody in production).
|
||||||
|
func deliverEscrowPackage(ctx context.Context, cfg config.Config, res escrow.CreateResult, resticPwSHA256 string, deferUpload bool,
|
||||||
|
put func(ctx context.Context, cfg config.Config, body []byte) error) (pending []byte, uploaded bool, err error) {
|
||||||
|
if cfg.Hub.URL == "" || cfg.Hub.HostID == "" || cfg.Hub.APIKey == "" {
|
||||||
|
return nil, false, fmt.Errorf("hub not configured (url/host_id/api_key)")
|
||||||
|
}
|
||||||
|
body := escrowUploadBody(res, resticPwSHA256)
|
||||||
|
if deferUpload {
|
||||||
|
return body, false, nil
|
||||||
|
}
|
||||||
|
if err := put(ctx, cfg, body); err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return nil, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
// printEscrowTextBanner prints the text-mode header line (shared by the success and the
|
// printEscrowTextBanner prints the text-mode header line (shared by the success and the
|
||||||
// post-banner failure paths so the pre-extraction output stays byte-identical — including the
|
// post-banner failure paths so the pre-extraction output stays byte-identical — including the
|
||||||
// posture: the historical banner always printed the DEFAULT posture, not the result's).
|
// posture: the historical banner always printed the DEFAULT posture, not the result's).
|
||||||
@@ -3029,10 +3117,7 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo
|
|||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
|
||||||
out, cerr := escrowCeremony(ctx, cfg, logger, escrowCeremonyOpts{
|
out, cerr := escrowCeremony(ctx, cfg, logger, escrowOptsFromCLI(storage, paperkey, offline, upload, identityBundlePath, jsonMode))
|
||||||
storage: storage, paperkey: paperkey, offline: offline, upload: upload,
|
|
||||||
identityBundlePath: identityBundlePath,
|
|
||||||
})
|
|
||||||
if cerr != nil {
|
if cerr != nil {
|
||||||
switch cerr.kind {
|
switch cerr.kind {
|
||||||
case "usage":
|
case "usage":
|
||||||
@@ -3070,6 +3155,9 @@ func runSelftestEscrowCreate(ctx context.Context, cfg config.Config, logger *slo
|
|||||||
if out.Sum.Uploaded {
|
if out.Sum.Uploaded {
|
||||||
fmt.Fprintln(os.Stderr, " uploaded the opaque blob(s) to the hub (host record); the hub cannot open them")
|
fmt.Fprintln(os.Stderr, " uploaded the opaque blob(s) to the hub (host record); the hub cannot open them")
|
||||||
}
|
}
|
||||||
|
if len(out.Sum.PendingUpload) > 0 {
|
||||||
|
fmt.Fprintln(os.Stderr, " package prepared, NOT uploaded: the agent uploads it when the new code is claimed (R-978)")
|
||||||
|
}
|
||||||
wire := out.Sum
|
wire := out.Sum
|
||||||
wire.RecoveryCode = out.R
|
wire.RecoveryCode = out.R
|
||||||
if err := json.NewEncoder(os.Stdout).Encode(wire); err != nil {
|
if err := json.NewEncoder(os.Stdout).Encode(wire); err != nil {
|
||||||
@@ -3223,13 +3311,10 @@ type escrowUploadRequest struct {
|
|||||||
ResticPwSHA256 string `json:"restic_pw_sha256,omitempty"`
|
ResticPwSHA256 string `json:"restic_pw_sha256,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// uploadEscrowBlob PUTs the opaque blob (and, for 10D, the identity blob) to
|
// escrowUploadBody is the hub upload body for one ceremony's result: the opaque blob (and, for 10D, the
|
||||||
// the hub, authed with the per-host key. The hub stores ciphertext + non-secret fields; no usable
|
// identity blob) plus non-secret fields. Ciphertext only — useless without R; no usable secret leaves
|
||||||
// secret leaves the agent.
|
// the agent. putEscrowBody sends it, authed with the per-host key.
|
||||||
func uploadEscrowBlob(ctx context.Context, cfg config.Config, res escrow.CreateResult, resticPwSHA256 string) error {
|
func escrowUploadBody(res escrow.CreateResult, resticPwSHA256 string) []byte {
|
||||||
if cfg.Hub.URL == "" || cfg.Hub.HostID == "" || cfg.Hub.APIKey == "" {
|
|
||||||
return fmt.Errorf("hub not configured (url/host_id/api_key)")
|
|
||||||
}
|
|
||||||
upReq := escrowUploadRequest{
|
upReq := escrowUploadRequest{
|
||||||
BlobB64: base64.StdEncoding.EncodeToString(res.Blob),
|
BlobB64: base64.StdEncoding.EncodeToString(res.Blob),
|
||||||
KeyFingerprint: res.KeyFingerprint,
|
KeyFingerprint: res.KeyFingerprint,
|
||||||
@@ -3241,6 +3326,15 @@ func uploadEscrowBlob(ctx context.Context, cfg config.Config, res escrow.CreateR
|
|||||||
upReq.IdentityBlobB64 = base64.StdEncoding.EncodeToString(res.IdentityBlob)
|
upReq.IdentityBlobB64 = base64.StdEncoding.EncodeToString(res.IdentityBlob)
|
||||||
}
|
}
|
||||||
body, _ := json.Marshal(upReq)
|
body, _ := json.Marshal(upReq)
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
|
// putEscrowBody PUTs an escrow upload body to this host's hub record (the per-host key). Used by the
|
||||||
|
// text-mode ceremony at once, and by the daemon when the customer claims the code (R-978).
|
||||||
|
func putEscrowBody(ctx context.Context, cfg config.Config, body []byte) error {
|
||||||
|
if cfg.Hub.URL == "" || cfg.Hub.HostID == "" || cfg.Hub.APIKey == "" {
|
||||||
|
return fmt.Errorf("hub not configured (url/host_id/api_key)")
|
||||||
|
}
|
||||||
url := strings.TrimRight(cfg.Hub.URL, "/") + "/api/v1/hosts/" + cfg.Hub.HostID + "/escrow"
|
url := strings.TrimRight(cfg.Hub.URL, "/") + "/api/v1/hosts/" + cfg.Hub.HostID + "/escrow"
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(body))
|
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -3905,3 +3999,12 @@ func (f *factsReporter) SystemFacts(ctx context.Context) (int, json.RawMessage,
|
|||||||
f.facts, f.err = f.leg.Facts(ctx, f.vmid)
|
f.facts, f.err = f.leg.Facts(ctx, f.vmid)
|
||||||
return f.vmid, f.facts, f.err
|
return f.vmid, f.facts, f.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// agentStateDir is the agent's durable state dir: beside the nonce store when one is configured, else the default.
|
||||||
|
// The intent store, the guest-bind record and the internal place record (v0.158.0) all live here.
|
||||||
|
func agentStateDir(cfg config.Config) string {
|
||||||
|
if cfg.Authz.NonceStorePath != "" {
|
||||||
|
return filepath.Dir(cfg.Authz.NonceStorePath)
|
||||||
|
}
|
||||||
|
return "/var/lib/felhom-agent"
|
||||||
|
}
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ func main() {
|
|||||||
|
|
||||||
func run() error {
|
func run() error {
|
||||||
var (
|
var (
|
||||||
op = flag.String("op", "", "op class to sign, e.g. storage_wipe | guest_destroy | decommission | agent_update | os_docker_step | os_pve_step | os_kernel_step | agent_config_update")
|
op = flag.String("op", "", "op class to sign, e.g. storage_wipe | guest_destroy | decommission | agent_update | os_docker_step | os_pve_step | os_kernel_step | agent_config_update | add_internal_place")
|
||||||
host = flag.String("host", "", "target host_id (anti-retarget — the op runs ONLY on this host)")
|
host = flag.String("host", "", "target host_id (anti-retarget — the op runs ONLY on this host)")
|
||||||
guest = flag.String("guest", "", "target guest_id (\"\" = host-scoped op)")
|
guest = flag.String("guest", "", "target guest_id (\"\" = host-scoped op)")
|
||||||
keyID = flag.String("key-id", "", "key id of the signing key (must match a pinned agent signer)")
|
keyID = flag.String("key-id", "", "key id of the signing key (must match a pinned agent signer)")
|
||||||
|
|||||||
@@ -25,6 +25,7 @@
|
|||||||
|
|
||||||
Cmnd_Alias FELHOM_MOUNT = \
|
Cmnd_Alias FELHOM_MOUNT = \
|
||||||
/usr/local/sbin/felhom-priv-apply ^unit mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$, \
|
/usr/local/sbin/felhom-priv-apply ^unit mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$, \
|
||||||
|
/usr/local/sbin/felhom-priv-apply ^unit-forget mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
||||||
/usr/bin/systemctl daemon-reload, \
|
/usr/bin/systemctl daemon-reload, \
|
||||||
/usr/bin/systemctl ^enable --now -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
/usr/bin/systemctl ^enable --now -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
||||||
/usr/bin/systemctl ^disable -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
/usr/bin/systemctl ^disable -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
||||||
@@ -96,6 +97,19 @@ Cmnd_Alias FELHOM_POWEROFF = \
|
|||||||
Cmnd_Alias FELHOM_VZPARTIAL = \
|
Cmnd_Alias FELHOM_VZPARTIAL = \
|
||||||
/usr/local/sbin/felhom-vzdump-partial-clean ^[A-Za-z][A-Za-z0-9_-]{0\,63} [1-9][0-9]{2\,8} vzdump-(lxc|qemu)-[1-9][0-9]{2\,8}-[0-9]{4}_[0-9]{2}_[0-9]{2}-[0-9]{2}_[0-9]{2}_[0-9]{2}\.(tar\.dat|vma\.dat|tmp)$
|
/usr/local/sbin/felhom-vzdump-partial-clean ^[A-Za-z][A-Za-z0-9_-]{0\,63} [1-9][0-9]{2\,8} vzdump-(lxc|qemu)-[1-9][0-9]{2\,8}-[0-9]{4}_[0-9]{2}_[0-9]{2}-[0-9]{2}_[0-9]{2}_[0-9]{2}\.(tar\.dat|vma\.dat|tmp)$
|
||||||
|
|
||||||
|
# Internal data place („Belső tárhely", felhom.eu architecture/16 §5, §7; agent v0.158.0). ONE root helper
|
||||||
|
# (configs/felhom-internal-place, from the config bundle) creates <vg>/felhom-belso in the local-lvm thin pool by the
|
||||||
|
# ruled size rule and formats ONLY that volume, ONLY when it carries no signature. The agent calls it from the signed
|
||||||
|
# `add_internal_place` job (an old box), always with 0 GiB planned: its guest's volumes already exist. `plan` changes
|
||||||
|
# nothing. The installer runs the helper as root itself (a new box).
|
||||||
|
# R-1012 (agent v0.159.0): the READ-ONLY orphan-thin check before and after a whole-guest backup — lists the pool's thin
|
||||||
|
# device ids (through a metadata snapshot it reserves and releases) next to LVM's. It sends no other pool message; the
|
||||||
|
# repair stays a human step. Pinned by configs/test_felhom_thin_check.py.
|
||||||
|
Cmnd_Alias FELHOM_THINCHECK = \
|
||||||
|
/usr/local/sbin/felhom-thin-check ^[A-Za-z0-9_+][A-Za-z0-9_.+-]{0\,63} [A-Za-z0-9_+][A-Za-z0-9_.+-]{0\,63}$
|
||||||
|
Cmnd_Alias FELHOM_INTERNALPLACE = \
|
||||||
|
/usr/local/sbin/felhom-internal-place ^(plan|create) 0$
|
||||||
|
|
||||||
# Intermediary mount model (the drive hot-swap re-architecture). The agent keeps a SHARED host parent
|
# Intermediary mount model (the drive hot-swap re-architecture). The agent keeps a SHARED host parent
|
||||||
# /mnt/felhom-drives (self-bind + make-shared) and binds/unbinds each drive's felhom-data namespace UNDERNEATH it so the
|
# /mnt/felhom-drives (self-bind + make-shared) and binds/unbinds each drive's felhom-data namespace UNDERNEATH it so the
|
||||||
# change propagates into the running guest live. The boot-persistence script + unit are FIXED files from the config
|
# change propagates into the running guest live. The boot-persistence script + unit are FIXED files from the config
|
||||||
@@ -328,4 +342,4 @@ Cmnd_Alias FELHOM_GUESTNET = \
|
|||||||
/usr/sbin/pct ^exec [0-9]+ -- pgrep -x dhclient$, \
|
/usr/sbin/pct ^exec [0-9]+ -- pgrep -x dhclient$, \
|
||||||
/usr/sbin/pct ^exec [0-9]+ -- dhclient -pf /run/dhclient\.eth0\.pid -lf /var/lib/dhcp/dhclient\.eth0\.leases eth0$
|
/usr/sbin/pct ^exec [0-9]+ -- dhclient -pf /run/dhclient\.eth0\.pid -lf /var/lib/dhcp/dhclient\.eth0\.leases eth0$
|
||||||
|
|
||||||
felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_FSTRIM, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_BACKUPTARGET, FELHOM_SELFHEAL, FELHOM_ESCROW, FELHOM_GUESTNET, FELHOM_SCRATCH_TEARDOWN, FELHOM_OSAPPLY, FELHOM_POWEROFF, FELHOM_VZPARTIAL
|
felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_FSTRIM, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_BACKUPTARGET, FELHOM_SELFHEAL, FELHOM_ESCROW, FELHOM_GUESTNET, FELHOM_SCRATCH_TEARDOWN, FELHOM_OSAPPLY, FELHOM_POWEROFF, FELHOM_VZPARTIAL, FELHOM_INTERNALPLACE, FELHOM_THINCHECK
|
||||||
|
|||||||
Executable
+118
@@ -0,0 +1,118 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# felhom-internal-place — make the one-disk box's internal data place („Belső tárhely"), felhom.eu
|
||||||
|
# architecture/16-storage-physical-identity.md §5, §7 (sizes ruled by the operator 2026-10-16). Installed as
|
||||||
|
# /usr/local/sbin/felhom-internal-place (config bundle). Called by felhom-host-install.sh (a new box) and by the agent's
|
||||||
|
# signed `add_internal_place` job (an old box) — ONE implementation of the size rule and of the only mkfs it does.
|
||||||
|
#
|
||||||
|
# felhom-internal-place plan <planned GiB> → prints the numbers, changes nothing
|
||||||
|
# felhom-internal-place create <planned GiB> → makes the place (or finds it), prints the result
|
||||||
|
#
|
||||||
|
# <planned GiB> is thin space still to be promised to the guest (rootfs + data volume) that does not exist yet:
|
||||||
|
# the installer passes 32 + the data volume; an old box passes 0 (its guest's volumes are already counted).
|
||||||
|
#
|
||||||
|
# The rule (16 §7, ruled): reserve = max(16 GiB, 10 % of the pool), at most 48 GiB; internal = pool − reserve −
|
||||||
|
# every thin volume already promised in the pool − <planned>, made only when ≥ 20 GiB. So nothing in the pool is ever
|
||||||
|
# promised beyond pool − reserve: a pool at 100 % stops every volume on the box.
|
||||||
|
#
|
||||||
|
# It trusts nothing it is told. Every rule is checked here, as root, and any doubt REFUSES (exit 1):
|
||||||
|
# - the pool is the `lvmthin: local-lvm` section of /etc/pve/storage.cfg (its vgname/thinpool), nothing else;
|
||||||
|
# - the only volume it creates, formats or reads is <vg>/felhom-belso;
|
||||||
|
# - it formats ONLY a felhom-belso that carries no signature at all (blkid -p finds nothing) — the one it just made,
|
||||||
|
# or one a crash left between lvcreate and mkfs; any other content REFUSES; it never resizes or removes anything.
|
||||||
|
# Output, one line: „created uuid=<fs-uuid> gib=<n>", „formatted uuid=… gib=…" (a crash left it empty), „existing uuid=<fs-uuid> gib=<n>", „skipped <why>" (exit 0),
|
||||||
|
# or „plan pool=… reserve=… promised=… planned=… internal=…". Pinned by configs/test_felhom_internal_place.py.
|
||||||
|
set -u
|
||||||
|
STORAGE_CFG=${FELHOM_STORAGE_CFG:-/etc/pve/storage.cfg}
|
||||||
|
LV_NAME=felhom-belso
|
||||||
|
LABEL=felhom-belso
|
||||||
|
MIN_GIB=20
|
||||||
|
|
||||||
|
die() { echo "felhom-internal-place: REFUSED: $*" >&2; exit 1; }
|
||||||
|
|
||||||
|
pool_of_local_lvm() { # prints "<vg> <thinpool>" of the lvmthin storage local-lvm, or nothing
|
||||||
|
awk '
|
||||||
|
/^[a-z]+:[ \t]/ { split($0, h, /:[ \t]+/); type = h[1]; id = h[2]; sub(/[ \t]+$/, "", id); in_s = (type == "lvmthin" && id == "local-lvm"); next }
|
||||||
|
in_s && /^[ \t]+vgname[ \t]/ { vg = $2 }
|
||||||
|
in_s && /^[ \t]+thinpool[ \t]/ { tp = $2 }
|
||||||
|
END { if (vg != "" && tp != "") print vg, tp }
|
||||||
|
' "$STORAGE_CFG"
|
||||||
|
}
|
||||||
|
|
||||||
|
gib_floor() { echo $(( $1 / 1073741824 )); }
|
||||||
|
|
||||||
|
[[ $# -eq 2 ]] || die "usage: felhom-internal-place plan|create <planned GiB>"
|
||||||
|
MODE=$1; PLANNED=$2
|
||||||
|
[[ "$MODE" == plan || "$MODE" == create ]] || die "mode must be plan or create"
|
||||||
|
[[ "$PLANNED" =~ ^(0|[1-9][0-9]{0,5})$ ]] || die "planned GiB must be a whole number"
|
||||||
|
|
||||||
|
read -r VG POOL < <(pool_of_local_lvm) || true
|
||||||
|
[[ -n "${VG:-}" && -n "${POOL:-}" ]] || die "no lvmthin storage local-lvm in $STORAGE_CFG"
|
||||||
|
[[ "$VG" =~ ^[A-Za-z0-9_.+-]+$ && "$POOL" =~ ^[A-Za-z0-9_.+-]+$ ]] || die "unexpected vg/pool name"
|
||||||
|
|
||||||
|
pool_b=$(lvs --noheadings --nosuffix --units b -o lv_size "$VG/$POOL" 2>/dev/null | tr -d ' ')
|
||||||
|
[[ "$pool_b" =~ ^[0-9]+$ ]] || die "cannot read the size of $VG/$POOL"
|
||||||
|
pool_gib=$(gib_floor "$pool_b")
|
||||||
|
|
||||||
|
# Every thin volume promised in the pool (its virtual size) — except the place itself and vzdump's short-lived
|
||||||
|
# snapshots (snap_*_vzdump), which exist only during a whole-box copy and are what the reserve is for.
|
||||||
|
# FAIL CLOSED: a failed or empty listing must never read as „nothing promised" (that would size the place too big).
|
||||||
|
listing=$(lvs --noheadings --nosuffix --units b -o lv_name,pool_lv,lv_size "$VG" 2>/dev/null) || die "cannot list the volumes of $VG"
|
||||||
|
pool_seen=false; promised_b=0; have_lv=false
|
||||||
|
while read -r name pl size; do
|
||||||
|
[[ -n "${name:-}" ]] || continue
|
||||||
|
[[ "$name" == "$POOL" ]] && pool_seen=true
|
||||||
|
[[ "$pl" == "$POOL" ]] || continue
|
||||||
|
if [[ "$name" == "$LV_NAME" ]]; then have_lv=true; continue; fi
|
||||||
|
[[ "$name" =~ ^snap_.*_vzdump$ ]] && continue
|
||||||
|
[[ "$size" =~ ^[0-9]+$ ]] || die "cannot read the size of $VG/$name"
|
||||||
|
promised_b=$(( promised_b + size ))
|
||||||
|
done <<< "$listing"
|
||||||
|
$pool_seen || die "the listing of $VG does not contain the pool $POOL — refusing to guess what is promised"
|
||||||
|
promised_gib=$(( (promised_b + 1073741823) / 1073741824 )) # rounded UP: never promise a partial GiB twice
|
||||||
|
|
||||||
|
reserve=$(( pool_gib / 10 )); (( reserve < 16 )) && reserve=16; (( reserve > 48 )) && reserve=48
|
||||||
|
internal=$(( pool_gib - reserve - promised_gib - PLANNED ))
|
||||||
|
|
||||||
|
if [[ "$MODE" == plan ]]; then
|
||||||
|
echo "plan pool=$pool_gib reserve=$reserve promised=$promised_gib planned=$PLANNED internal=$internal exists=$have_lv"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
DEV=/dev/$VG/$LV_NAME
|
||||||
|
is_block() { [[ "$(stat -L -c %F "$1" 2>/dev/null)" == "block special file" ]]; }
|
||||||
|
fs_uuid() { blkid -p -s UUID -o value "$DEV" 2>/dev/null; }
|
||||||
|
fs_type() { blkid -p -s TYPE -o value "$DEV" 2>/dev/null; }
|
||||||
|
fs_label() { blkid -p -s LABEL -o value "$DEV" 2>/dev/null; }
|
||||||
|
# has_signature: FAIL CLOSED. blkid -p exits 0 when it found something and 2 when it found NOTHING; any other result (an
|
||||||
|
# error, an ambivalent probe, blkid missing) must never read as „empty" — that is the only gate in front of mkfs.
|
||||||
|
has_signature() {
|
||||||
|
local rc=0
|
||||||
|
blkid -p -o value "$DEV" >/dev/null 2>&1 || rc=$?
|
||||||
|
case $rc in
|
||||||
|
0) return 0 ;;
|
||||||
|
2) return 1 ;;
|
||||||
|
*) die "cannot probe $DEV for a signature (blkid exit $rc) — refusing" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
lv_gib() { gib_floor "$(lvs --noheadings --nosuffix --units b -o lv_size "$VG/$LV_NAME" 2>/dev/null | tr -d ' ')"; }
|
||||||
|
|
||||||
|
if $have_lv; then
|
||||||
|
is_block "$DEV" || die "$DEV is not an active block device"
|
||||||
|
t=$(fs_type)
|
||||||
|
if [[ -n "$t" ]]; then
|
||||||
|
[[ "$t" == ext4 && "$(fs_label)" == "$LABEL" ]] || die "$DEV holds something that is not Felhom's ($t) — left alone"
|
||||||
|
echo "existing uuid=$(fs_uuid) gib=$(lv_gib)"; exit 0
|
||||||
|
fi
|
||||||
|
# A felhom-belso with NO signature: a crash between lvcreate and mkfs. Nothing can be on it — format it.
|
||||||
|
has_signature && die "$DEV carries a signature blkid cannot name — left alone"
|
||||||
|
else
|
||||||
|
(( internal >= MIN_GIB )) || { echo "skipped only $internal GiB free for it (pool $pool_gib, reserve $reserve, promised $promised_gib, planned $PLANNED; needs $MIN_GIB)"; exit 0; }
|
||||||
|
lvcreate -q -y -V "${internal}G" -T "$VG/$POOL" -n "$LV_NAME" >/dev/null || die "lvcreate failed"
|
||||||
|
is_block "$DEV" || die "$DEV did not appear after lvcreate"
|
||||||
|
has_signature && die "the new $DEV already carries a signature — left alone"
|
||||||
|
made=created
|
||||||
|
fi
|
||||||
|
mkfs.ext4 -q -L "$LABEL" "$DEV" >/dev/null || die "mkfs.ext4 failed on $DEV"
|
||||||
|
u=$(fs_uuid)
|
||||||
|
[[ "$u" =~ ^[0-9a-fA-F-]{36}$ ]] || die "no filesystem UUID after mkfs on $DEV"
|
||||||
|
echo "${made:-formatted} uuid=$u gib=$(lv_gib)"
|
||||||
@@ -58,14 +58,26 @@ fi
|
|||||||
|
|
||||||
# 3) mounted OUTSIDE Felhom's own drive area = a live foreign filesystem → catastrophic. Mounts under
|
# 3) mounted OUTSIDE Felhom's own drive area = a live foreign filesystem → catastrophic. Mounts under
|
||||||
# /mnt/felhom-drives are our own drives (the agent detaches before a re-init) → allowed.
|
# /mnt/felhom-drives are our own drives (the agent detaches before a re-init) → allowed.
|
||||||
while read -r mp; do
|
# Agent v0.158.0: read from the kernel's mount table for EVERY node of the disk (lsblk's MOUNTPOINT shows one mount
|
||||||
[[ -z "$mp" ]] && continue
|
# per node; a bind elsewhere hid behind it), and a disk whose nodes cannot be listed REFUSES (it passed before).
|
||||||
case "$mp" in
|
# Deliberately NO exemption for a sibling that is Felhom's own enrolled drive (raw /mnt/<name> + /mnt/felhom-drives):
|
||||||
/mnt/felhom-drives|/mnt/felhom-drives/*) : ;;
|
# the only corroboration available here is a mount the agent itself can make, so trusting it would let a compromised
|
||||||
*) die "device (or a partition) is mounted at $mp ($dev)" ;;
|
# agent unlock a host owner's disk. Consequence (felhom.eu `16` §13): the free part next to an in-use Felhom drive
|
||||||
esac
|
# cannot be formatted; the dashboard does not offer it.
|
||||||
done < <(lsblk -nro MOUNTPOINT "$whole" 2>/dev/null || true)
|
PROC_MOUNTS=/proc/mounts
|
||||||
|
node_mounts() { awk -v d="$1" '$1 == d { print $2 }' "$PROC_MOUNTS"; }
|
||||||
|
disk_nodes="$(lsblk -nrpo NAME "$whole" 2>/dev/null || true)"
|
||||||
|
[[ -n "$disk_nodes" ]] || die "cannot list the nodes of $whole — refusing"
|
||||||
|
while read -r node; do
|
||||||
|
[[ -z "$node" ]] && continue
|
||||||
|
while read -r mp; do
|
||||||
|
[[ -z "$mp" ]] && continue
|
||||||
|
case "$mp" in
|
||||||
|
/mnt/felhom-drives|/mnt/felhom-drives/*) : ;;
|
||||||
|
*) die "device (or a partition) is mounted at $mp ($node)" ;;
|
||||||
|
esac
|
||||||
|
done < <(node_mounts "$node")
|
||||||
|
done <<< "$disk_nodes"
|
||||||
# 4) read-only device (audit D1): a device the kernel marks RO is never a formattable data disk.
|
# 4) read-only device (audit D1): a device the kernel marks RO is never a formattable data disk.
|
||||||
wbase="${whole#/dev/}"
|
wbase="${whole#/dev/}"
|
||||||
rof="/sys/block/$wbase/ro"
|
rof="/sys/block/$wbase/ro"
|
||||||
|
|||||||
@@ -191,6 +191,10 @@ BUNDLE_FILES = [
|
|||||||
("/usr/local/sbin/felhom-backup-target-apply", "felhom-backup-target-apply", 0o755, "bash", "replace"),
|
("/usr/local/sbin/felhom-backup-target-apply", "felhom-backup-target-apply", 0o755, "bash", "replace"),
|
||||||
# R-1006 (agent v0.156.0): the only delete of backup files — a dead half-written vzdump, re-checked as root.
|
# R-1006 (agent v0.156.0): the only delete of backup files — a dead half-written vzdump, re-checked as root.
|
||||||
("/usr/local/sbin/felhom-vzdump-partial-clean", "felhom-vzdump-partial-clean", 0o755, "bash", "replace"),
|
("/usr/local/sbin/felhom-vzdump-partial-clean", "felhom-vzdump-partial-clean", 0o755, "bash", "replace"),
|
||||||
|
# R-1012 (agent v0.159.0): the read-only orphan-thin check around a whole-guest backup (names an orphan, never removes).
|
||||||
|
("/usr/local/sbin/felhom-thin-check", "felhom-thin-check", 0o755, "bash", "replace"),
|
||||||
|
# felhom.eu architecture/16 §5 (agent v0.158.0): the internal data place — the only lvcreate/mkfs of a system-disk volume.
|
||||||
|
("/usr/local/sbin/felhom-internal-place", "felhom-internal-place", 0o755, "bash", "replace"),
|
||||||
("/usr/local/sbin/felhom-os-apply", "felhom-os-apply", 0o755, "python", "replace"),
|
("/usr/local/sbin/felhom-os-apply", "felhom-os-apply", 0o755, "python", "replace"),
|
||||||
("/usr/local/sbin/felhom-crash-guard", "felhom-crash-guard", 0o755, "python", "replace"),
|
("/usr/local/sbin/felhom-crash-guard", "felhom-crash-guard", 0o755, "python", "replace"),
|
||||||
# R-861 (agent v0.146.0): the content checker for every agent-staged file a root program reads, and the two FIXED
|
# R-861 (agent v0.146.0): the content checker for every agent-staged file a root program reads, and the two FIXED
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ longer name the destination: each verb has a fixed source and a fixed (or strict
|
|||||||
|
|
||||||
Verbs (each one sudoers line, exact-match pattern):
|
Verbs (each one sudoers line, exact-match pattern):
|
||||||
unit <name> /var/lib/felhom-agent/units/<name> -> /etc/systemd/system/<name> (.mount | .automount)
|
unit <name> /var/lib/felhom-agent/units/<name> -> /etc/systemd/system/<name> (.mount | .automount)
|
||||||
|
unit-forget <name> REMOVES /etc/systemd/system/<name> (R-1124): only a felhom by-UUID drive .mount unit that is
|
||||||
|
disabled, not mounted, and whose drive is absent (its /dev/disk/by-uuid link is gone)
|
||||||
dnsmasq <tmp> <name> /tmp/felhom-resolver-<digits>.conf -> /etc/dnsmasq.d/felhom-<...>.conf
|
dnsmasq <tmp> <name> /tmp/felhom-resolver-<digits>.conf -> /etc/dnsmasq.d/felhom-<...>.conf
|
||||||
wg /var/lib/felhom-agent/wg/wg-felhom.conf -> /etc/wireguard/wg-felhom.conf (0600)
|
wg /var/lib/felhom-agent/wg/wg-felhom.conf -> /etc/wireguard/wg-felhom.conf (0600)
|
||||||
sshd-config /var/lib/felhom-agent/felhom-sshd/sshd_config -> /etc/felhom-sshd/sshd_config
|
sshd-config /var/lib/felhom-agent/felhom-sshd/sshd_config -> /etc/felhom-sshd/sshd_config
|
||||||
@@ -41,7 +43,7 @@ WG_SRC, WG_DEST = STATE + "/wg/wg-felhom.conf", "/etc/wireguard/wg-felhom.conf"
|
|||||||
SSHD_SRC, SSHD_DEST = STATE + "/felhom-sshd/sshd_config", "/etc/felhom-sshd/sshd_config"
|
SSHD_SRC, SSHD_DEST = STATE + "/felhom-sshd/sshd_config", "/etc/felhom-sshd/sshd_config"
|
||||||
KEY_SRC, KEY_DEST = STATE + "/felhom-sshd/authorized_keys.felhom-op", "/etc/felhom-sshd/authorized_keys/felhom-op"
|
KEY_SRC, KEY_DEST = STATE + "/felhom-sshd/authorized_keys.felhom-op", "/etc/felhom-sshd/authorized_keys/felhom-op"
|
||||||
MAX_BYTES = 64 * 1024
|
MAX_BYTES = 64 * 1024
|
||||||
VERBS = ("unit", "dnsmasq", "wg", "sshd-config", "sshd-key", "controller-image")
|
VERBS = ("unit", "unit-forget", "dnsmasq", "wg", "sshd-config", "sshd-key", "controller-image")
|
||||||
# R-861 (a) A1 (`09` §3 decision 165): the SAME pattern as the agent's controllerImageRe (internal/localapi/
|
# R-861 (a) A1 (`09` §3 decision 165): the SAME pattern as the agent's controllerImageRe (internal/localapi/
|
||||||
# controllerswap.go) — a compromised agent cannot hand the guest's bootstrap any other image. Pinned by
|
# controllerswap.go) — a compromised agent cannot hand the guest's bootstrap any other image. Pinned by
|
||||||
# configs/test_felhom_priv_apply.py ControllerImage.
|
# configs/test_felhom_priv_apply.py ControllerImage.
|
||||||
@@ -51,6 +53,11 @@ CONTROLLER_IMAGE_MAX = 256
|
|||||||
VMID_RE = re.compile(r"^[0-9]{1,9}$")
|
VMID_RE = re.compile(r"^[0-9]{1,9}$")
|
||||||
|
|
||||||
UNIT_NAME_RE = re.compile(r"^mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$")
|
UNIT_NAME_RE = re.compile(r"^mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$")
|
||||||
|
DRIVE_UNIT_NAME_RE = re.compile(r"^mnt-[A-Za-z0-9_.\\-]+\.mount$")
|
||||||
|
DRIVE_UNIT_MARKER = "Managed by felhom-agent"
|
||||||
|
NET_UNIT_MARKER = "felhom network storage"
|
||||||
|
WANTS_DIR = UNIT_DIR + "/multi-user.target.wants"
|
||||||
|
BY_UUID_DIR = "/dev/disk/by-uuid"
|
||||||
DNSMASQ_TMP_RE = re.compile(r"^/tmp/felhom-resolver-[0-9]+\.conf$")
|
DNSMASQ_TMP_RE = re.compile(r"^/tmp/felhom-resolver-[0-9]+\.conf$")
|
||||||
DNSMASQ_NAME_RE = re.compile(r"^felhom-[a-z0-9][a-z0-9._-]*\.conf$")
|
DNSMASQ_NAME_RE = re.compile(r"^felhom-[a-z0-9][a-z0-9._-]*\.conf$")
|
||||||
SEG = r"[A-Za-z0-9_-][A-Za-z0-9_.-]*"
|
SEG = r"[A-Za-z0-9_-][A-Za-z0-9_.-]*"
|
||||||
@@ -135,6 +142,36 @@ class Host:
|
|||||||
def read_stdin(self, limit):
|
def read_stdin(self, limit):
|
||||||
return sys.stdin.buffer.read(limit + 1)
|
return sys.stdin.buffer.read(limit + 1)
|
||||||
|
|
||||||
|
# ---- unit-forget (R-1124) ----
|
||||||
|
def read_installed(self, path):
|
||||||
|
"""The installed unit: None when absent; Refused when it is not a regular file (a symlink is never followed)."""
|
||||||
|
try:
|
||||||
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None
|
||||||
|
except OSError as e:
|
||||||
|
raise Refused("F1", f"cannot open {path}: {e.strerror}")
|
||||||
|
with os.fdopen(fd, "rb") as f:
|
||||||
|
if not stat.S_ISREG(os.fstat(f.fileno()).st_mode):
|
||||||
|
raise Refused("F1", f"{path} is not a regular file")
|
||||||
|
return f.read(MAX_BYTES + 1)
|
||||||
|
|
||||||
|
def lexists(self, path):
|
||||||
|
return os.path.lexists(path)
|
||||||
|
|
||||||
|
def mountpoints(self):
|
||||||
|
"""Every mount point in /proc/self/mountinfo (field 5, octal escapes decoded)."""
|
||||||
|
out = set()
|
||||||
|
with open("/proc/self/mountinfo", "r", encoding="utf-8", errors="replace") as f:
|
||||||
|
for line in f:
|
||||||
|
parts = line.split(" ")
|
||||||
|
if len(parts) > 4:
|
||||||
|
out.add(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), parts[4]))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def remove(self, path):
|
||||||
|
os.remove(path)
|
||||||
|
|
||||||
def write_guest_image(self, vmid, data):
|
def write_guest_image(self, vmid, data):
|
||||||
"""As root: `pct exec <vmid> -- tee <the fixed file>` with the checked ref on stdin (no shell)."""
|
"""As root: `pct exec <vmid> -- tee <the fixed file>` with the checked ref on stdin (no shell)."""
|
||||||
r = subprocess.run(["/usr/sbin/pct", "exec", str(vmid), "--", "tee", CONTROLLER_IMAGE_FILE],
|
r = subprocess.run(["/usr/sbin/pct", "exec", str(vmid), "--", "tee", CONTROLLER_IMAGE_FILE],
|
||||||
@@ -424,6 +461,45 @@ def controller_image(rest, host):
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def unit_forget(rest, host):
|
||||||
|
"""R-1124: remove ONE felhom drive mount unit that the agent has already disabled, whose drive is gone. A drive that
|
||||||
|
is still present or still mounted keeps its unit — this verb can only forget, never unmount or take a drive away."""
|
||||||
|
try:
|
||||||
|
if len(rest) != 1 or not DRIVE_UNIT_NAME_RE.match(rest[0]) or "/" in rest[0]:
|
||||||
|
raise Refused("A1", "usage: felhom-priv-apply unit-forget mnt-<escaped path>.mount")
|
||||||
|
name = rest[0]
|
||||||
|
path = os.path.join(UNIT_DIR, name)
|
||||||
|
data = host.read_installed(path)
|
||||||
|
if data is None:
|
||||||
|
host.log(f"felhom-priv-apply: ABSENT unit-forget {path} (nothing to remove)")
|
||||||
|
return 0
|
||||||
|
text = text_of(data, path)
|
||||||
|
if DRIVE_UNIT_MARKER not in text or NET_UNIT_MARKER in text:
|
||||||
|
raise Refused("F2", f"{name} is not a felhom drive unit")
|
||||||
|
check_unit(name, text)
|
||||||
|
s = parse_ini(text, name)
|
||||||
|
what, where = s["Mount"].get("What", ""), s["Mount"]["Where"]
|
||||||
|
mu = re.match(r"^/dev/disk/by-uuid/(.+)$", what)
|
||||||
|
if not mu:
|
||||||
|
raise Refused("F2", f"{name} is not a by-UUID drive unit")
|
||||||
|
if host.lexists(os.path.join(WANTS_DIR, name)):
|
||||||
|
raise Refused("F3", f"{name} is still enabled (disable it first)")
|
||||||
|
if where in host.mountpoints():
|
||||||
|
raise Refused("F4", f"{where} is mounted")
|
||||||
|
if host.lexists(os.path.join(BY_UUID_DIR, mu.group(1))):
|
||||||
|
raise Refused("F5", f"the drive of {name} is present (filesystem UUID {mu.group(1)})")
|
||||||
|
except Refused as e:
|
||||||
|
host.log(f"felhom-priv-apply: REFUSED [{e.rule}] unit-forget {' '.join(rest)[:120]}: {e.reason}")
|
||||||
|
return 2 if e.rule == "A1" else 3
|
||||||
|
try:
|
||||||
|
host.remove(path)
|
||||||
|
except OSError as e:
|
||||||
|
host.log(f"felhom-priv-apply: FAILED unit-forget {path}: {e}")
|
||||||
|
return 4
|
||||||
|
host.log(f"felhom-priv-apply: REMOVED unit-forget {path} (where={where})")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def main(argv, host=None):
|
def main(argv, host=None):
|
||||||
host = host or Host()
|
host = host or Host()
|
||||||
if argv == ["--self-check"]:
|
if argv == ["--self-check"]:
|
||||||
@@ -445,6 +521,8 @@ def main(argv, host=None):
|
|||||||
return 0
|
return 0
|
||||||
if argv and argv[0] == "controller-image":
|
if argv and argv[0] == "controller-image":
|
||||||
return controller_image(argv[1:], host)
|
return controller_image(argv[1:], host)
|
||||||
|
if argv and argv[0] == "unit-forget":
|
||||||
|
return unit_forget(argv[1:], host)
|
||||||
try:
|
try:
|
||||||
verb, src, dest, mode, checker = plan(argv)
|
verb, src, dest, mode, checker = plan(argv)
|
||||||
data = host.read_source(src)
|
data = host.read_source(src)
|
||||||
|
|||||||
Executable
+74
@@ -0,0 +1,74 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# felhom-thin-check — list the thin device ids a thin pool holds next to the ones LVM knows (R-1012). READ-ONLY: it
|
||||||
|
# names an orphan, it never removes one. Installed as /usr/local/sbin/felhom-thin-check (config bundle).
|
||||||
|
#
|
||||||
|
# felhom-thin-check <vg> <pool>
|
||||||
|
#
|
||||||
|
# A full host root while vzdump takes its snapshot can leave a thin snapshot inside the pool that LVM no longer knows
|
||||||
|
# (demo-hp 2026-10-14: `thin_ls` device 4, LVM only 1 and 2); from then on every snapshot backup of that guest fails
|
||||||
|
# („Creation of new snapshot 4 of device 2 failed") and nothing said why. The repair (`dmsetup message … delete <id>`)
|
||||||
|
# stays a HUMAN step, decided by the operator — this helper only reads.
|
||||||
|
#
|
||||||
|
# Reading the live pool's device list needs a metadata snapshot: the kernel's documented way to read metadata while
|
||||||
|
# the pool is in use (`reserve_metadata_snap`, `thin_ls --metadata-snap`, `release_metadata_snap`). It changes no data
|
||||||
|
# and no mapping; it is held for the length of one `thin_ls -o DEV` (well under a second on these pools). Rules:
|
||||||
|
# 1. the arguments are an LVM name pair; <vg>/<pool> is a thin pool (lv_attr starts with `t`);
|
||||||
|
# 2. only one run at a time (a lock file); a metadata snapshot that someone ELSE holds is never touched — the run
|
||||||
|
# refuses (exit 3) and does not release it;
|
||||||
|
# 3. LVM's list is read before AND after the pool's, so a volume created in between is not reported as an orphan
|
||||||
|
# (the caller treats an id as an orphan only when it is in the pool and in neither LVM list).
|
||||||
|
# Output, one record per line: lvm-before <id> <lv> | pool <id> | lvm-after <id> <lv> | ok
|
||||||
|
# Exit: 0 read, 1 refused (arguments / not a thin pool), 3 a metadata snapshot is already held or cannot be reserved,
|
||||||
|
# 4 the pool could not be read. Pinned by configs/test_felhom_thin_check.py.
|
||||||
|
set -u
|
||||||
|
LVS=/usr/sbin/lvs
|
||||||
|
DMSETUP=/usr/sbin/dmsetup
|
||||||
|
THIN_LS=/usr/sbin/thin_ls
|
||||||
|
LOCK=/run/lock/felhom-thin-check.lock
|
||||||
|
NAME_RE='^[A-Za-z0-9_+][A-Za-z0-9_.+-]{0,63}$'
|
||||||
|
|
||||||
|
die() { echo "felhom-thin-check: REFUSED: $1" >&2; exit "${2:-1}"; } # $1 the reason, $2 the exit code
|
||||||
|
|
||||||
|
lvm_ids() { # $1 tag → "<tag> <id> <lv>" for every thin volume of this pool
|
||||||
|
"$LVS" --noheadings --separator , -o lv_name,thin_id --select "pool_lv=$POOL" -- "$VG" 2>/dev/null |
|
||||||
|
awk -F, -v tag="$1" '{ gsub(/ /, ""); if ($2 ~ /^[0-9]+$/) print tag, $2, $1 }'
|
||||||
|
}
|
||||||
|
|
||||||
|
release() { "$DMSETUP" message "$TPOOL" 0 release_metadata_snap >/dev/null 2>&1; }
|
||||||
|
|
||||||
|
main() {
|
||||||
|
[[ $# -eq 2 ]] || die "usage: felhom-thin-check <vg> <pool>"
|
||||||
|
VG=$1 POOL=$2
|
||||||
|
local attr held ids
|
||||||
|
[[ "$VG" =~ $NAME_RE && "$POOL" =~ $NAME_RE ]] || die "not an LVM name pair"
|
||||||
|
|
||||||
|
attr=$("$LVS" --noheadings -o lv_attr -- "$VG/$POOL" 2>/dev/null | tr -d ' ')
|
||||||
|
[[ "$attr" == t* ]] || die "$VG/$POOL is not a thin pool"
|
||||||
|
|
||||||
|
DM="${VG//-/--}-${POOL//-/--}"
|
||||||
|
TPOOL="$DM-tpool"
|
||||||
|
TMETA="/dev/mapper/${DM}_tmeta"
|
||||||
|
|
||||||
|
exec 9>"$LOCK" || die "cannot open the lock file" 4
|
||||||
|
flock -w 30 9 || die "another check holds the lock" 3
|
||||||
|
|
||||||
|
# Field 7 of the thin-pool status is the held metadata root: "-" when no metadata snapshot exists.
|
||||||
|
held=$("$DMSETUP" status "$TPOOL" 2>/dev/null | awk '$3 == "thin-pool" { print $7 }')
|
||||||
|
[[ -n "$held" ]] || die "cannot read the status of $TPOOL" 4
|
||||||
|
[[ "$held" == "-" ]] || die "a metadata snapshot of $TPOOL is already held (by someone else) — not touched" 3
|
||||||
|
|
||||||
|
lvm_ids lvm-before
|
||||||
|
"$DMSETUP" message "$TPOOL" 0 reserve_metadata_snap >/dev/null 2>&1 || die "could not reserve a metadata snapshot of $TPOOL" 3
|
||||||
|
trap release EXIT
|
||||||
|
trap 'release; trap - EXIT; exit 4' INT TERM HUP
|
||||||
|
if ! ids=$("$THIN_LS" --metadata-snap --no-headers -o DEV "$TMETA" 2>/dev/null); then
|
||||||
|
die "thin_ls could not read $TMETA" 4
|
||||||
|
fi
|
||||||
|
release
|
||||||
|
trap - EXIT INT TERM HUP
|
||||||
|
awk '$1 ~ /^[0-9]+$/ { print "pool", $1 }' <<<"$ids"
|
||||||
|
lvm_ids lvm-after
|
||||||
|
echo ok
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@"; fi
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Tests for configs/felhom-internal-place (felhom.eu architecture/16 §5, §7 — sizes ruled 2026-10-16).
|
||||||
|
|
||||||
|
The helper is run for real under bash; `lvs`, `lvcreate`, `blkid`, `mkfs.ext4` and `stat` are PATH stubs that read and
|
||||||
|
write one JSON state file, so nothing on this machine is touched. Every refusal has a positive control: the same setup
|
||||||
|
without the one condition goes through — so a test cannot pass because the helper refuses everything.
|
||||||
|
Run: python3 configs/test_felhom_internal_place.py (also run by the Go suite, internal/internalplace)
|
||||||
|
"""
|
||||||
|
import json, os, pathlib, stat, subprocess, tempfile, unittest
|
||||||
|
|
||||||
|
HELPER = pathlib.Path(__file__).resolve().parent / "felhom-internal-place"
|
||||||
|
GIB = 1 << 30
|
||||||
|
|
||||||
|
STUB_LVS = r'''#!/usr/bin/env python3
|
||||||
|
import json, os, sys
|
||||||
|
st = json.load(open(os.environ["IP_STATE"]))
|
||||||
|
if st.get("lvs_fail_listing") and "lv_name,pool_lv,lv_size" in sys.argv: sys.exit(5)
|
||||||
|
args = [a for a in sys.argv[1:] if not a.startswith("-") and a not in ("b",)]
|
||||||
|
fields = sys.argv[sys.argv.index("-o") + 1].split(",")
|
||||||
|
target = args[-1]
|
||||||
|
listing = target == st["vg"]
|
||||||
|
for lv in st["lvs"]:
|
||||||
|
full = st["vg"] + "/" + lv["name"]
|
||||||
|
if listing and st.get("hide_pool_in_listing") and lv.get("pool", "") == "":
|
||||||
|
continue # a listing that silently lost the pool row (the pool's own size stays readable)
|
||||||
|
if listing or target == full:
|
||||||
|
row = {"lv_name": lv["name"], "pool_lv": lv.get("pool", ""), "lv_size": str(lv["size"])}
|
||||||
|
print(" " + " ".join(row[f] for f in fields))
|
||||||
|
'''
|
||||||
|
STUB_LVCREATE = r'''#!/usr/bin/env python3
|
||||||
|
import json, os, sys
|
||||||
|
p = os.environ["IP_STATE"]; st = json.load(open(p))
|
||||||
|
a = sys.argv[1:]
|
||||||
|
size = int(a[a.index("-V") + 1].rstrip("G")) * (1 << 30)
|
||||||
|
vg, pool = a[a.index("-T") + 1].split("/")
|
||||||
|
st["lvs"].append({"name": a[a.index("-n") + 1], "pool": pool, "size": size, "active": True})
|
||||||
|
st.setdefault("calls", []).append("lvcreate " + " ".join(a))
|
||||||
|
if st.get("dirty_new"): st["fs"]["felhom-belso"] = {"TYPE": "", "RAW": "junk"}
|
||||||
|
json.dump(st, open(p, "w"))
|
||||||
|
'''
|
||||||
|
STUB_BLKID = r'''#!/usr/bin/env python3
|
||||||
|
# like blkid -p: exit 0 when something is found, 2 when nothing is; st["blkid_rc"] forces an error exit
|
||||||
|
import json, os, sys
|
||||||
|
st = json.load(open(os.environ["IP_STATE"]))
|
||||||
|
if st.get("blkid_rc"): sys.exit(st["blkid_rc"])
|
||||||
|
fs = st["fs"].get(os.path.basename(sys.argv[-1]), {})
|
||||||
|
if "-s" in sys.argv:
|
||||||
|
v = fs.get(sys.argv[sys.argv.index("-s") + 1], "")
|
||||||
|
if v: print(v)
|
||||||
|
sys.exit(0 if v else 2)
|
||||||
|
vals = [fs.get(k, "") for k in ("TYPE", "LABEL", "UUID", "RAW") if fs.get(k)]
|
||||||
|
if vals: print("\n".join(vals))
|
||||||
|
sys.exit(0 if vals else 2)
|
||||||
|
'''
|
||||||
|
STUB_MKFS = r'''#!/usr/bin/env python3
|
||||||
|
import json, os, sys
|
||||||
|
p = os.environ["IP_STATE"]; st = json.load(open(p))
|
||||||
|
dev = os.path.basename(sys.argv[-1])
|
||||||
|
st["fs"][dev] = {"TYPE": "ext4", "LABEL": sys.argv[sys.argv.index("-L") + 1], "UUID": "0b6e4c5a-1d2e-4f3a-9b8c-7d6e5f4a3b2c"}
|
||||||
|
st.setdefault("calls", []).append("mkfs.ext4 " + " ".join(sys.argv[1:]))
|
||||||
|
json.dump(st, open(p, "w"))
|
||||||
|
'''
|
||||||
|
STUB_STAT = r'''#!/usr/bin/env python3
|
||||||
|
import json, os, sys
|
||||||
|
st = json.load(open(os.environ["IP_STATE"]))
|
||||||
|
name = os.path.basename(sys.argv[-1])
|
||||||
|
if any(lv["name"] == name and lv.get("active", True) for lv in st["lvs"]): print("block special file")
|
||||||
|
else: sys.exit(1)
|
||||||
|
'''
|
||||||
|
|
||||||
|
CFG = "dir: local\n\tpath /var/lib/vz\n\tcontent iso,backup\n\nlvmthin: local-lvm\n\tthinpool data\n\tvgname pve\n\tcontent rootdir,images\n"
|
||||||
|
|
||||||
|
|
||||||
|
class Box:
|
||||||
|
def __init__(self, pool_gib, lvs=(), fs=None, cfg=CFG, **extra):
|
||||||
|
self.t = tempfile.TemporaryDirectory()
|
||||||
|
r = pathlib.Path(self.t.name)
|
||||||
|
self.bin = r / "bin"; self.bin.mkdir()
|
||||||
|
for name, body in (("lvs", STUB_LVS), ("lvcreate", STUB_LVCREATE), ("blkid", STUB_BLKID),
|
||||||
|
("mkfs.ext4", STUB_MKFS), ("stat", STUB_STAT)):
|
||||||
|
f = self.bin / name; f.write_text(body); f.chmod(0o755)
|
||||||
|
self.cfg = r / "storage.cfg"; self.cfg.write_text(cfg)
|
||||||
|
self.state = r / "state.json"
|
||||||
|
st = {"vg": "pve", "lvs": [{"name": "data", "pool": "", "size": pool_gib * GIB}] + [dict(l) for l in lvs],
|
||||||
|
"fs": fs or {}}
|
||||||
|
st.update(extra)
|
||||||
|
self.state.write_text(json.dumps(st))
|
||||||
|
|
||||||
|
def run(self, *args):
|
||||||
|
env = {"PATH": f"{self.bin}:/usr/bin:/bin", "IP_STATE": str(self.state), "FELHOM_STORAGE_CFG": str(self.cfg)}
|
||||||
|
p = subprocess.run(["bash", str(HELPER), *args], env=env, capture_output=True, text=True)
|
||||||
|
return p.returncode, p.stdout.strip(), p.stderr.strip()
|
||||||
|
|
||||||
|
def st(self):
|
||||||
|
return json.loads(self.state.read_text())
|
||||||
|
|
||||||
|
def calls(self):
|
||||||
|
return self.st().get("calls", [])
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.t.cleanup()
|
||||||
|
|
||||||
|
|
||||||
|
def thin(name, gib, pool="data"):
|
||||||
|
return {"name": name, "pool": pool, "size": gib * GIB}
|
||||||
|
|
||||||
|
|
||||||
|
class Sizes(unittest.TestCase):
|
||||||
|
"""The operator's table (SPACE-CHECK §4, ruled 2026-10-16), as the helper computes it on a fresh box."""
|
||||||
|
|
||||||
|
def plan(self, pool, planned, lvs=()):
|
||||||
|
b = Box(pool, lvs)
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("plan", str(planned))
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertEqual(b.calls(), [], "plan must change nothing")
|
||||||
|
return dict(kv.split("=") for kv in out.split()[1:])
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_the_ruled_table(self):
|
||||||
|
# disk → pool (GiB), data volume (planned = 32 + mp0), expected internal
|
||||||
|
for pool, mp0, reserve, internal in ((140, 50, 16, 42), (349, 100, 34, 183), (800, 130, 48, 590), (1720, 130, 48, 1510)):
|
||||||
|
got = self.plan(pool, 32 + mp0)
|
||||||
|
self.assertEqual((int(got["reserve"]), int(got["internal"])), (reserve, internal), (pool, got))
|
||||||
|
|
||||||
|
def test_promised_volumes_count_but_vzdump_snapshots_and_other_pools_do_not(self):
|
||||||
|
got = self.plan(349, 0, [thin("vm-9201-disk-0", 32), thin("vm-9201-disk-1", 250),
|
||||||
|
thin("snap_vm-9201-disk-1_vzdump", 250), thin("vm-500-disk-0", 40, pool="other")])
|
||||||
|
self.assertEqual((int(got["promised"]), int(got["internal"])), (282, 33), got)
|
||||||
|
|
||||||
|
|
||||||
|
class Create(unittest.TestCase):
|
||||||
|
def test_a_fresh_box_gets_the_place(self):
|
||||||
|
b = Box(349)
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("create", "132")
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertRegex(out, r"^created uuid=[0-9a-f-]{36} gib=183$")
|
||||||
|
self.assertEqual(len([c for c in b.calls() if c.startswith("lvcreate")]), 1)
|
||||||
|
self.assertIn("-V 183G -T pve/data -n felhom-belso", b.calls()[0])
|
||||||
|
self.assertIn("-L felhom-belso", b.calls()[1])
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_too_little_space_makes_nothing(self):
|
||||||
|
b = Box(53) # demo-hp's 128 GB disk
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("create", str(32 + 50))
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertTrue(out.startswith("skipped "), out)
|
||||||
|
self.assertEqual(b.calls(), [])
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_an_existing_place_is_found_never_formatted(self):
|
||||||
|
fs = {"felhom-belso": {"TYPE": "ext4", "LABEL": "felhom-belso", "UUID": "11111111-2222-3333-4444-555555555555"}}
|
||||||
|
b = Box(349, [thin("felhom-belso", 183)], fs)
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("create", "0")
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertEqual(out, "existing uuid=11111111-2222-3333-4444-555555555555 gib=183")
|
||||||
|
self.assertEqual(b.calls(), [], "an existing place must never be created or formatted again")
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_a_foreign_filesystem_on_the_name_is_refused(self):
|
||||||
|
# positive control: test_an_existing_place_is_found_never_formatted (same LV, Felhom's label) goes through
|
||||||
|
for fs in ({"TYPE": "xfs", "LABEL": "felhom-belso", "UUID": "x"}, {"TYPE": "ext4", "LABEL": "films", "UUID": "y"}):
|
||||||
|
b = Box(349, [thin("felhom-belso", 183)], {"felhom-belso": fs})
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("create", "0")
|
||||||
|
self.assertEqual(rc, 1, (fs, out))
|
||||||
|
self.assertIn("REFUSED", err)
|
||||||
|
self.assertEqual(b.calls(), [], fs)
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_a_crash_between_lvcreate_and_mkfs_is_finished(self):
|
||||||
|
b = Box(349, [thin("felhom-belso", 183)], {})
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("create", "0")
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertRegex(out, r"^formatted uuid=")
|
||||||
|
self.assertEqual([c.split()[0] for c in b.calls()], ["mkfs.ext4"])
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_an_unnamed_signature_is_refused(self):
|
||||||
|
b = Box(349, [thin("felhom-belso", 183)], {"felhom-belso": {"RAW": "something"}})
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("create", "0")
|
||||||
|
self.assertEqual(rc, 1, out)
|
||||||
|
self.assertEqual(b.calls(), [])
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_a_new_volume_that_is_not_empty_is_not_formatted(self):
|
||||||
|
b = Box(349, dirty_new=True)
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("create", "132")
|
||||||
|
self.assertEqual(rc, 1, out)
|
||||||
|
self.assertNotIn("mkfs", " ".join(b.calls()))
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_bad_arguments_and_no_local_lvm_are_refused(self):
|
||||||
|
b = Box(349, cfg="dir: local\n\tpath /var/lib/vz\n")
|
||||||
|
try:
|
||||||
|
self.assertEqual(b.run("create", "0")[0], 1)
|
||||||
|
self.assertEqual(b.calls(), [])
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
b = Box(349)
|
||||||
|
try:
|
||||||
|
for args in (("create", "-1"), ("create", "1x"), ("wipe", "0"), ("create",)):
|
||||||
|
self.assertEqual(b.run(*args)[0], 1, args)
|
||||||
|
self.assertEqual(b.calls(), [])
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
|
||||||
|
class FailsClosed(unittest.TestCase):
|
||||||
|
"""A read that fails must never read as „empty" or „nothing promised". Positive control: Create (same setups)."""
|
||||||
|
|
||||||
|
def test_a_failed_volume_listing_creates_nothing(self):
|
||||||
|
b = Box(349, lvs_fail_listing=True)
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("create", "132")
|
||||||
|
self.assertEqual(rc, 1, out)
|
||||||
|
self.assertEqual(b.calls(), [])
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_a_listing_without_the_pool_creates_nothing(self):
|
||||||
|
b = Box(349, [thin("vm-9201-disk-1", 250)], hide_pool_in_listing=True)
|
||||||
|
try:
|
||||||
|
self.assertEqual(b.run("create", "132")[0], 1)
|
||||||
|
self.assertEqual(b.calls(), [])
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
def test_a_blkid_error_never_reads_as_empty(self):
|
||||||
|
for setup in ({"lvs": [thin("felhom-belso", 183)], "fs": {}}, {}):
|
||||||
|
b = Box(349, **({"lvs": setup["lvs"], "fs": setup["fs"]} if setup else {}), blkid_rc=4)
|
||||||
|
try:
|
||||||
|
rc, out, err = b.run("create", "0" if setup else "132")
|
||||||
|
self.assertEqual(rc, 1, (setup, out))
|
||||||
|
self.assertNotIn("mkfs", " ".join(b.calls()), setup)
|
||||||
|
finally:
|
||||||
|
b.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Tests for configs/felhom-mkfs-guarded.sh — the ONLY mkfs path the agent's sudoers permits.
|
||||||
|
|
||||||
|
First tests of this wrapper (agent v0.158.0), written for partitions as places (felhom.eu architecture/16 §6). Every
|
||||||
|
mount of every node of the disk outside /mnt/felhom-drives refuses — INCLUDING a sibling that is Felhom's own drive
|
||||||
|
(its raw /mnt/<name> mount): the agent can make that corroborating mount itself, so the root wrapper must not trust
|
||||||
|
it (security review of 912f7c1). An unlistable disk refuses.
|
||||||
|
|
||||||
|
HOW IT RUNS ANYWHERE: the wrapper is copied with five literal host paths replaced (each replacement is asserted, so a
|
||||||
|
drifted wrapper fails here instead of silently testing nothing): /proc/mounts → a fake mount table, the block-device
|
||||||
|
test → true (no real /dev nodes here), the pvs candidates → a stub, /sys/block → a fake tree, the mkfs binaries → a
|
||||||
|
recorder. `lsblk` is a PATH stub answering the three calls the wrapper makes from one JSON disk description.
|
||||||
|
Every refusal has a positive control (the same disk without the one condition is formatted).
|
||||||
|
Run: python3 configs/test_felhom_mkfs_guarded.py (also run by the Go suite, internal/storage)
|
||||||
|
"""
|
||||||
|
import json, os, pathlib, subprocess, tempfile, unittest
|
||||||
|
|
||||||
|
WRAPPER = pathlib.Path(__file__).resolve().parent / "felhom-mkfs-guarded.sh"
|
||||||
|
|
||||||
|
STUB_LSBLK = r'''#!/usr/bin/env python3
|
||||||
|
import json, os, sys
|
||||||
|
st = json.load(open(os.environ["MG_STATE"]))
|
||||||
|
a = sys.argv[1:]; dev = a[-1]
|
||||||
|
nodes = st["nodes"] # [{"name": "/dev/sdb", "pk": "", "fstype": ""}, ...]
|
||||||
|
if a[0] == "-ndo" and a[1] == "PKNAME":
|
||||||
|
for n in nodes:
|
||||||
|
if n["name"] == dev and n.get("pk"): print(n["pk"])
|
||||||
|
elif a[0] == "-nrpo" and a[1] == "NAME":
|
||||||
|
if not st.get("lsblk_empty"):
|
||||||
|
for n in nodes: print(n["name"])
|
||||||
|
elif a[0] == "-nro" and a[1] == "FSTYPE":
|
||||||
|
for n in nodes: print(n.get("fstype", ""))
|
||||||
|
elif a[0] == "-nro" and a[1] == "MOUNTPOINT": # the pre-v0.158.0 wrapper's call: one mountpoint per node, like lsblk
|
||||||
|
mt = [l.split()[:2] for l in open(os.environ["MG_MOUNTS"]) if l.strip()]
|
||||||
|
for n in nodes:
|
||||||
|
mps = [m for d, m in mt if d == n["name"]]
|
||||||
|
print(mps[0] if mps else "")
|
||||||
|
'''
|
||||||
|
STUB_MKFS = '#!/bin/sh\necho "$0 $*" >> "$MG_CALLS"\n'
|
||||||
|
STUB_PVS = '#!/bin/sh\nprintf "%s\\n" $MG_PVS\n'
|
||||||
|
|
||||||
|
|
||||||
|
class Disk:
|
||||||
|
def __init__(self, nodes, mounts, ro="0", pvs="", lsblk_empty=False):
|
||||||
|
self.t = tempfile.TemporaryDirectory()
|
||||||
|
r = pathlib.Path(self.t.name)
|
||||||
|
self.bin = r / "bin"; self.bin.mkdir()
|
||||||
|
(self.bin / "lsblk").write_text(STUB_LSBLK); (self.bin / "lsblk").chmod(0o755)
|
||||||
|
for n in ("mkfs.ext4", "mkfs.xfs"):
|
||||||
|
(self.bin / n).write_text(STUB_MKFS); (self.bin / n).chmod(0o755)
|
||||||
|
(self.bin / "pvs").write_text(STUB_PVS); (self.bin / "pvs").chmod(0o755)
|
||||||
|
self.state = r / "state.json"
|
||||||
|
self.state.write_text(json.dumps({"nodes": nodes, "lsblk_empty": lsblk_empty}))
|
||||||
|
self.mounts = r / "mounts"
|
||||||
|
self.mounts.write_text("".join(f"{d} {m} ext4 rw 0 0\n" for d, m in mounts))
|
||||||
|
self.sys = r / "sys"
|
||||||
|
for n in nodes:
|
||||||
|
if not n.get("pk"):
|
||||||
|
d = self.sys / os.path.basename(n["name"]); d.mkdir(parents=True); (d / "ro").write_text(ro + "\n")
|
||||||
|
self.calls = r / "calls"
|
||||||
|
self.pvs = pvs
|
||||||
|
src = WRAPPER.read_text()
|
||||||
|
subs = [
|
||||||
|
("/proc/mounts", str(self.mounts)),
|
||||||
|
('[[ -b "$dev" ]]', "true"),
|
||||||
|
("for c in /usr/sbin/pvs /sbin/pvs; do", f"for c in {self.bin}/pvs; do"),
|
||||||
|
('rof="/sys/block/', f'rof="{self.sys}/'),
|
||||||
|
("exec /usr/sbin/mkfs.ext4", f"exec {self.bin}/mkfs.ext4"),
|
||||||
|
("exec /usr/sbin/mkfs.xfs ", f"exec {self.bin}/mkfs.xfs "),
|
||||||
|
]
|
||||||
|
for a, b in subs:
|
||||||
|
assert a in src, f"the wrapper no longer contains {a!r} — update this test's substitutions"
|
||||||
|
src = src.replace(a, b)
|
||||||
|
self.copy = r / "mkfs-guarded"; self.copy.write_text(src)
|
||||||
|
|
||||||
|
def run(self, dev, fstype="ext4"):
|
||||||
|
env = {"PATH": f"{self.bin}:/usr/bin:/bin", "MG_STATE": str(self.state), "MG_CALLS": str(self.calls),
|
||||||
|
"MG_MOUNTS": str(self.mounts),
|
||||||
|
"MG_PVS": self.pvs}
|
||||||
|
p = subprocess.run(["bash", str(self.copy), dev, fstype], env=env, capture_output=True, text=True)
|
||||||
|
made = self.calls.read_text().split("\n")[0] if self.calls.exists() else ""
|
||||||
|
return p.returncode, p.stderr.strip(), made
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.t.cleanup()
|
||||||
|
|
||||||
|
|
||||||
|
def disk_sdb(sdb1_fstype="ext4", sdb2_fstype=""):
|
||||||
|
return [{"name": "/dev/sdb", "pk": "", "fstype": ""},
|
||||||
|
{"name": "/dev/sdb1", "pk": "sdb", "fstype": sdb1_fstype},
|
||||||
|
{"name": "/dev/sdb2", "pk": "sdb", "fstype": sdb2_fstype}]
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = ("/dev/sda2", "/")
|
||||||
|
OWN_SDB1 = [("/dev/sdb1", "/mnt/hdd_1"), ("/dev/sdb1", "/mnt/felhom-drives/hdd_1")]
|
||||||
|
|
||||||
|
|
||||||
|
class Partitions(unittest.TestCase):
|
||||||
|
def check(self, mounts, dev="/dev/sdb2", nodes=None, want_ok=True, **kw):
|
||||||
|
d = Disk(nodes or disk_sdb(), [ROOT] + mounts, **kw)
|
||||||
|
try:
|
||||||
|
rc, err, made = d.run(dev)
|
||||||
|
if want_ok:
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertTrue(made.endswith(f"mkfs.ext4 -F {dev}"), made)
|
||||||
|
else:
|
||||||
|
self.assertEqual(rc, 1, (mounts, made))
|
||||||
|
self.assertIn("REFUSED", err)
|
||||||
|
self.assertEqual(made, "", "a refused format must not run mkfs")
|
||||||
|
return err
|
||||||
|
finally:
|
||||||
|
d.close()
|
||||||
|
|
||||||
|
def test_a_free_part_on_an_unmounted_disk_is_formatted(self):
|
||||||
|
self.check([]) # positive control for every refusal below
|
||||||
|
|
||||||
|
def test_the_free_part_next_to_felhoms_own_drive_is_refused(self):
|
||||||
|
# The agent could make the „corroborating" managed mount itself: the root wrapper does not trust it.
|
||||||
|
self.check(OWN_SDB1, want_ok=False)
|
||||||
|
|
||||||
|
def test_a_sibling_mounted_raw_only_is_refused(self):
|
||||||
|
self.check([("/dev/sdb1", "/mnt/hdd_1")], want_ok=False)
|
||||||
|
|
||||||
|
def test_a_sibling_mounted_elsewhere_is_refused_even_with_a_managed_mount(self):
|
||||||
|
for elsewhere in ("/srv/data", "/media/usb", "/mnt/a/b", "/home/x"):
|
||||||
|
self.check([("/dev/sdb1", elsewhere), ("/dev/sdb1", "/mnt/felhom-drives/hdd_1")], want_ok=False)
|
||||||
|
|
||||||
|
def test_the_target_itself_mounted_raw_is_refused(self):
|
||||||
|
self.check([("/dev/sdb2", "/mnt/films"), ("/dev/sdb2", "/mnt/felhom-drives/films")], want_ok=False)
|
||||||
|
|
||||||
|
def test_the_target_mounted_only_under_felhom_drives_is_formatted(self):
|
||||||
|
self.check([("/dev/sdb2", "/mnt/felhom-drives/films")])
|
||||||
|
|
||||||
|
def test_an_unlistable_disk_is_refused(self):
|
||||||
|
self.check([], want_ok=False, lsblk_empty=True)
|
||||||
|
|
||||||
|
def test_the_catastrophic_cases_still_refuse(self):
|
||||||
|
self.check([("/dev/sdb1", "/")], want_ok=False) # system disk
|
||||||
|
self.check([], nodes=disk_sdb(sdb1_fstype="LVM2_member"), want_ok=False) # a member signature
|
||||||
|
self.check([], pvs="/dev/sdb1", want_ok=False) # an LVM PV (pvs)
|
||||||
|
self.check([], ro="1", want_ok=False) # read-only
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -376,5 +376,67 @@ class ControllerImage(unittest.TestCase):
|
|||||||
self.assertIn("controller-image", buf.getvalue())
|
self.assertIn("controller-image", buf.getvalue())
|
||||||
|
|
||||||
|
|
||||||
|
class UnitForget(unittest.TestCase):
|
||||||
|
"""R-1124: the agent forgets a decommissioned drive whose disk is gone — the root half removes its unit file, and
|
||||||
|
ONLY a felhom by-UUID drive unit that is disabled, not mounted, and whose /dev/disk/by-uuid link is absent.
|
||||||
|
RED-PROOF: on the pre-R-1124 wrapper `unit-forget` is not a verb (A1 usage, rc 2) — the removal case fails."""
|
||||||
|
NAME = "mnt-hdd_1.mount"
|
||||||
|
PATH = "/etc/systemd/system/mnt-hdd_1.mount"
|
||||||
|
UUID_LINK = "/dev/disk/by-uuid/91d2dc2d-2d28-4929-9bdd-3e11fa2f41ae"
|
||||||
|
|
||||||
|
def host(self, text=LOCAL_UNIT, kind="file", links=(), mounted=()):
|
||||||
|
h = FakeHost()
|
||||||
|
h.installed = {} if text is None else {self.PATH: (kind, text.encode())}
|
||||||
|
h.links, h.mounted, h.removed = set(links), set(mounted), []
|
||||||
|
h.read_installed = lambda p: (None if p not in h.installed else
|
||||||
|
(_ for _ in ()).throw(pa.Refused("F1", f"{p} is not a regular file"))
|
||||||
|
if h.installed[p][0] != "file" else h.installed[p][1])
|
||||||
|
h.lexists = lambda p: p in h.links
|
||||||
|
h.mountpoints = lambda: set(h.mounted)
|
||||||
|
h.remove = lambda p: h.removed.append(p)
|
||||||
|
return h
|
||||||
|
|
||||||
|
def test_an_absent_disabled_unmounted_drive_unit_is_removed(self):
|
||||||
|
h = self.host()
|
||||||
|
self.assertEqual(run(h, "unit-forget", self.NAME), 0, h.logs)
|
||||||
|
self.assertEqual(h.removed, [self.PATH])
|
||||||
|
|
||||||
|
def test_already_gone_is_fine(self):
|
||||||
|
h = self.host(text=None)
|
||||||
|
self.assertEqual(run(h, "unit-forget", self.NAME), 0, h.logs)
|
||||||
|
self.assertEqual(h.removed, [])
|
||||||
|
|
||||||
|
def refused(self, h, rule, *argv):
|
||||||
|
rc = run(h, *(argv or ("unit-forget", self.NAME)))
|
||||||
|
self.assertIn(rc, (2, 3), h.logs)
|
||||||
|
self.assertEqual(h.removed, [], h.logs)
|
||||||
|
self.assertTrue(any(f"[{rule}]" in l for l in h.logs), h.logs)
|
||||||
|
|
||||||
|
def test_F5_a_present_drive_keeps_its_unit(self):
|
||||||
|
self.refused(self.host(links=[self.UUID_LINK]), "F5")
|
||||||
|
|
||||||
|
def test_F4_a_mounted_drive_keeps_its_unit(self):
|
||||||
|
self.refused(self.host(mounted=["/mnt/hdd_1"]), "F4")
|
||||||
|
|
||||||
|
def test_F3_an_enabled_unit_is_not_removed(self):
|
||||||
|
self.refused(self.host(links=["/etc/systemd/system/multi-user.target.wants/" + self.NAME]), "F3")
|
||||||
|
|
||||||
|
def test_F2_a_network_unit_or_a_foreign_unit_is_not_removed(self):
|
||||||
|
self.refused(self.host(text=LOCAL_UNIT.replace("Managed by felhom-agent", "Written by someone")), "F2")
|
||||||
|
net = LOCAL_UNIT.replace("# Managed by felhom-agent", "# Managed by felhom-agent felhom network storage")
|
||||||
|
self.refused(self.host(text=net), "F2")
|
||||||
|
|
||||||
|
def test_F1_a_symlinked_unit_is_not_followed(self):
|
||||||
|
self.refused(self.host(kind="symlink"), "F1")
|
||||||
|
|
||||||
|
def test_A1_names(self):
|
||||||
|
for argv in (("unit-forget", "mnt-hdd_1.automount"), ("unit-forget", "sshd.service"),
|
||||||
|
("unit-forget", "../mnt-hdd_1.mount"), ("unit-forget",), ("unit-forget", self.NAME, "x")):
|
||||||
|
self.refused(self.host(), "A1", *argv)
|
||||||
|
|
||||||
|
def test_U3_a_unit_whose_name_lies_about_where_is_not_removed(self):
|
||||||
|
self.refused(self.host(text=LOCAL_UNIT.replace("Where=/mnt/hdd_1", "Where=/mnt/other")), "U3")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main(verbosity=2)
|
unittest.main(verbosity=2)
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Tests for configs/felhom-thin-check (R-1012): it lists the thin pool's device ids next to LVM's and NEVER removes
|
||||||
|
anything. Fake `lvs`, `dmsetup` and `thin_ls` record every call; nothing real is touched. Each refusal has a positive
|
||||||
|
control (the healthy read), so a test cannot pass because the helper refuses everything.
|
||||||
|
Run: python3 configs/test_felhom_thin_check.py (also run by internal/thincheck's Go test)
|
||||||
|
"""
|
||||||
|
import os, pathlib, re, subprocess, tempfile, unittest
|
||||||
|
|
||||||
|
HELPER = pathlib.Path(__file__).resolve().parent / "felhom-thin-check"
|
||||||
|
|
||||||
|
|
||||||
|
class Box:
|
||||||
|
def __init__(self, pool_ids=(1, 2), lvm=(("vm-9201-disk-0", 1), ("vm-9201-disk-1", 2)), attr="twi-aotz--",
|
||||||
|
held="-", reserve_rc=0, thin_ls_rc=0):
|
||||||
|
self.t = tempfile.TemporaryDirectory()
|
||||||
|
r = pathlib.Path(self.t.name)
|
||||||
|
self.log = r / "calls"
|
||||||
|
self.log.write_text("")
|
||||||
|
lvm_lines = "\\n".join(f" {n},{i}" for n, i in lvm)
|
||||||
|
pool_lines = "\\n".join(str(i) for i in pool_ids)
|
||||||
|
self.bins = {}
|
||||||
|
for name, body in {
|
||||||
|
"lvs": f'''case "$*" in *lv_attr*) echo " {attr}";; *thin_id*) printf "{lvm_lines}\\n";; esac''',
|
||||||
|
"dmsetup": f'''case "$1" in
|
||||||
|
status) echo "0 112975872 thin-pool 585 6860/262144 517080/882624 {held} rw discard_passdown queue_if_no_space - 1024";;
|
||||||
|
message) [ "$4" = reserve_metadata_snap ] && exit {reserve_rc}; exit 0;;
|
||||||
|
esac''',
|
||||||
|
"thin_ls": f'''[ {thin_ls_rc} -ne 0 ] && exit {thin_ls_rc}; printf "{pool_lines}\\n"''',
|
||||||
|
}.items():
|
||||||
|
p = r / name
|
||||||
|
p.write_text(f'#!/bin/bash\necho "{name} $*" >> "{self.log}"\n{body}\n')
|
||||||
|
p.chmod(0o755)
|
||||||
|
self.bins[name] = p
|
||||||
|
self.lock = r / "lock"
|
||||||
|
|
||||||
|
def run(self, *args):
|
||||||
|
script = (f'source "{HELPER}"; LVS="{self.bins["lvs"]}"; DMSETUP="{self.bins["dmsetup"]}"; '
|
||||||
|
f'THIN_LS="{self.bins["thin_ls"]}"; LOCK="{self.lock}"; main "$@"')
|
||||||
|
r = subprocess.run(["bash", "-c", script, "x", *args], capture_output=True, text=True)
|
||||||
|
return r.returncode, r.stdout, r.stderr
|
||||||
|
|
||||||
|
def calls(self):
|
||||||
|
return self.log.read_text().splitlines()
|
||||||
|
|
||||||
|
|
||||||
|
def messages(calls):
|
||||||
|
return [c.split()[-1] for c in calls if c.startswith("dmsetup message")]
|
||||||
|
|
||||||
|
|
||||||
|
class Reads(unittest.TestCase):
|
||||||
|
def test_healthy_pool_lists_both_sides_and_releases(self):
|
||||||
|
b = Box()
|
||||||
|
rc, out, err = b.run("pve", "data")
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertEqual(out.splitlines(), ["lvm-before 1 vm-9201-disk-0", "lvm-before 2 vm-9201-disk-1",
|
||||||
|
"pool 1", "pool 2",
|
||||||
|
"lvm-after 1 vm-9201-disk-0", "lvm-after 2 vm-9201-disk-1", "ok"])
|
||||||
|
self.assertEqual(messages(b.calls()), ["reserve_metadata_snap", "release_metadata_snap"])
|
||||||
|
self.assertIn("thin_ls --metadata-snap --no-headers -o DEV /dev/mapper/pve-data_tmeta", b.calls())
|
||||||
|
|
||||||
|
def test_the_2026_10_14_orphan_is_listed(self):
|
||||||
|
rc, out, _ = Box(pool_ids=(1, 2, 4)).run("pve", "data")
|
||||||
|
self.assertEqual(rc, 0)
|
||||||
|
self.assertIn("pool 4", out.splitlines())
|
||||||
|
|
||||||
|
def test_dashed_names_map_to_the_device_mapper_name(self):
|
||||||
|
b = Box()
|
||||||
|
rc, _, err = b.run("felhom-scratch", "thin-pool")
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertIn("dmsetup status felhom--scratch-thin--pool-tpool", b.calls())
|
||||||
|
|
||||||
|
|
||||||
|
class Refuses(unittest.TestCase):
|
||||||
|
def refused(self, box, rc_want, *args):
|
||||||
|
rc, out, err = box.run(*args)
|
||||||
|
self.assertEqual(rc, rc_want, (out, err))
|
||||||
|
self.assertNotIn("ok", out.splitlines())
|
||||||
|
return box.calls()
|
||||||
|
|
||||||
|
def test_arguments(self):
|
||||||
|
for args in (("pve",), ("pve", "data", "x"), ("pve;id", "data"), ("-pve", "data"), ("pve", "../data"),
|
||||||
|
("pve", "da ta")):
|
||||||
|
calls = self.refused(Box(), 1, *args)
|
||||||
|
self.assertEqual(calls, [], args)
|
||||||
|
|
||||||
|
def test_not_a_thin_pool(self):
|
||||||
|
calls = self.refused(Box(attr="Vwi-aotz--"), 1, "pve", "vm-9201-disk-1")
|
||||||
|
self.assertEqual(messages(calls), [])
|
||||||
|
|
||||||
|
def test_a_snapshot_someone_else_holds_is_never_touched(self):
|
||||||
|
b = Box(held="1234")
|
||||||
|
rc, _, err = b.run("pve", "data")
|
||||||
|
self.assertEqual(rc, 3)
|
||||||
|
self.assertEqual(messages(b.calls()), []) # neither reserve nor release
|
||||||
|
# the reason line ends with the reason, not the exit code (found live on demo-hp 2026-10-17)
|
||||||
|
self.assertTrue(err.strip().endswith("not touched"), err)
|
||||||
|
|
||||||
|
def test_reserve_failure_does_not_release(self):
|
||||||
|
calls = self.refused(Box(reserve_rc=1), 3, "pve", "data")
|
||||||
|
self.assertEqual(messages(calls), ["reserve_metadata_snap"])
|
||||||
|
|
||||||
|
def test_thin_ls_failure_still_releases(self):
|
||||||
|
calls = self.refused(Box(thin_ls_rc=1), 4, "pve", "data")
|
||||||
|
self.assertEqual(messages(calls), ["reserve_metadata_snap", "release_metadata_snap"])
|
||||||
|
|
||||||
|
|
||||||
|
class NeverDeletes(unittest.TestCase):
|
||||||
|
def test_the_only_pool_messages_are_reserve_and_release(self):
|
||||||
|
src = HELPER.read_text()
|
||||||
|
sent = re.findall(r'"\$DMSETUP" message "\$TPOOL" 0 (\S+)', src)
|
||||||
|
self.assertEqual(sorted(set(sent)), ["release_metadata_snap", "reserve_metadata_snap"])
|
||||||
|
for word in ("delete", "lvremove", "thin_repair", "create_thin", "create_snap", " rm "):
|
||||||
|
code = "\n".join(l for l in src.splitlines() if not l.lstrip().startswith("#"))
|
||||||
|
self.assertNotIn(word, code, word)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
@@ -1,7 +1,8 @@
|
|||||||
module gitea.dooplex.hu/admin/felhom-agent
|
module gitea.dooplex.hu/admin/felhom-agent
|
||||||
|
|
||||||
go 1.25.0
|
go 1.27.2
|
||||||
|
|
||||||
require golang.org/x/crypto v0.52.0
|
require (
|
||||||
|
golang.org/x/crypto v0.58.0
|
||||||
require golang.org/x/sys v0.45.0 // indirect
|
golang.org/x/sys v0.49.0
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
|
||||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
|
||||||
|
|||||||
@@ -39,13 +39,25 @@ type savedSuccess struct {
|
|||||||
target string
|
target string
|
||||||
vmid int
|
vmid int
|
||||||
at time.Time
|
at time.Time
|
||||||
|
// recent (R-973) holds the start times of the newest successes (at most maxRecentSuccesses, newest
|
||||||
|
// last) — what the local-API backup cadence cap counts, so an agent restart does not reset the cap.
|
||||||
|
recent []time.Time
|
||||||
|
// refusedAt (R-973) is the newest time the cap refused a backup of this guest on this tier.
|
||||||
|
refusedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// backupSuccessJSON is one entry on disk.
|
// maxRecentSuccesses bounds the per-tier success history kept for the cadence cap (R-973). The cap
|
||||||
|
// allows at most 3 runs in a window, so 8 is ample.
|
||||||
|
const maxRecentSuccesses = 8
|
||||||
|
|
||||||
|
// backupSuccessJSON is one entry on disk. Recent and LastRefusedAt are additive (R-973): a file written
|
||||||
|
// before them reads as "no history", and an older agent ignores them.
|
||||||
type backupSuccessJSON struct {
|
type backupSuccessJSON struct {
|
||||||
Target string `json:"target"`
|
Target string `json:"target"`
|
||||||
VMID int `json:"vmid"`
|
VMID int `json:"vmid"`
|
||||||
StartedAt string `json:"started_at"`
|
StartedAt string `json:"started_at"`
|
||||||
|
Recent []string `json:"recent,omitempty"`
|
||||||
|
LastRefusedAt string `json:"last_refused_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func backupStateKey(target string, vmid int) string { return target + "/" + strconv.Itoa(vmid) }
|
func backupStateKey(target string, vmid int) string { return target + "/" + strconv.Itoa(vmid) }
|
||||||
@@ -67,7 +79,19 @@ func NewBackupSuccessState(path string) *BackupSuccessState {
|
|||||||
if perr != nil {
|
if perr != nil {
|
||||||
continue // one unreadable entry must not lose the others
|
continue // one unreadable entry must not lose the others
|
||||||
}
|
}
|
||||||
s.last[backupStateKey(e.Target, e.VMID)] = savedSuccess{target: e.Target, vmid: e.VMID, at: t.UTC()}
|
ss := savedSuccess{target: e.Target, vmid: e.VMID, at: t.UTC()}
|
||||||
|
for _, r := range e.Recent {
|
||||||
|
if rt, rerr := time.Parse(time.RFC3339, r); rerr == nil {
|
||||||
|
ss.recent = append(ss.recent, rt.UTC())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ss.recent) == 0 {
|
||||||
|
ss.recent = []time.Time{ss.at} // a pre-R-973 file: its one known success counts
|
||||||
|
}
|
||||||
|
if rt, rerr := time.Parse(time.RFC3339, e.LastRefusedAt); rerr == nil {
|
||||||
|
ss.refusedAt = rt.UTC()
|
||||||
|
}
|
||||||
|
s.last[backupStateKey(e.Target, e.VMID)] = ss
|
||||||
}
|
}
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
@@ -85,13 +109,72 @@ func (s *BackupSuccessState) RecordBackupSuccess(target string, b hub.Backup) er
|
|||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
k := backupStateKey(target, b.VMID)
|
k := backupStateKey(target, b.VMID)
|
||||||
if old, ok := s.last[k]; ok && !t.After(old.at) {
|
old, had := s.last[k]
|
||||||
|
if had && !t.After(old.at) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
s.last[k] = savedSuccess{target: target, vmid: b.VMID, at: t.UTC()}
|
ss := savedSuccess{target: target, vmid: b.VMID, at: t.UTC(), refusedAt: old.refusedAt}
|
||||||
|
ss.recent = append(append([]time.Time(nil), old.recent...), t.UTC())
|
||||||
|
if len(ss.recent) > maxRecentSuccesses {
|
||||||
|
ss.recent = ss.recent[len(ss.recent)-maxRecentSuccesses:]
|
||||||
|
}
|
||||||
|
s.last[k] = ss
|
||||||
return s.saveLocked()
|
return s.saveLocked()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RecentBackupSuccesses returns the saved success start times for this tier and guest that are not
|
||||||
|
// older than since (R-973 — the cadence cap's restart-proof count).
|
||||||
|
func (s *BackupSuccessState) RecentBackupSuccesses(target string, vmid int, since time.Time) []time.Time {
|
||||||
|
if s == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
var out []time.Time
|
||||||
|
for _, t := range s.last[backupStateKey(target, vmid)].recent {
|
||||||
|
if !t.Before(since) {
|
||||||
|
out = append(out, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecordBackupRefusal saves that the cadence cap refused a backup of this guest on this tier at at
|
||||||
|
// (R-973), so the operator signal it raises survives an agent restart.
|
||||||
|
func (s *BackupSuccessState) RecordBackupRefusal(target string, vmid int, at time.Time) error {
|
||||||
|
if s == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
k := backupStateKey(target, vmid)
|
||||||
|
ss, ok := s.last[k]
|
||||||
|
if !ok {
|
||||||
|
return nil // nothing to cap against without a success; the cap never refuses then
|
||||||
|
}
|
||||||
|
if !at.After(ss.refusedAt) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ss.refusedAt = at.UTC()
|
||||||
|
s.last[k] = ss
|
||||||
|
return s.saveLocked()
|
||||||
|
}
|
||||||
|
|
||||||
|
// LastBackupRefusal returns the newest saved cadence-cap refusal across every tier and guest.
|
||||||
|
func (s *BackupSuccessState) LastBackupRefusal() (target string, vmid int, at time.Time, ok bool) {
|
||||||
|
if s == nil {
|
||||||
|
return "", 0, time.Time{}, false
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
for _, e := range s.last {
|
||||||
|
if !e.refusedAt.IsZero() && e.refusedAt.After(at) {
|
||||||
|
target, vmid, at, ok = e.target, e.vmid, e.refusedAt, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return target, vmid, at, ok
|
||||||
|
}
|
||||||
|
|
||||||
// LastKnownSuccess returns the newest saved success for this tier and guest (ok=false = none on file).
|
// LastKnownSuccess returns the newest saved success for this tier and guest (ok=false = none on file).
|
||||||
func (s *BackupSuccessState) LastKnownSuccess(target string, vmid int) (time.Time, bool) {
|
func (s *BackupSuccessState) LastKnownSuccess(target string, vmid int) (time.Time, bool) {
|
||||||
if s == nil {
|
if s == nil {
|
||||||
@@ -129,7 +212,14 @@ func (s *BackupSuccessState) KnownBackupSuccesses() []hub.Backup {
|
|||||||
func (s *BackupSuccessState) saveLocked() error {
|
func (s *BackupSuccessState) saveLocked() error {
|
||||||
entries := make([]backupSuccessJSON, 0, len(s.last))
|
entries := make([]backupSuccessJSON, 0, len(s.last))
|
||||||
for _, e := range s.last {
|
for _, e := range s.last {
|
||||||
entries = append(entries, backupSuccessJSON{Target: e.target, VMID: e.vmid, StartedAt: e.at.Format(time.RFC3339)})
|
j := backupSuccessJSON{Target: e.target, VMID: e.vmid, StartedAt: e.at.Format(time.RFC3339)}
|
||||||
|
for _, r := range e.recent {
|
||||||
|
j.Recent = append(j.Recent, r.Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
if !e.refusedAt.IsZero() {
|
||||||
|
j.LastRefusedAt = e.refusedAt.Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
entries = append(entries, j)
|
||||||
}
|
}
|
||||||
// Deterministic file content (Go's map order is random).
|
// Deterministic file content (Go's map order is random).
|
||||||
sort.Slice(entries, func(i, j int) bool {
|
sort.Slice(entries, func(i, j int) bool {
|
||||||
|
|||||||
@@ -74,3 +74,40 @@ func TestBackupSuccessState_KnownBackupSuccessesSurviveAReopen(t *testing.T) {
|
|||||||
t.Fatal("uncovered_volumes must marshal as [], not null")
|
t.Fatal("uncovered_volumes must marshal as [], not null")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// R-973: the cadence cap's history and refusals survive a reopen; the history is bounded; a file
|
||||||
|
// written before R-973 counts its one known success (never "no history" for a tier that has a copy).
|
||||||
|
func TestBackupSuccessState_R973_RecentHistoryAndRefusalSurviveAReopen(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "backup-success-state.json")
|
||||||
|
s := NewBackupSuccessState(path)
|
||||||
|
base := time.Date(2026, 10, 18, 2, 30, 0, 0, time.UTC)
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
at := base.Add(time.Duration(i) * time.Hour)
|
||||||
|
if err := s.RecordBackupSuccess("local", hub.Backup{VMID: 9201, Success: true, StartedAt: at.Format(time.RFC3339)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
refused := base.Add(11 * time.Hour)
|
||||||
|
if err := s.RecordBackupRefusal("local", 9201, refused); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
r := NewBackupSuccessState(path)
|
||||||
|
all := r.RecentBackupSuccesses("local", 9201, time.Time{})
|
||||||
|
if len(all) != maxRecentSuccesses || !all[len(all)-1].Equal(base.Add(9*time.Hour)) {
|
||||||
|
t.Fatalf("recent history after a reopen: %v (want the newest %d, ending at +9h)", all, maxRecentSuccesses)
|
||||||
|
}
|
||||||
|
if got := r.RecentBackupSuccesses("local", 9201, base.Add(7*time.Hour)); len(got) != 3 {
|
||||||
|
t.Fatalf("successes since +7h: %d, want 3", len(got))
|
||||||
|
}
|
||||||
|
if tg, vm, at, ok := r.LastBackupRefusal(); !ok || tg != "local" || vm != 9201 || !at.Equal(refused) {
|
||||||
|
t.Fatalf("refusal after a reopen: %q %d %v %v", tg, vm, at, ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
old := filepath.Join(t.TempDir(), "old.json")
|
||||||
|
if err := os.WriteFile(old, []byte(`[{"target":"felhom-pbs","vmid":9201,"started_at":"2026-10-17T03:00:00Z"}]`), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := NewBackupSuccessState(old).RecentBackupSuccesses("felhom-pbs", 9201, time.Time{}); len(got) != 1 {
|
||||||
|
t.Fatalf("a pre-R-973 file must count its one success: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package backup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
|
)
|
||||||
|
|
||||||
|
// R-1012: a snapshot backup that fails carries the orphan finding in its error — the text the controller mails to
|
||||||
|
// the operator (`whole_guest_backup_failed`). The check runs ONCE, after the failure — never before the run.
|
||||||
|
// RED-PROOF (P1/redproof-thincheck.txt): with explainFailure's append removed this test fails. RED-PROOF (2026-10-17
|
||||||
|
// night, felhom-agent REPORT.md): with the v0.159.0 pre-run check restored, calls = 2 and this test fails.
|
||||||
|
func TestBackup_SnapshotFailure_NamesTheOrphan(t *testing.T) {
|
||||||
|
const why = "probable cause: thin pool pve/data holds device 4 that LVM does not know"
|
||||||
|
for name, api := range map[string]*fakeBackupAPI{
|
||||||
|
"task fails": {vzdumpUPID: "UPID:x", waitErr: errors.New("task failed: job errors")},
|
||||||
|
"start fails": {vzdumpErr: errors.New("vzdump: 500")},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
r := NewBackupRunner(api, "local", "", "", "", quiet())
|
||||||
|
calls := 0
|
||||||
|
r.SetThinCheck(func(context.Context, int) string { calls++; return why })
|
||||||
|
rec, err := r.Backup(context.Background(), 9201)
|
||||||
|
if err == nil || rec.Success {
|
||||||
|
t.Fatalf("want a failure, got %+v", rec)
|
||||||
|
}
|
||||||
|
if !strings.Contains(rec.Error, why) {
|
||||||
|
t.Fatalf("the failure does not name the orphan: %q", rec.Error)
|
||||||
|
}
|
||||||
|
if calls != 1 {
|
||||||
|
t.Fatalf("thin check ran %d times, want 1 (after the failure only)", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A successful backup never runs the check (it takes a metadata snapshot of the live pool, and nobody needs the answer)
|
||||||
|
// and keeps a clean record; a stop-mode backup takes no snapshot, so the check never runs; an unwired runner behaves
|
||||||
|
// exactly as before. RED-PROOF (2026-10-17 night): with the v0.159.0 pre-run check restored, the success leg fails
|
||||||
|
// with calls=1.
|
||||||
|
func TestBackup_ThinCheck_SilentWhenNotApplicable(t *testing.T) {
|
||||||
|
ok := &fakeBackupAPI{vzdumpUPID: "UPID:x",
|
||||||
|
content: []proxmox.StorageContent{{VolID: "local:backup/vzdump-lxc-9201-x", Content: "backup", VMID: 9201, Size: 100, CTime: 1}}}
|
||||||
|
r := NewBackupRunner(ok, "local", "", "", "", quiet())
|
||||||
|
calls := 0
|
||||||
|
r.SetThinCheck(func(context.Context, int) string { calls++; return "probable cause: x" })
|
||||||
|
rec, err := r.Backup(context.Background(), 9201)
|
||||||
|
if err != nil || !rec.Success || rec.Error != "" || calls != 0 {
|
||||||
|
t.Fatalf("success: err=%v rec=%+v calls=%d (want no check on a successful backup, clean record)", err, rec, calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
stop := NewBackupRunner(&fakeBackupAPI{vzdumpUPID: "UPID:x", waitErr: errors.New("boom")}, "local", proxmox.ModeStop, "", "", quiet())
|
||||||
|
calls = 0
|
||||||
|
stop.SetThinCheck(func(context.Context, int) string { calls++; return "probable cause: x" })
|
||||||
|
rec, _ = stop.Backup(context.Background(), 9201)
|
||||||
|
if calls != 0 || strings.Contains(rec.Error, "probable cause") {
|
||||||
|
t.Fatalf("stop mode: calls=%d error=%q", calls, rec.Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
plain := NewBackupRunner(&fakeBackupAPI{vzdumpUPID: "UPID:x", waitErr: errors.New("boom")}, "local", "", "", "", quiet())
|
||||||
|
if rec, _ := plain.Backup(context.Background(), 9201); rec.Error != "boom" {
|
||||||
|
t.Fatalf("unwired: error = %q, want the task error unchanged", rec.Error)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -61,8 +61,13 @@ type BackupRunner struct {
|
|||||||
// beforeBackup runs just before the space preflight (R-1006): it removes a dead half-written vzdump of this guest on
|
// beforeBackup runs just before the space preflight (R-1006): it removes a dead half-written vzdump of this guest on
|
||||||
// this target, so a leftover from a power cut cannot hold the space this backup needs. nil = nothing.
|
// this target, so a leftover from a power cut cannot hold the space this backup needs. nil = nothing.
|
||||||
beforeBackup func(ctx context.Context, storage string, vmid int)
|
beforeBackup func(ctx context.Context, storage string, vmid int)
|
||||||
logger *slog.Logger
|
// thinCheck (R-1012) names an orphan thin device in the guest's pools in plain words ("" = none / not checked). It
|
||||||
now func() time.Time
|
// runs ONLY when a snapshot-mode vzdump has failed (start or task), so the failure the controller mails to the
|
||||||
|
// operator says why; a successful backup never runs it (it takes a metadata snapshot of the live pool). It never
|
||||||
|
// removes anything. nil = not wired.
|
||||||
|
thinCheck func(ctx context.Context, vmid int) string
|
||||||
|
logger *slog.Logger
|
||||||
|
now func() time.Time
|
||||||
// rejected remembers the volids already announced by warnRejectedArchiveOnce, so an incomplete
|
// rejected remembers the volids already announced by warnRejectedArchiveOnce, so an incomplete
|
||||||
// archive is reported ONCE rather than on every 5-minute due-check. Bounded in practice: one
|
// archive is reported ONCE rather than on every 5-minute due-check. Bounded in practice: one
|
||||||
// entry per aborted upload, and a process restart clears it. Guarded by rejectedMu because the
|
// entry per aborted upload, and a process restart clears it. Guarded by rejectedMu because the
|
||||||
@@ -110,6 +115,22 @@ func (r *BackupRunner) SetBeforeBackup(fn func(ctx context.Context, storage stri
|
|||||||
r.beforeBackup = fn
|
r.beforeBackup = fn
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetThinCheck wires the R-1012 orphan check (see BackupRunner.thinCheck).
|
||||||
|
func (r *BackupRunner) SetThinCheck(fn func(ctx context.Context, vmid int) string) { r.thinCheck = fn }
|
||||||
|
|
||||||
|
// explainFailure appends the orphan finding to a failed snapshot-mode backup's error — the only place the check runs, so
|
||||||
|
// it sees the pool as the failure left it. Bounded, so a stuck helper cannot hold the job.
|
||||||
|
func (r *BackupRunner) explainFailure(ctx context.Context, vmid int, rec *hub.Backup) {
|
||||||
|
if r.thinCheck == nil || r.mode != proxmox.ModeSnapshot {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if why := r.thinCheck(cctx, vmid); why != "" {
|
||||||
|
rec.Error = rec.Error + " — " + why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// localPruneSpec returns the `--prune-backups` spec to apply to THIS backup, or "" to skip pruning. It
|
// localPruneSpec returns the `--prune-backups` spec to apply to THIS backup, or "" to skip pruning. It
|
||||||
// applies the configured retention ONLY when the target is a non-PBS storage — PBS offsite retention is a
|
// applies the configured retention ONLY when the target is a non-PBS storage — PBS offsite retention is a
|
||||||
// separate lifecycle and must never be pruned by the per-run flag (§9). Fail-safe: if the target's type
|
// separate lifecycle and must never be pruned by the per-run flag (§9). Fail-safe: if the target's type
|
||||||
@@ -201,12 +222,15 @@ func (r *BackupRunner) backup(ctx context.Context, vmid int, onSnapshot func())
|
|||||||
return rec, fmt.Errorf("backup: %s", rec.Error)
|
return rec, fmt.Errorf("backup: %s", rec.Error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// R-1012: no orphan check here. It takes a metadata snapshot of the live pool, and its answer is only needed when the
|
||||||
|
// snapshot step has failed — explainFailure runs it then (pinned by TestBackup_ThinCheck_SilentWhenNotApplicable).
|
||||||
upid, err := r.api.Vzdump(ctx, proxmox.VzdumpOptions{
|
upid, err := r.api.Vzdump(ctx, proxmox.VzdumpOptions{
|
||||||
VMID: vmid, Storage: r.target, Mode: r.mode, Notes: r.notes,
|
VMID: vmid, Storage: r.target, Mode: r.mode, Notes: r.notes,
|
||||||
PruneBackups: r.localPruneSpec(ctx), // local target → keep-last=N; PBS/unknown → "" (no prune)
|
PruneBackups: r.localPruneSpec(ctx), // local target → keep-last=N; PBS/unknown → "" (no prune)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rec.Error = err.Error()
|
rec.Error = err.Error()
|
||||||
|
r.explainFailure(ctx, vmid, &rec)
|
||||||
rec.DurationSeconds = time.Since(start).Seconds()
|
rec.DurationSeconds = time.Since(start).Seconds()
|
||||||
return rec, fmt.Errorf("backup: vzdump vmid %d: %w", vmid, err)
|
return rec, fmt.Errorf("backup: vzdump vmid %d: %w", vmid, err)
|
||||||
}
|
}
|
||||||
@@ -220,6 +244,7 @@ func (r *BackupRunner) backup(ctx context.Context, vmid int, onSnapshot func())
|
|||||||
}
|
}
|
||||||
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: r.waitTimeout}); err != nil {
|
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: r.waitTimeout}); err != nil {
|
||||||
rec.Error = err.Error()
|
rec.Error = err.Error()
|
||||||
|
r.explainFailure(ctx, vmid, &rec)
|
||||||
rec.DurationSeconds = time.Since(start).Seconds()
|
rec.DurationSeconds = time.Since(start).Seconds()
|
||||||
return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err)
|
return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -91,9 +91,11 @@ var manifest = []Capability{
|
|||||||
{"disk-mkfs-xfs", "guarded format (xfs)", "/usr/local/sbin/felhom-mkfs-guarded", []string{"/dev/sda", "xfs"}, false, ""},
|
{"disk-mkfs-xfs", "guarded format (xfs)", "/usr/local/sbin/felhom-mkfs-guarded", []string{"/dev/sda", "xfs"}, false, ""},
|
||||||
{"disk-smart", "disk SMART health read", "/usr/sbin/smartctl", []string{"-a", "-j", "/dev/sda"}, false, ""},
|
{"disk-smart", "disk SMART health read", "/usr/sbin/smartctl", []string{"-a", "-j", "/dev/sda"}, false, ""},
|
||||||
{"disk-lvs", "thin-pool usage read", "/usr/sbin/lvs", []string{"--reportformat", "json", "--units", "b", "-o", "lv_name,data_percent,metadata_percent", "--", "pve/data"}, false, ""},
|
{"disk-lvs", "thin-pool usage read", "/usr/sbin/lvs", []string{"--reportformat", "json", "--units", "b", "-o", "lv_name,data_percent,metadata_percent", "--", "pve/data"}, false, ""},
|
||||||
|
{"thin-orphan-check", "orphan thin device check after a failed whole-guest backup (read-only, R-1012)", "/usr/local/sbin/felhom-thin-check", []string{"pve", "data"}, false, ""},
|
||||||
|
|
||||||
// ---- Storage mount units (watchdog re-mount) ----
|
// ---- Storage mount units (watchdog re-mount) ----
|
||||||
{"mount-unit-install", "fs-UUID mount unit install (root content check, R-861)", "/usr/local/sbin/felhom-priv-apply", []string{"unit", "mnt-felhom\\x2dx.mount"}, false, ""},
|
{"mount-unit-install", "fs-UUID mount unit install (root content check, R-861)", "/usr/local/sbin/felhom-priv-apply", []string{"unit", "mnt-felhom\\x2dx.mount"}, false, ""},
|
||||||
|
{"mount-unit-forget", "decommissioned-drive unit removal (root check, R-1124)", "/usr/local/sbin/felhom-priv-apply", []string{"unit-forget", "mnt-felhom\\x2dx.mount"}, false, ""},
|
||||||
{"mount-daemon-reload", "systemd reload after unit write", "/usr/bin/systemctl", []string{"daemon-reload"}, false, ""},
|
{"mount-daemon-reload", "systemd reload after unit write", "/usr/bin/systemctl", []string{"daemon-reload"}, false, ""},
|
||||||
{"mount-unit-enable", "mount unit enable", "/usr/bin/systemctl", []string{"enable", "--now", "--", "mnt-felhom\\x2dx.mount"}, false, ""},
|
{"mount-unit-enable", "mount unit enable", "/usr/bin/systemctl", []string{"enable", "--now", "--", "mnt-felhom\\x2dx.mount"}, false, ""},
|
||||||
{"mount-unit-disable", "mount unit disable", "/usr/bin/systemctl", []string{"disable", "--", "mnt-felhom\\x2dx.mount"}, false, ""},
|
{"mount-unit-disable", "mount unit disable", "/usr/bin/systemctl", []string{"disable", "--", "mnt-felhom\\x2dx.mount"}, false, ""},
|
||||||
@@ -118,6 +120,8 @@ var manifest = []Capability{
|
|||||||
{"guest-shutdown", "household power-off: guest shutdown (R-998)", "/usr/sbin/pct", []string{"shutdown", "9201", "--timeout", "120"}, false, ""},
|
{"guest-shutdown", "household power-off: guest shutdown (R-998)", "/usr/sbin/pct", []string{"shutdown", "9201", "--timeout", "120"}, false, ""},
|
||||||
{"host-poweroff", "household power-off: host power-off (R-998)", "/usr/bin/systemctl", []string{"poweroff"}, false, ""},
|
{"host-poweroff", "household power-off: host power-off (R-998)", "/usr/bin/systemctl", []string{"poweroff"}, false, ""},
|
||||||
{"vzdump-partial-clean", "dead half-written vzdump removal (R-1006)", "/usr/local/sbin/felhom-vzdump-partial-clean", []string{"local", "9201", "vzdump-lxc-9201-2026_10_14-04_42_18.tar.dat"}, false, ""},
|
{"vzdump-partial-clean", "dead half-written vzdump removal (R-1006)", "/usr/local/sbin/felhom-vzdump-partial-clean", []string{"local", "9201", "vzdump-lxc-9201-2026_10_14-04_42_18.tar.dat"}, false, ""},
|
||||||
|
{"internal-place-plan", "internal data place: the size plan (16 §7)", "/usr/local/sbin/felhom-internal-place", []string{"plan", "0"}, false, ""},
|
||||||
|
{"internal-place-create", "internal data place: create on an old box (signed add_internal_place)", "/usr/local/sbin/felhom-internal-place", []string{"create", "0"}, false, ""},
|
||||||
|
|
||||||
// ---- LAN split-horizon resolver (dnsmasq) ----
|
// ---- LAN split-horizon resolver (dnsmasq) ----
|
||||||
{"dnsmasq-install", "dnsmasq package install", "/usr/bin/apt-get", []string{"install", "-y", "-q", "dnsmasq"}, false, ""},
|
{"dnsmasq-install", "dnsmasq package install", "/usr/bin/apt-get", []string{"install", "-y", "-q", "dnsmasq"}, false, ""},
|
||||||
|
|||||||
@@ -72,6 +72,11 @@ var r861Injections = []string{
|
|||||||
"/usr/local/sbin/felhom-vzdump-partial-clean local;id 9201 vzdump-lxc-9201-2026_10_14-04_42_18.tar.dat",
|
"/usr/local/sbin/felhom-vzdump-partial-clean local;id 9201 vzdump-lxc-9201-2026_10_14-04_42_18.tar.dat",
|
||||||
"/usr/local/sbin/felhom-vzdump-partial-clean local 9201",
|
"/usr/local/sbin/felhom-vzdump-partial-clean local 9201",
|
||||||
"/usr/local/sbin/felhom-vzdump-partial-clean --help",
|
"/usr/local/sbin/felhom-vzdump-partial-clean --help",
|
||||||
|
"/usr/local/sbin/felhom-internal-place create 132",
|
||||||
|
"/usr/local/sbin/felhom-internal-place create 0 extra",
|
||||||
|
"/usr/local/sbin/felhom-internal-place create 0;id",
|
||||||
|
"/usr/local/sbin/felhom-internal-place wipe 0",
|
||||||
|
"/usr/local/sbin/felhom-internal-place create",
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSudoersRefusesTheR861Injections(t *testing.T) {
|
func TestSudoersRefusesTheR861Injections(t *testing.T) {
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
package escrow
|
package escrow
|
||||||
|
|
||||||
|
import "encoding/json"
|
||||||
|
|
||||||
// Controller-driven ceremony contract (v0.88.0, TASK 2026-07-13; mechanics validated by
|
// Controller-driven ceremony contract (v0.88.0, TASK 2026-07-13; mechanics validated by
|
||||||
// felhom.eu/documentation/audits/SPIKE-controller-escrow-2026-07-13.md). The agent's local API
|
// felhom.eu/documentation/audits/SPIKE-controller-escrow-2026-07-13.md). The agent's local API
|
||||||
// re-invokes the agent binary as root via `sudo -n` with ONE fixed argument vector; sudoers
|
// re-invokes the agent binary as root via `sudo -n` with ONE fixed argument vector; sudoers
|
||||||
@@ -43,7 +45,15 @@ type CeremonyOutput struct {
|
|||||||
IdentityBlobBytes int `json:"identity_blob_bytes"`
|
IdentityBlobBytes int `json:"identity_blob_bytes"`
|
||||||
ResticPwSealed bool `json:"restic_pw_sealed"`
|
ResticPwSealed bool `json:"restic_pw_sealed"`
|
||||||
Uploaded bool `json:"uploaded"`
|
Uploaded bool `json:"uploaded"`
|
||||||
|
// PendingUpload (version 2, R-978) is the hub upload body — the R-wrapped blobs (ciphertext), the key
|
||||||
|
// fingerprint and the sealed password's hash — that json mode hands back INSTEAD of uploading. The agent
|
||||||
|
// daemon uploads it only when the customer claims the new recovery code, so a ceremony that is started
|
||||||
|
// and abandoned never replaces the package the customer's previous code opens. Not a secret without R,
|
||||||
|
// but never logged. Uploaded is false whenever this is set.
|
||||||
|
PendingUpload json.RawMessage `json:"pending_upload,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CeremonyOutputVersion is the current CeremonyOutput.Version value.
|
// CeremonyOutputVersion is the current CeremonyOutput.Version value. 2 (R-978): json mode no longer
|
||||||
const CeremonyOutputVersion = 1
|
// uploads; it returns PendingUpload. The daemon refuses any other version (a binary/daemon mismatch
|
||||||
|
// fails the ceremony instead of losing a package).
|
||||||
|
const CeremonyOutputVersion = 2
|
||||||
|
|||||||
@@ -0,0 +1,146 @@
|
|||||||
|
package localapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The whole-guest backup cadence cap (R-973, security review 2026-10-12).
|
||||||
|
//
|
||||||
|
// POST /backup is reachable from inside the guest, and every successful run prunes the tier
|
||||||
|
// (`--prune-backups keep-last=N` on a local tier; ep0's server-side keep-last 2 on the off-site tier). So
|
||||||
|
// a compromised controller calling it in a loop pushed every pre-compromise whole-guest restore point out
|
||||||
|
// within minutes — the threat `07` §8 row 10 counts the off-site tier as the answer to. The agent now
|
||||||
|
// refuses a run that the schedule in `07` §6 would never ask for:
|
||||||
|
//
|
||||||
|
// - a tier whose cadence is at most 24 h (the local tier): at most 3 successful runs in any rolling
|
||||||
|
// 24 h. The schedule is one night run; a „Mentés most" press makes a LOCAL copy and, since R-899, the
|
||||||
|
// night still takes its own after it, which can land inside the same 24 h as the previous night (the
|
||||||
|
// window opens anywhere in [W+2h, W+6h)). So a normal day needs at most 3: two nights plus one press.
|
||||||
|
// A second press inside 24 h is refused — the margin is spent on the night, which must never be refused
|
||||||
|
// because of the household's presses.
|
||||||
|
// - a longer tier (the 7-day off-site tier): at most 1 successful run per max(cadence − 24 h, cadence/2)
|
||||||
|
// — 6 days for 7 days. The controller asks /backup/due first and runs only when the newest copy is older
|
||||||
|
// than the cadence, and the button never targets this tier (`07` §6, controller v0.301.0), so a
|
||||||
|
// legitimate run never comes sooner; 24 h of slack absorbs clock and archive-time differences.
|
||||||
|
//
|
||||||
|
// Only SUCCESSES count — a failed run prunes nothing and must stay retryable. The count survives an agent
|
||||||
|
// restart: it reads the success history saved on disk (LastKnownBackupStore, the R-894 file) as well as
|
||||||
|
// this process's own record. A refusal answers 429, is logged at WARN, is saved, and raises the operator
|
||||||
|
// signal BackupCapRefusal feeds (a critical capability in the host report → the hub's existing
|
||||||
|
// `agent_capability_degraded` event, operator-only). Pinned by TestBackupCap_* (backup_cap_test.go).
|
||||||
|
|
||||||
|
// backupCapRefusalHold is how long one refusal keeps the operator signal raised.
|
||||||
|
const backupCapRefusalHold = 24 * time.Hour
|
||||||
|
|
||||||
|
// backupCadenceCap returns the rolling window and the most successful runs allowed in it for a tier.
|
||||||
|
func backupCadenceCap(cadence time.Duration) (window time.Duration, max int) {
|
||||||
|
if cadence <= 0 {
|
||||||
|
cadence = defaultBackupCadence
|
||||||
|
}
|
||||||
|
if cadence <= 24*time.Hour {
|
||||||
|
return 24 * time.Hour, 3
|
||||||
|
}
|
||||||
|
w := cadence - 24*time.Hour
|
||||||
|
if w < cadence/2 {
|
||||||
|
w = cadence / 2
|
||||||
|
}
|
||||||
|
return w, 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// recentCapRuns returns the start times of the successful runs on this tier for this guest inside the
|
||||||
|
// window ending now — this process's record and the saved one, each counted once. Caller holds no lock.
|
||||||
|
func (s *Server) recentCapRuns(key backupJobKey, since time.Time) []time.Time {
|
||||||
|
seen := map[int64]bool{}
|
||||||
|
var out []time.Time
|
||||||
|
add := func(t time.Time) {
|
||||||
|
if t.Before(since) || seen[t.Unix()] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seen[t.Unix()] = true
|
||||||
|
out = append(out, t)
|
||||||
|
}
|
||||||
|
s.capMu.Lock()
|
||||||
|
for _, t := range s.capRuns[key] {
|
||||||
|
add(t)
|
||||||
|
}
|
||||||
|
s.capMu.Unlock()
|
||||||
|
if s.lastKnown != nil {
|
||||||
|
for _, t := range s.lastKnown.RecentBackupSuccesses(key.target, key.vmid, since) {
|
||||||
|
add(t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Before(out[j]) })
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordCapRun notes a successful run for the cap (the saved half is written by RecordBackupSuccess).
|
||||||
|
func (s *Server) recordCapRun(key backupJobKey, b hub.Backup) {
|
||||||
|
t, err := time.Parse(time.RFC3339, b.StartedAt)
|
||||||
|
if err != nil {
|
||||||
|
t = s.now()
|
||||||
|
}
|
||||||
|
s.capMu.Lock()
|
||||||
|
defer s.capMu.Unlock()
|
||||||
|
if s.capRuns == nil {
|
||||||
|
s.capRuns = map[backupJobKey][]time.Time{}
|
||||||
|
}
|
||||||
|
runs := append(s.capRuns[key], t.UTC())
|
||||||
|
if len(runs) > 8 {
|
||||||
|
runs = runs[len(runs)-8:]
|
||||||
|
}
|
||||||
|
s.capRuns[key] = runs
|
||||||
|
}
|
||||||
|
|
||||||
|
// capRefuses applies the cap to a new run request. It writes the 429 and returns true when refused.
|
||||||
|
func (s *Server) capRefuses(w http.ResponseWriter, key backupJobKey, tier BackupTier) bool {
|
||||||
|
window, max := backupCadenceCap(tier.Cadence)
|
||||||
|
now := s.now()
|
||||||
|
runs := s.recentCapRuns(key, now.Add(-window))
|
||||||
|
if len(runs) < max {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
next := runs[len(runs)-max].Add(window)
|
||||||
|
s.capMu.Lock()
|
||||||
|
s.capRefusal = capRefusal{target: key.target, vmid: key.vmid, at: now.UTC()}
|
||||||
|
s.capMu.Unlock()
|
||||||
|
if s.lastKnown != nil {
|
||||||
|
if err := s.lastKnown.RecordBackupRefusal(key.target, key.vmid, now); err != nil {
|
||||||
|
s.logger.Warn("local-api: could not save the backup refusal on disk (R-973)", "vmid", key.vmid, "target", key.target, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.logger.Warn("local-api: backup REFUSED — more whole-guest backups than the schedule allows (R-973 cadence cap; check the guest)",
|
||||||
|
"vmid", key.vmid, "target", key.target, "runs_in_window", len(runs), "allowed", max,
|
||||||
|
"window_h", int(window.Hours()), "next_allowed", next.UTC().Format(time.RFC3339))
|
||||||
|
writeStatus(w, http.StatusTooManyRequests, false, nil, fmt.Sprintf(
|
||||||
|
"too many whole-guest backups on target %q: %d successful in the last %d h, the schedule allows %d — refused (the next one is allowed after %s)",
|
||||||
|
key.target, len(runs), int(window.Hours()), max, next.UTC().Format(time.RFC3339)))
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
type capRefusal struct {
|
||||||
|
target string
|
||||||
|
vmid int
|
||||||
|
at time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackupCapRefusal reports the newest cadence-cap refusal still inside the 24 h hold (this process's or
|
||||||
|
// the saved one) — the source of the operator signal in the host report (R-973). ok=false = none.
|
||||||
|
func (s *Server) BackupCapRefusal() (target string, vmid int, at time.Time, ok bool) {
|
||||||
|
s.capMu.Lock()
|
||||||
|
r := s.capRefusal
|
||||||
|
s.capMu.Unlock()
|
||||||
|
if s.lastKnown != nil {
|
||||||
|
if t, v, a, found := s.lastKnown.LastBackupRefusal(); found && a.After(r.at) {
|
||||||
|
r = capRefusal{target: t, vmid: v, at: a}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if r.at.IsZero() || s.now().Sub(r.at) > backupCapRefusalHold {
|
||||||
|
return "", 0, time.Time{}, false
|
||||||
|
}
|
||||||
|
return r.target, r.vmid, r.at, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,293 @@
|
|||||||
|
package localapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||||
|
)
|
||||||
|
|
||||||
|
// R-973 — the whole-guest backup cadence cap. A compromised controller calling POST /backup in a loop
|
||||||
|
// pushed out every pre-compromise restore point: each success prunes the tier. These tests model the tier
|
||||||
|
// the way Proxmox and ep0 keep it (keep-last N, pruned after each success) and ask what is LEFT.
|
||||||
|
|
||||||
|
// keepLastTier is a whole-guest tier: every successful run adds an archive stamped with the clock and
|
||||||
|
// prunes to keepLast, as vzdump's --prune-backups keep-last (local) and ep0's keep-last 2 (off-site) do.
|
||||||
|
type keepLastTier struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
clock *testClock
|
||||||
|
keepLast int
|
||||||
|
archives []time.Time
|
||||||
|
fail bool
|
||||||
|
runs int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *keepLastTier) BackupWithSnapshotHook(_ context.Context, vmid int, _ func()) (hub.Backup, error) {
|
||||||
|
k.mu.Lock()
|
||||||
|
defer k.mu.Unlock()
|
||||||
|
k.runs++
|
||||||
|
now := k.clock.Now()
|
||||||
|
if k.fail {
|
||||||
|
return hub.Backup{VMID: vmid, StartedAt: now.Format(time.RFC3339)}, fmt.Errorf("vzdump failed")
|
||||||
|
}
|
||||||
|
k.archives = append(k.archives, now)
|
||||||
|
if len(k.archives) > k.keepLast {
|
||||||
|
k.archives = k.archives[len(k.archives)-k.keepLast:]
|
||||||
|
}
|
||||||
|
return hub.Backup{VMID: vmid, Success: true, Archive: "x", StartedAt: now.Format(time.RFC3339)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// oldestBefore reports whether the tier still holds an archive made before t (a pre-compromise point).
|
||||||
|
func (k *keepLastTier) oldestBefore(t time.Time) bool {
|
||||||
|
k.mu.Lock()
|
||||||
|
defer k.mu.Unlock()
|
||||||
|
return len(k.archives) > 0 && k.archives[0].Before(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *keepLastTier) count() (runs, kept int) {
|
||||||
|
k.mu.Lock()
|
||||||
|
defer k.mu.Unlock()
|
||||||
|
return k.runs, len(k.archives)
|
||||||
|
}
|
||||||
|
|
||||||
|
type testClock struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
t time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *testClock) Now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t }
|
||||||
|
func (c *testClock) Advance(d time.Duration) { c.mu.Lock(); c.t = c.t.Add(d); c.mu.Unlock() }
|
||||||
|
|
||||||
|
// capServer builds the two-tier box of `07` §6: local 24 h keep-last 3, off-site 7 d keep-last 2, the
|
||||||
|
// success history on disk at path (a second capServer on the same path is an agent restart).
|
||||||
|
func capServer(t *testing.T, path string, clock *testClock, local, offsite *keepLastTier) *Server {
|
||||||
|
t.Helper()
|
||||||
|
srv, err := NewServer(Options{
|
||||||
|
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
|
||||||
|
Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local"}, {Name: "felhom-pbs"}}},
|
||||||
|
Tokens: staticTokens{"A": 8200},
|
||||||
|
BackupTiers: []BackupTier{
|
||||||
|
{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: local},
|
||||||
|
{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: offsite},
|
||||||
|
},
|
||||||
|
LastKnownBackups: backup.NewBackupSuccessState(path),
|
||||||
|
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv.baseCtx = context.Background()
|
||||||
|
srv.now = clock.Now
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
// runBackup POSTs one backup and waits for the job to end. Returns the HTTP status of the POST.
|
||||||
|
func runBackup(t *testing.T, srv *Server, target string) int {
|
||||||
|
t.Helper()
|
||||||
|
rr := do(t, srv.Handler(), "POST", "/backup?target="+target, "A", "")
|
||||||
|
if rr.Code == http.StatusAccepted {
|
||||||
|
waitFor(t, func() bool {
|
||||||
|
srv.jobsMu.Lock()
|
||||||
|
defer srv.jobsMu.Unlock()
|
||||||
|
j := srv.jobs[backupJobKey{vmid: 8200, target: target}]
|
||||||
|
return j != nil && !backupInFlight(j.Phase)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return rr.Code
|
||||||
|
}
|
||||||
|
|
||||||
|
// The off-site tier — the one `07` §8 row 10 counts on against ransomware. Before the cap ten calls in a
|
||||||
|
// row left ep0 holding only the attacker's two copies. Now ONE runs (the tier was due), the other nine
|
||||||
|
// are refused, and the newest pre-compromise copy survives; five days of flooding later it still does.
|
||||||
|
//
|
||||||
|
// COMPANION RED-PROOF (observed 2026-10-18): make capRefuses always return false → this fails with
|
||||||
|
// "the off-site tier lost every pre-compromise copy: 10 runs" . Restored.
|
||||||
|
func TestBackupCap_OffsiteFloodKeepsAPreCompromiseCopy(t *testing.T) {
|
||||||
|
clock := &testClock{t: testNow}
|
||||||
|
local := &keepLastTier{clock: clock, keepLast: 3}
|
||||||
|
off := &keepLastTier{clock: clock, keepLast: 2}
|
||||||
|
path := filepath.Join(t.TempDir(), "backup-success-state.json")
|
||||||
|
srv := capServer(t, path, clock, local, off)
|
||||||
|
|
||||||
|
// The honest history: two weekly copies, the newest 7 days and 1 hour old (so the tier is due).
|
||||||
|
for _, age := range []time.Duration{14*24*time.Hour + time.Hour, 7*24*time.Hour + time.Hour} {
|
||||||
|
clock.t = testNow.Add(-age)
|
||||||
|
if code := runBackup(t, srv, "felhom-pbs"); code != http.StatusAccepted {
|
||||||
|
t.Fatalf("seeding the honest weekly copy: %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
clock.t = testNow
|
||||||
|
compromise := testNow
|
||||||
|
|
||||||
|
refused := 0
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
if runBackup(t, srv, "felhom-pbs") == http.StatusTooManyRequests {
|
||||||
|
refused++
|
||||||
|
}
|
||||||
|
clock.Advance(time.Minute)
|
||||||
|
}
|
||||||
|
runs, _ := off.count()
|
||||||
|
if !off.oldestBefore(compromise) {
|
||||||
|
t.Fatalf("the off-site tier lost every pre-compromise copy: %d runs", runs-2)
|
||||||
|
}
|
||||||
|
if runs-2 != 1 || refused != 9 {
|
||||||
|
t.Fatalf("flood: %d runs, %d refused — want 1 run (the tier was due) and 9 refused", runs-2, refused)
|
||||||
|
}
|
||||||
|
for d := 0; d < 5; d++ { // five more days of flooding, every hour
|
||||||
|
for h := 0; h < 24; h++ {
|
||||||
|
clock.Advance(time.Hour)
|
||||||
|
if code := runBackup(t, srv, "felhom-pbs"); code != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("day %d hour %d: an off-site run was allowed %v after the last one", d, h, clock.Now().Sub(compromise))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !off.oldestBefore(compromise) {
|
||||||
|
t.Fatal("five days of flooding pushed out the pre-compromise off-site copy")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The local tier: a flood runs exactly 3 in a day (the schedule's own day: two nights + one press) and
|
||||||
|
// every further request is refused. Honest about the limit: with keep-last 3 those 3 replace the 3 kept
|
||||||
|
// points — the cap slows a flush from minutes to a day; it cannot keep a point a 3-per-day pace prunes.
|
||||||
|
func TestBackupCap_LocalFloodRunsAtMostThreeADay(t *testing.T) {
|
||||||
|
clock := &testClock{t: testNow}
|
||||||
|
local := &keepLastTier{clock: clock, keepLast: 3}
|
||||||
|
srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2})
|
||||||
|
refused := 0
|
||||||
|
for i := 0; i < 20; i++ {
|
||||||
|
if runBackup(t, srv, "local") == http.StatusTooManyRequests {
|
||||||
|
refused++
|
||||||
|
}
|
||||||
|
clock.Advance(10 * time.Minute)
|
||||||
|
}
|
||||||
|
if runs, _ := local.count(); runs != 3 || refused != 17 {
|
||||||
|
t.Fatalf("local flood over ~3 h: %d runs, %d refused — want 3 and 17", runs, refused)
|
||||||
|
}
|
||||||
|
clock.Advance(24 * time.Hour) // a day later the window has moved on
|
||||||
|
if code := runBackup(t, srv, "local"); code != http.StatusAccepted {
|
||||||
|
t.Fatalf("a run a day later must be allowed: %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The schedule itself is never refused: a night at W+6h, a press at 10:00, the next night at W+2h (all
|
||||||
|
// three inside one rolling 24 h) run; a second press that day is refused; nights keep running after it.
|
||||||
|
func TestBackupCap_TheScheduledDayIsNeverRefused(t *testing.T) {
|
||||||
|
day := time.Date(2026, 6, 10, 0, 0, 0, 0, time.UTC)
|
||||||
|
clock := &testClock{t: day.Add(8*time.Hour + 29*time.Minute)} // W=02:30 → W+6h ≈ 08:29
|
||||||
|
local := &keepLastTier{clock: clock, keepLast: 3}
|
||||||
|
srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2})
|
||||||
|
steps := []struct {
|
||||||
|
at time.Time
|
||||||
|
want int
|
||||||
|
what string
|
||||||
|
}{
|
||||||
|
{day.Add(8*time.Hour + 29*time.Minute), http.StatusAccepted, "night 1 at W+6h"},
|
||||||
|
{day.Add(10 * time.Hour), http.StatusAccepted, "a press at 10:00"},
|
||||||
|
{day.Add(24*time.Hour + 4*time.Hour + 30*time.Minute), http.StatusAccepted, "night 2 at W+2h (20 h later)"},
|
||||||
|
{day.Add(24*time.Hour + 7*time.Hour), http.StatusTooManyRequests, "a second press inside the same 24 h"},
|
||||||
|
{day.Add(48*time.Hour + 4*time.Hour + 30*time.Minute), http.StatusAccepted, "night 3"},
|
||||||
|
}
|
||||||
|
for _, s := range steps {
|
||||||
|
clock.t = s.at
|
||||||
|
if got := runBackup(t, srv, "local"); got != s.want {
|
||||||
|
t.Fatalf("%s: got %d, want %d", s.what, got, s.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A restart must not reset the cap: the history is read back from disk.
|
||||||
|
//
|
||||||
|
// COMPANION RED-PROOF (observed 2026-10-18): make recentCapRuns ignore s.lastKnown → this fails with
|
||||||
|
// "after a restart the 4th run in a day was allowed (got 202)". Restored.
|
||||||
|
func TestBackupCap_SurvivesARestart(t *testing.T) {
|
||||||
|
clock := &testClock{t: testNow}
|
||||||
|
path := filepath.Join(t.TempDir(), "s.json")
|
||||||
|
local := &keepLastTier{clock: clock, keepLast: 3}
|
||||||
|
srv := capServer(t, path, clock, local, &keepLastTier{clock: clock, keepLast: 2})
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if code := runBackup(t, srv, "local"); code != http.StatusAccepted {
|
||||||
|
t.Fatalf("run %d: %d", i+1, code)
|
||||||
|
}
|
||||||
|
clock.Advance(time.Minute)
|
||||||
|
}
|
||||||
|
srv2 := capServer(t, path, clock, local, &keepLastTier{clock: clock, keepLast: 2}) // the restart
|
||||||
|
if code := runBackup(t, srv2, "local"); code != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("after a restart the 4th run in a day was allowed (got %d)", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failed runs prune nothing and must stay retryable: they do not count.
|
||||||
|
func TestBackupCap_FailuresDoNotCount(t *testing.T) {
|
||||||
|
clock := &testClock{t: testNow}
|
||||||
|
local := &keepLastTier{clock: clock, keepLast: 3, fail: true}
|
||||||
|
srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2})
|
||||||
|
for i := 0; i < 6; i++ {
|
||||||
|
if code := runBackup(t, srv, "local"); code != http.StatusAccepted {
|
||||||
|
t.Fatalf("failed attempt %d was refused: %d", i+1, code)
|
||||||
|
}
|
||||||
|
clock.Advance(15 * time.Minute)
|
||||||
|
}
|
||||||
|
local.mu.Lock()
|
||||||
|
local.fail = false
|
||||||
|
local.mu.Unlock()
|
||||||
|
if code := runBackup(t, srv, "local"); code != http.StatusAccepted {
|
||||||
|
t.Fatalf("a success after six failures was refused: %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A refusal raises the operator signal for 24 h, and the signal survives a restart.
|
||||||
|
func TestBackupCap_RefusalRaisesTheOperatorSignal(t *testing.T) {
|
||||||
|
clock := &testClock{t: testNow}
|
||||||
|
path := filepath.Join(t.TempDir(), "s.json")
|
||||||
|
off := &keepLastTier{clock: clock, keepLast: 2}
|
||||||
|
srv := capServer(t, path, clock, &keepLastTier{clock: clock, keepLast: 3}, off)
|
||||||
|
if _, _, _, ok := srv.BackupCapRefusal(); ok {
|
||||||
|
t.Fatal("a fresh box must not raise the signal")
|
||||||
|
}
|
||||||
|
runBackup(t, srv, "felhom-pbs")
|
||||||
|
clock.Advance(time.Hour)
|
||||||
|
if code := runBackup(t, srv, "felhom-pbs"); code != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("second off-site run an hour later: %d, want 429", code)
|
||||||
|
}
|
||||||
|
target, vmid, at, ok := srv.BackupCapRefusal()
|
||||||
|
if !ok || target != "felhom-pbs" || vmid != 8200 || !at.Equal(clock.Now()) {
|
||||||
|
t.Fatalf("signal after a refusal: ok=%v target=%q vmid=%d at=%v", ok, target, vmid, at)
|
||||||
|
}
|
||||||
|
clock.Advance(23 * time.Hour)
|
||||||
|
srv2 := capServer(t, path, clock, &keepLastTier{clock: clock, keepLast: 3}, off) // a restart
|
||||||
|
if _, _, _, ok := srv2.BackupCapRefusal(); !ok {
|
||||||
|
t.Fatal("the signal did not survive a restart")
|
||||||
|
}
|
||||||
|
clock.Advance(2 * time.Hour)
|
||||||
|
if _, _, _, ok := srv2.BackupCapRefusal(); ok {
|
||||||
|
t.Fatal("the signal must clear 24 h after the last refusal")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackupCadenceCap(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
cadence time.Duration
|
||||||
|
window time.Duration
|
||||||
|
max int
|
||||||
|
}{
|
||||||
|
{0, 24 * time.Hour, 3},
|
||||||
|
{24 * time.Hour, 24 * time.Hour, 3},
|
||||||
|
{12 * time.Hour, 24 * time.Hour, 3},
|
||||||
|
{7 * 24 * time.Hour, 6 * 24 * time.Hour, 1},
|
||||||
|
{36 * time.Hour, 18 * time.Hour, 1},
|
||||||
|
{30 * 24 * time.Hour, 29 * 24 * time.Hour, 1},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if w, m := backupCadenceCap(c.cadence); w != c.window || m != c.max {
|
||||||
|
t.Errorf("cadence %v: got %v/%d, want %v/%d", c.cadence, w, m, c.window, c.max)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,9 +24,16 @@ import (
|
|||||||
// be structurally incapable of carrying R).
|
// be structurally incapable of carrying R).
|
||||||
// - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase
|
// - One claim, then the holder is zeroed. Unclaimed past the TTL → zeroed + phase
|
||||||
// unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes").
|
// unclaimed_void ("R unclaimed → ceremony void → a re-run supersedes").
|
||||||
// - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R, which is SAFE (the blob is
|
// - Crash-safety is IN-MEMORY BY DESIGN: an agent restart loses R AND the package it opens,
|
||||||
// on the hub; a re-run supersedes it). Status reporting "none" after a restart is the honest
|
// which is SAFE (R-978: the package was never uploaded, so the hub still holds the previous one
|
||||||
// answer. No journal, deliberately — persistence is the one property R must never have.
|
// and the customer's previous code still opens it; a re-run starts over). Status reporting "none"
|
||||||
|
// after a restart is the honest answer. No journal, deliberately — persistence is the one
|
||||||
|
// property R must never have.
|
||||||
|
// - R-978: the NEW package reaches the hub only at the claim, and the code is handed out only after
|
||||||
|
// the hub took it. Until then the previous package stays current — so a ceremony that is started
|
||||||
|
// and abandoned (never claimed, superseded, TTL, restart) leaves the customer's written-down code
|
||||||
|
// working. The TTL and a new start both wait while a claim's upload is in flight, so a package is
|
||||||
|
// never stored without its code being handed out by that same claim.
|
||||||
// - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for
|
// - Nothing from stdout is ever logged; stderr (log-clean, spike §2.1) is tail-captured for
|
||||||
// failure diagnostics only.
|
// failure diagnostics only.
|
||||||
|
|
||||||
@@ -50,8 +57,16 @@ type EscrowCeremonyConfig struct {
|
|||||||
// DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured,
|
// DRConfigured answers "is the DR tier applied on this box?" (pbsdr.Manager.DRConfigured,
|
||||||
// late-bound). nil → reported not-applied.
|
// late-bound). nil → reported not-applied.
|
||||||
DRConfigured func() bool
|
DRConfigured func() bool
|
||||||
|
// Upload sends a ceremony's package (CeremonyOutput.PendingUpload, the hub upload body) to this
|
||||||
|
// host's hub record — called by the claim, BEFORE the code is handed out (R-978). nil → a claim of
|
||||||
|
// a pending package is refused (503): a code must never be handed out for a package the hub lacks.
|
||||||
|
Upload func(ctx context.Context, pkg []byte) error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// escrowUploadTimeout bounds the claim-time package upload (one small PUT; the subprocess's whole
|
||||||
|
// ceremony incl. its upload was measured at ~2.4 s).
|
||||||
|
const escrowUploadTimeout = 30 * time.Second
|
||||||
|
|
||||||
// ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until
|
// ceremonyRunner executes the fixed-argv sudo self-invocation. stdout is SECRET-BEARING until
|
||||||
// parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned
|
// parsed (it carries R inside the JSON object); the caller must zero it. Tests inject canned
|
||||||
// spike-shaped output; production is runCeremonySubprocess.
|
// spike-shaped output; production is runCeremonySubprocess.
|
||||||
@@ -129,6 +144,9 @@ func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool {
|
|||||||
if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning {
|
if s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
if s.escrowClaiming {
|
||||||
|
return false // R-978: a claim is uploading this job's package — its code must reach the claimer
|
||||||
|
}
|
||||||
s.wipeEscrowRLocked()
|
s.wipeEscrowRLocked()
|
||||||
cp := *job
|
cp := *job
|
||||||
s.escrowJob = &cp
|
s.escrowJob = &cp
|
||||||
@@ -137,7 +155,7 @@ func (s *Server) tryStartEscrowCeremony(job *escrowCeremonyJob) bool {
|
|||||||
|
|
||||||
// finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder.
|
// finishEscrowCeremony records the terminal phase and (on success) arms the one-shot R holder.
|
||||||
// R is stored as a byte slice so the claim/TTL paths can zero it in place.
|
// R is stored as a byte slice so the claim/TTL paths can zero it in place.
|
||||||
func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string, pending []byte) {
|
||||||
s.escrowMu.Lock()
|
s.escrowMu.Lock()
|
||||||
defer s.escrowMu.Unlock()
|
defer s.escrowMu.Unlock()
|
||||||
job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||||
@@ -145,6 +163,7 @@ func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
|||||||
s.escrowJob = &cp
|
s.escrowJob = &cp
|
||||||
if job.Phase == escrowPhaseDone && r != "" {
|
if job.Phase == escrowPhaseDone && r != "" {
|
||||||
s.escrowR = []byte(r)
|
s.escrowR = []byte(r)
|
||||||
|
s.escrowPending = append([]byte(nil), pending...)
|
||||||
s.escrowRClaimed = false
|
s.escrowRClaimed = false
|
||||||
s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL)
|
s.escrowRExpiry = s.nowFn().Add(escrowClaimTTL)
|
||||||
// Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in
|
// Active TTL belt: zero the holder even if nobody ever polls again. The lazy check in
|
||||||
@@ -164,6 +183,7 @@ func (s *Server) finishEscrowCeremony(job *escrowCeremonyJob, r string) {
|
|||||||
func (s *Server) wipeEscrowRLocked() {
|
func (s *Server) wipeEscrowRLocked() {
|
||||||
zeroBytes(s.escrowR)
|
zeroBytes(s.escrowR)
|
||||||
s.escrowR = nil
|
s.escrowR = nil
|
||||||
|
s.escrowPending = nil // R-978: the package goes with its code — never uploaded once the code is gone
|
||||||
}
|
}
|
||||||
|
|
||||||
// expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job
|
// expireEscrowRLocked applies the TTL outcome: an armed, unclaimed holder is zeroed and the job
|
||||||
@@ -172,6 +192,9 @@ func (s *Server) expireEscrowRLocked() {
|
|||||||
if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed {
|
if s.escrowJob == nil || s.escrowJob.Phase != escrowPhaseDone || s.escrowRClaimed {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if s.escrowClaiming {
|
||||||
|
return // R-978: the claim is uploading the package; voiding now would store a package nobody gets the code to
|
||||||
|
}
|
||||||
if len(s.escrowR) == 0 {
|
if len(s.escrowR) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -217,7 +240,7 @@ func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Reques
|
|||||||
UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
|
UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||||
}
|
}
|
||||||
if !s.tryStartEscrowCeremony(job) {
|
if !s.tryStartEscrowCeremony(job) {
|
||||||
writeErr(w, http.StatusConflict, "an escrow ceremony is already running")
|
writeErr(w, http.StatusConflict, "an escrow ceremony is already running, or its code is being claimed")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID)
|
s.logger.Info("local-api: escrow ceremony started (controller-driven)", "vmid", vmid, "job_id", job.JobID)
|
||||||
@@ -247,7 +270,7 @@ func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Reques
|
|||||||
detail += " | stderr: " + tail
|
detail += " | stderr: " + tail
|
||||||
}
|
}
|
||||||
job.Phase, job.Detail = escrowPhaseFailed, detail
|
job.Phase, job.Detail = escrowPhaseFailed, detail
|
||||||
s.finishEscrowCeremony(job, "")
|
s.finishEscrowCeremony(job, "", nil)
|
||||||
s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID,
|
s.logger.Warn("local-api: escrow ceremony failed", "job_id", job.JobID,
|
||||||
"exit", exit, "duration_ms", time.Since(start).Milliseconds())
|
"exit", exit, "duration_ms", time.Since(start).Milliseconds())
|
||||||
return
|
return
|
||||||
@@ -255,24 +278,34 @@ func (s *Server) handleEscrowCeremonyStart(w http.ResponseWriter, r *http.Reques
|
|||||||
var out escrow.CeremonyOutput
|
var out escrow.CeremonyOutput
|
||||||
if err := json.Unmarshal(stdout, &out); err != nil {
|
if err := json.Unmarshal(stdout, &out); err != nil {
|
||||||
job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object"
|
job.Phase, job.Detail = escrowPhaseFailed, "ceremony output is not the expected JSON object"
|
||||||
s.finishEscrowCeremony(job, "")
|
s.finishEscrowCeremony(job, "", nil)
|
||||||
s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID)
|
s.logger.Warn("local-api: escrow ceremony output unparseable (never logged)", "job_id", job.JobID)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" {
|
if out.Version != escrow.CeremonyOutputVersion || out.RecoveryCode == "" {
|
||||||
out.RecoveryCode = ""
|
out.RecoveryCode = ""
|
||||||
job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version)
|
job.Phase, job.Detail = escrowPhaseFailed, fmt.Sprintf("unexpected ceremony output (version %d)", out.Version)
|
||||||
s.finishEscrowCeremony(job, "")
|
s.finishEscrowCeremony(job, "", nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// R-978: the package must come back un-uploaded. An output that uploaded by itself (or carries no
|
||||||
|
// package) would mean the previous package is already gone, or the new one can never be stored.
|
||||||
|
if out.Uploaded || len(out.PendingUpload) == 0 {
|
||||||
|
out.RecoveryCode = ""
|
||||||
|
job.Phase, job.Detail = escrowPhaseFailed, "ceremony output carries no pending package"
|
||||||
|
s.finishEscrowCeremony(job, "", nil)
|
||||||
|
s.logger.Warn("local-api: escrow ceremony output has no pending package — failed (R-978)", "job_id", job.JobID,
|
||||||
|
"uploaded", out.Uploaded)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
job.Phase = escrowPhaseDone
|
job.Phase = escrowPhaseDone
|
||||||
job.KeyFingerprint = out.KeyFingerprint
|
job.KeyFingerprint = out.KeyFingerprint
|
||||||
job.EntropyBits = out.EntropyBits
|
job.EntropyBits = out.EntropyBits
|
||||||
job.ResticPwSealed = out.ResticPwSealed
|
job.ResticPwSealed = out.ResticPwSealed
|
||||||
job.Uploaded = out.Uploaded
|
job.Uploaded = false // R-978: uploaded by the claim
|
||||||
s.finishEscrowCeremony(job, out.RecoveryCode)
|
s.finishEscrowCeremony(job, out.RecoveryCode, out.PendingUpload)
|
||||||
out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort)
|
out.RecoveryCode = "" // drop the parsed reference promptly (GC caveat: best-effort)
|
||||||
s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot)",
|
s.logger.Info("local-api: escrow ceremony complete — R claimable (in-memory, one-shot); package uploads at the claim",
|
||||||
"job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded,
|
"job_id", job.JobID, "restic_pw_sealed", job.ResticPwSealed, "uploaded", job.Uploaded,
|
||||||
"claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds())
|
"claim_ttl_s", int(escrowClaimTTL.Seconds()), "duration_ms", time.Since(start).Milliseconds())
|
||||||
}()
|
}()
|
||||||
@@ -339,11 +372,60 @@ func (s *Server) handleEscrowCeremonyClaim(w http.ResponseWriter, r *http.Reques
|
|||||||
s.escrowMu.Unlock()
|
s.escrowMu.Unlock()
|
||||||
writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim")
|
writeErr(w, http.StatusConflict, "ceremony failed — nothing to claim")
|
||||||
return
|
return
|
||||||
|
case s.escrowClaiming:
|
||||||
|
s.escrowMu.Unlock()
|
||||||
|
writeErr(w, http.StatusConflict, "the recovery code is being claimed — try again in a moment")
|
||||||
|
return
|
||||||
case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid:
|
case s.escrowRClaimed || len(s.escrowR) == 0 || s.escrowJob.Phase == escrowPhaseVoid:
|
||||||
s.escrowMu.Unlock()
|
s.escrowMu.Unlock()
|
||||||
writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes")
|
writeErr(w, http.StatusGone, "the recovery code is no longer available (already claimed or expired) — run a new ceremony; the new code supersedes")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// R-978: store the NEW package on the hub first; only then hand out the code that opens it. Until this
|
||||||
|
// upload succeeds the hub keeps the previous package, and the customer's previous code keeps working.
|
||||||
|
if len(s.escrowPending) > 0 {
|
||||||
|
if s.escrowCeremony.Upload == nil {
|
||||||
|
s.escrowMu.Unlock()
|
||||||
|
writeErr(w, http.StatusServiceUnavailable, "the recovery package cannot be stored on this host (no upload configured) — the previous code still works")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pkg := s.escrowPending
|
||||||
|
jobID := s.escrowJob.JobID
|
||||||
|
s.escrowClaiming = true
|
||||||
|
s.escrowMu.Unlock()
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
base := s.baseCtx
|
||||||
|
if base == nil {
|
||||||
|
base = context.Background()
|
||||||
|
}
|
||||||
|
uctx, cancel := context.WithTimeout(base, escrowUploadTimeout)
|
||||||
|
err := s.escrowCeremony.Upload(uctx, pkg)
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
s.escrowMu.Lock()
|
||||||
|
s.escrowClaiming = false
|
||||||
|
if err != nil {
|
||||||
|
s.escrowMu.Unlock()
|
||||||
|
s.logger.Warn("local-api: escrow claim — package upload to the hub failed; code kept claimable, previous package stays current (R-978)",
|
||||||
|
"vmid", vmid, "job_id", jobID, "err", err, "duration_ms", time.Since(start).Milliseconds())
|
||||||
|
writeErr(w, http.StatusBadGateway, "the new recovery package could not be stored on the hub — the previous code still works; try again")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.escrowPending = nil
|
||||||
|
if s.escrowJob != nil && s.escrowJob.JobID == jobID {
|
||||||
|
s.escrowJob.Uploaded = true
|
||||||
|
s.escrowJob.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
s.logger.Info("local-api: escrow claim — package uploaded to the hub; handing out the code (R-978)",
|
||||||
|
"vmid", vmid, "job_id", jobID, "duration_ms", time.Since(start).Milliseconds())
|
||||||
|
if len(s.escrowR) == 0 { // cannot happen (TTL and supersede wait for the claim) — never hand out nothing
|
||||||
|
s.escrowMu.Unlock()
|
||||||
|
s.logger.Error("local-api: escrow claim — code vanished during the upload (invariant broken)", "job_id", jobID)
|
||||||
|
writeErr(w, http.StatusInternalServerError, "the recovery code was lost during the claim — run a new ceremony")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
recovery := string(s.escrowR)
|
recovery := string(s.escrowR)
|
||||||
s.wipeEscrowRLocked()
|
s.wipeEscrowRLocked()
|
||||||
s.escrowRClaimed = true
|
s.escrowRClaimed = true
|
||||||
|
|||||||
@@ -2,12 +2,14 @@ package localapi
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -19,25 +21,91 @@ import (
|
|||||||
|
|
||||||
const testR = "canary-alpha-bravo-charlie-delta-echo-foxtrot-golf-hotel-india"
|
const testR = "canary-alpha-bravo-charlie-delta-echo-foxtrot-golf-hotel-india"
|
||||||
|
|
||||||
// cannedCeremonyJSON is shaped exactly like the agent's --output=json object (spike §2.6 values).
|
// cannedCeremonyJSON is shaped exactly like the agent's --output=json object (spike §2.6 values;
|
||||||
|
// version 2, R-978: the package comes back un-uploaded in pending_upload). The package is a MODEL of
|
||||||
|
// the R-wrapped blob: it "opens" with exactly the code it was sealed under (fakeEscrowHub.opensWith).
|
||||||
func cannedCeremonyJSON(r string) string {
|
func cannedCeremonyJSON(r string) string {
|
||||||
return fmt.Sprintf(`{"version":1,"recovery_code":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","entropy_bits":129,"blob_bytes":383,"identity_blob_bytes":450,"restic_pw_sealed":true,"uploaded":true}`, r)
|
return fmt.Sprintf(`{"version":2,"recovery_code":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","entropy_bits":129,"blob_bytes":383,"identity_blob_bytes":450,"restic_pw_sealed":true,"uploaded":false,"pending_upload":%s}`,
|
||||||
|
r, sealedPackage(r))
|
||||||
|
}
|
||||||
|
|
||||||
|
// sealedPackage is the model hub upload body for a ceremony that minted code r.
|
||||||
|
func sealedPackage(r string) string {
|
||||||
|
return fmt.Sprintf(`{"blob_b64":%q,"key_fingerprint":"f2:87:68:2a:88:50:16:01","posture":"zero_knowledge","created_at":"2026-06-10T12:00:00Z"}`,
|
||||||
|
base64.StdEncoding.EncodeToString([]byte("sealed-under:"+r)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeEscrowHub is the hub's current escrow package for this host (the PUT target). Upload replaces it,
|
||||||
|
// as the hub does when the sealed secrets are unchanged (the R-978 case: no retained copy).
|
||||||
|
type fakeEscrowHub struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
current []byte
|
||||||
|
uploads int
|
||||||
|
err error // non-nil → the upload fails, nothing stored
|
||||||
|
gate chan struct{} // non-nil → the upload blocks until closed
|
||||||
|
entered chan struct{} // non-nil → closed when an upload starts
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeEscrowHub) upload(ctx context.Context, pkg []byte) error {
|
||||||
|
if f.entered != nil {
|
||||||
|
close(f.entered)
|
||||||
|
f.entered = nil
|
||||||
|
}
|
||||||
|
if f.gate != nil {
|
||||||
|
<-f.gate
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
if f.err != nil {
|
||||||
|
return f.err
|
||||||
|
}
|
||||||
|
f.current = append([]byte(nil), pkg...)
|
||||||
|
f.uploads++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// opensWith reports whether the hub's CURRENT package opens with code r — the customer's question.
|
||||||
|
func (f *fakeEscrowHub) opensWith(r string) bool {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
var pkg struct {
|
||||||
|
BlobB64 string `json:"blob_b64"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(f.current, &pkg) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
raw, err := base64.StdEncoding.DecodeString(pkg.BlobB64)
|
||||||
|
return err == nil && string(raw) == "sealed-under:"+r
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeEscrowHub) count() int {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return f.uploads
|
||||||
}
|
}
|
||||||
|
|
||||||
// newEscrowTestServer builds a server with the ceremony configured and every seam faked.
|
// newEscrowTestServer builds a server with the ceremony configured and every seam faked.
|
||||||
func newEscrowTestServer(t *testing.T, run ceremonyRunner) *Server {
|
func newEscrowTestServer(t *testing.T, run ceremonyRunner) *Server {
|
||||||
|
t.Helper()
|
||||||
|
srv, _ := newEscrowTestServerHub(t, run, &fakeEscrowHub{})
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
// newEscrowTestServerHub is newEscrowTestServer with the hub's escrow record exposed (R-978).
|
||||||
|
func newEscrowTestServerHub(t *testing.T, run ceremonyRunner, hub *fakeEscrowHub) (*Server, *fakeEscrowHub) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
|
srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
|
||||||
srv.escrowCeremony = &EscrowCeremonyConfig{
|
srv.escrowCeremony = &EscrowCeremonyConfig{
|
||||||
PBSStorageID: "felhom-pbs",
|
PBSStorageID: "felhom-pbs",
|
||||||
HubConfigured: true,
|
HubConfigured: true,
|
||||||
DRConfigured: func() bool { return true },
|
DRConfigured: func() bool { return true },
|
||||||
|
Upload: hub.upload,
|
||||||
}
|
}
|
||||||
srv.ceremonyRun = run
|
srv.ceremonyRun = run
|
||||||
srv.escrowSudoCheck = func(context.Context) error { return nil }
|
srv.escrowSudoCheck = func(context.Context) error { return nil }
|
||||||
srv.escrowLookPath = func(string) (string, error) { return "/usr/bin/age", nil }
|
srv.escrowLookPath = func(string) (string, error) { return "/usr/bin/age", nil }
|
||||||
srv.statFile = func(string) bool { return true }
|
srv.statFile = func(string) bool { return true }
|
||||||
return srv
|
return srv, hub
|
||||||
}
|
}
|
||||||
|
|
||||||
// okRunner returns the canned success output and counts invocations.
|
// okRunner returns the canned success output and counts invocations.
|
||||||
@@ -160,8 +228,8 @@ func TestEscrowCeremony_OneShotClaim(t *testing.T) {
|
|||||||
if !strings.Contains(body, `"phase":"done"`) || !strings.Contains(body, `"claimable":true`) {
|
if !strings.Contains(body, `"phase":"done"`) || !strings.Contains(body, `"claimable":true`) {
|
||||||
t.Fatalf("status not done/claimable: %s", body)
|
t.Fatalf("status not done/claimable: %s", body)
|
||||||
}
|
}
|
||||||
if !strings.Contains(body, `"restic_pw_sealed":true`) || !strings.Contains(body, `"uploaded":true`) {
|
if !strings.Contains(body, `"restic_pw_sealed":true`) || !strings.Contains(body, `"uploaded":false`) {
|
||||||
t.Fatalf("summary fields missing: %s", body)
|
t.Fatalf("summary fields missing (R-978: not uploaded before the claim): %s", body)
|
||||||
}
|
}
|
||||||
assertNoR(t, "status payload", body)
|
assertNoR(t, "status payload", body)
|
||||||
|
|
||||||
@@ -199,7 +267,8 @@ func TestEscrowCeremony_OneShotClaim(t *testing.T) {
|
|||||||
|
|
||||||
// Post-claim status: claimed, not claimable, still phase done.
|
// Post-claim status: claimed, not claimable, still phase done.
|
||||||
st2 := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
st2 := do(t, h, "GET", "/escrow/ceremony/status", "A", "")
|
||||||
if !strings.Contains(st2.Body.String(), `"claimed":true`) || strings.Contains(st2.Body.String(), `"claimable":true`) {
|
if !strings.Contains(st2.Body.String(), `"claimed":true`) || strings.Contains(st2.Body.String(), `"claimable":true`) ||
|
||||||
|
!strings.Contains(st2.Body.String(), `"uploaded":true`) {
|
||||||
t.Fatalf("post-claim status wrong: %s", st2.Body.String())
|
t.Fatalf("post-claim status wrong: %s", st2.Body.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -344,7 +413,7 @@ func TestEscrowCeremony_FailurePaths(t *testing.T) {
|
|||||||
})
|
})
|
||||||
t.Run("wrong version", func(t *testing.T) {
|
t.Run("wrong version", func(t *testing.T) {
|
||||||
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
srv := newEscrowTestServer(t, func(context.Context) ([]byte, []byte, int, error) {
|
||||||
return []byte(`{"version":2,"recovery_code":"` + testR + `"}`), nil, 0, nil
|
return []byte(`{"version":1,"recovery_code":"` + testR + `","uploaded":true}`), nil, 0, nil
|
||||||
})
|
})
|
||||||
h := srv.Handler()
|
h := srv.Handler()
|
||||||
startAndWait(t, srv, h)
|
startAndWait(t, srv, h)
|
||||||
|
|||||||
@@ -0,0 +1,219 @@
|
|||||||
|
package localapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// R-978 — starting the escrow ceremony again must not make the customer's written-down recovery code
|
||||||
|
// useless. The hub overwrites the current package when the sealed secrets are unchanged, so the NEW
|
||||||
|
// package may reach the hub only when its code is claimed. These tests ask the customer's question of
|
||||||
|
// the hub's current package — „does my code still open it?" — not whether some function ran.
|
||||||
|
|
||||||
|
const (
|
||||||
|
firstR = "first-ceremony-code-alpha"
|
||||||
|
secondR = "second-ceremony-code-bravo"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sequenceRunner mints the given codes in order, one per ceremony.
|
||||||
|
func sequenceRunner(codes ...string) ceremonyRunner {
|
||||||
|
i := 0
|
||||||
|
return func(context.Context) ([]byte, []byte, int, error) {
|
||||||
|
r := codes[i]
|
||||||
|
if i < len(codes)-1 {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
return []byte(cannedCeremonyJSON(r)), nil, 0, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func claimCode(t *testing.T, h http.Handler) (int, string) {
|
||||||
|
t.Helper()
|
||||||
|
w := do(t, h, "POST", "/escrow/ceremony/claim", "A", "")
|
||||||
|
return w.Code, w.Body.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// THE R-978 CASE. Ceremony 1 is claimed (the customer writes code 1 down). Ceremony 2 is started and
|
||||||
|
// abandoned — three ways: its code expires unclaimed, a third start supersedes it, the agent restarts.
|
||||||
|
// In every case the hub's current package must still open with code 1, and nothing was uploaded for 2.
|
||||||
|
//
|
||||||
|
// COMPANION RED-PROOF (observed 2026-10-18): restore the pre-R-978 shape — the package uploaded when the
|
||||||
|
// ceremony finishes (s.escrowCeremony.Upload(base, out.PendingUpload) in the job goroutine, as the root
|
||||||
|
// subprocess did with `--upload`) and nothing left for the claim (finishEscrowCeremony(job, R, nil)) →
|
||||||
|
// all three subtests FAIL with "the abandoned second ceremony replaced the package: code 1 no longer
|
||||||
|
// opens it (uploads=2)" (uploads=3 for the superseding start). Restored.
|
||||||
|
func TestEscrowCeremony_R978_AbandonedSecondCeremonyKeepsFirstCode(t *testing.T) {
|
||||||
|
setup := func(t *testing.T) (*Server, *fakeEscrowHub, http.Handler, *time.Time) {
|
||||||
|
srv, hub := newEscrowTestServerHub(t, sequenceRunner(firstR, secondR, "third-code-charlie"), &fakeEscrowHub{})
|
||||||
|
cur := testNow
|
||||||
|
srv.now = func() time.Time { return cur }
|
||||||
|
h := srv.Handler()
|
||||||
|
startAndWait(t, srv, h)
|
||||||
|
if code, body := claimCode(t, h); code != http.StatusOK || !strings.Contains(body, firstR) {
|
||||||
|
t.Fatalf("claim of ceremony 1: %d %s", code, body)
|
||||||
|
}
|
||||||
|
if !hub.opensWith(firstR) || hub.count() != 1 {
|
||||||
|
t.Fatalf("after the first claim the hub must hold code 1's package (uploads=%d)", hub.count())
|
||||||
|
}
|
||||||
|
startAndWait(t, srv, h) // ceremony 2 — about to be abandoned
|
||||||
|
return srv, hub, h, &cur
|
||||||
|
}
|
||||||
|
check := func(t *testing.T, hub *fakeEscrowHub) {
|
||||||
|
t.Helper()
|
||||||
|
if !hub.opensWith(firstR) {
|
||||||
|
t.Fatalf("the abandoned second ceremony replaced the package: code 1 no longer opens it (uploads=%d)", hub.count())
|
||||||
|
}
|
||||||
|
if hub.opensWith(secondR) || hub.count() != 1 {
|
||||||
|
t.Fatalf("a package for the never-claimed code 2 reached the hub (uploads=%d)", hub.count())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("code 2 expires unclaimed", func(t *testing.T) {
|
||||||
|
_, hub, h, cur := setup(t)
|
||||||
|
*cur = cur.Add(escrowClaimTTL + time.Minute)
|
||||||
|
if code, _ := claimCode(t, h); code != http.StatusGone {
|
||||||
|
t.Fatalf("claim after TTL: got %d, want 410", code)
|
||||||
|
}
|
||||||
|
check(t, hub)
|
||||||
|
})
|
||||||
|
t.Run("a third start supersedes code 2", func(t *testing.T) {
|
||||||
|
srv, hub, h, _ := setup(t)
|
||||||
|
startAndWait(t, srv, h) // ceremony 3, also left unclaimed
|
||||||
|
check(t, hub)
|
||||||
|
})
|
||||||
|
t.Run("the agent restarts before code 2 is claimed", func(t *testing.T) {
|
||||||
|
_, hub, _, _ := setup(t)
|
||||||
|
// A restart is a new process with an empty slot; only the hub's record survives it.
|
||||||
|
srv2, _ := newEscrowTestServerHub(t, sequenceRunner("unused"), hub)
|
||||||
|
if code, _ := claimCode(t, srv2.Handler()); code != http.StatusNotFound {
|
||||||
|
t.Fatalf("claim after a restart: got %d, want 404", code)
|
||||||
|
}
|
||||||
|
check(t, hub)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The positive control: CLAIMING code 2 is what replaces the package — the code handed out opens the
|
||||||
|
// package the hub now holds, and the claim answers only after the hub took it.
|
||||||
|
func TestEscrowCeremony_R978_ClaimStoresThePackageBeforeHandingOutTheCode(t *testing.T) {
|
||||||
|
srv, hub := newEscrowTestServerHub(t, sequenceRunner(firstR, secondR), &fakeEscrowHub{})
|
||||||
|
h := srv.Handler()
|
||||||
|
startAndWait(t, srv, h)
|
||||||
|
if hub.count() != 0 {
|
||||||
|
t.Fatalf("the package was uploaded before any claim (uploads=%d)", hub.count())
|
||||||
|
}
|
||||||
|
claimCode(t, h)
|
||||||
|
startAndWait(t, srv, h)
|
||||||
|
code, body := claimCode(t, h)
|
||||||
|
if code != http.StatusOK || !strings.Contains(body, secondR) {
|
||||||
|
t.Fatalf("claim of ceremony 2: %d %s", code, body)
|
||||||
|
}
|
||||||
|
if !hub.opensWith(secondR) || hub.count() != 2 {
|
||||||
|
t.Fatalf("after claiming code 2 the hub must hold its package (uploads=%d, opens with 2: %v)", hub.count(), hub.opensWith(secondR))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failed upload hands out NO code, keeps the code claimable and the previous package current; a retry
|
||||||
|
// that reaches the hub succeeds.
|
||||||
|
func TestEscrowCeremony_R978_UploadFailureKeepsTheCodeClaimable(t *testing.T) {
|
||||||
|
hub := &fakeEscrowHub{err: errors.New("hub returned HTTP 502")}
|
||||||
|
srv, _ := newEscrowTestServerHub(t, sequenceRunner(firstR), hub)
|
||||||
|
h := srv.Handler()
|
||||||
|
startAndWait(t, srv, h)
|
||||||
|
|
||||||
|
code, body := claimCode(t, h)
|
||||||
|
if code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("claim with the hub failing: got %d, want 502 (%s)", code, body)
|
||||||
|
}
|
||||||
|
assertNoRIn(t, "failed-upload claim payload", body, firstR)
|
||||||
|
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", "").Body.String(); !strings.Contains(st, `"claimable":true`) || !strings.Contains(st, `"uploaded":false`) {
|
||||||
|
t.Fatalf("after a failed upload the code must stay claimable and un-uploaded: %s", st)
|
||||||
|
}
|
||||||
|
|
||||||
|
hub.mu.Lock()
|
||||||
|
hub.err = nil
|
||||||
|
hub.mu.Unlock()
|
||||||
|
code, body = claimCode(t, h)
|
||||||
|
if code != http.StatusOK || !strings.Contains(body, firstR) || !hub.opensWith(firstR) {
|
||||||
|
t.Fatalf("retry after the hub recovered: %d %s (opens: %v)", code, body, hub.opensWith(firstR))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// While the claim's upload is in flight, neither the TTL nor a new start may take the code away — else
|
||||||
|
// the hub would hold a package whose code nobody was given (the R-978 failure, reached the other way).
|
||||||
|
func TestEscrowCeremony_R978_TTLAndRestartWaitForAnInFlightClaim(t *testing.T) {
|
||||||
|
hub := &fakeEscrowHub{gate: make(chan struct{}), entered: make(chan struct{})}
|
||||||
|
entered := hub.entered
|
||||||
|
srv, _ := newEscrowTestServerHub(t, sequenceRunner(firstR, secondR), hub)
|
||||||
|
var mu = make(chan struct{}, 1)
|
||||||
|
mu <- struct{}{}
|
||||||
|
cur := testNow
|
||||||
|
srv.now = func() time.Time { <-mu; defer func() { mu <- struct{}{} }(); return cur }
|
||||||
|
h := srv.Handler()
|
||||||
|
startAndWait(t, srv, h)
|
||||||
|
|
||||||
|
type res struct {
|
||||||
|
code int
|
||||||
|
body string
|
||||||
|
}
|
||||||
|
done := make(chan res, 1)
|
||||||
|
go func() { c, b := claimCode(t, h); done <- res{c, b} }()
|
||||||
|
<-entered
|
||||||
|
|
||||||
|
<-mu
|
||||||
|
cur = cur.Add(escrowClaimTTL + time.Minute) // the TTL passes mid-upload
|
||||||
|
mu <- struct{}{}
|
||||||
|
if st := do(t, h, "GET", "/escrow/ceremony/status", "A", "").Body.String(); strings.Contains(st, `"phase":"unclaimed_void"`) {
|
||||||
|
t.Fatalf("the TTL voided the code while its package was uploading: %s", st)
|
||||||
|
}
|
||||||
|
if w := do(t, h, "POST", "/escrow/ceremony", "A", ""); w.Code != http.StatusConflict {
|
||||||
|
t.Fatalf("a new start during the claim's upload: got %d, want 409", w.Code)
|
||||||
|
}
|
||||||
|
if code, _ := claimCode(t, h); code != http.StatusConflict {
|
||||||
|
t.Fatalf("a second claim during the upload: got %d, want 409", code)
|
||||||
|
}
|
||||||
|
close(hub.gate)
|
||||||
|
r := <-done
|
||||||
|
if r.code != http.StatusOK || !strings.Contains(r.body, firstR) || !hub.opensWith(firstR) {
|
||||||
|
t.Fatalf("the in-flight claim must deliver the code whose package it stored: %d %s", r.code, r.body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No upload configured → the claim refuses rather than hand out a code for a package the hub lacks.
|
||||||
|
func TestEscrowCeremony_R978_NoUploaderRefusesTheClaim(t *testing.T) {
|
||||||
|
srv, _ := newEscrowTestServerHub(t, sequenceRunner(firstR), &fakeEscrowHub{})
|
||||||
|
srv.escrowCeremony.Upload = nil
|
||||||
|
h := srv.Handler()
|
||||||
|
startAndWait(t, srv, h)
|
||||||
|
code, body := claimCode(t, h)
|
||||||
|
if code != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("claim with no uploader: got %d, want 503 (%s)", code, body)
|
||||||
|
}
|
||||||
|
assertNoRIn(t, "no-uploader claim payload", body, firstR)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An output that uploaded by itself (the pre-R-978 shape) or carries no package fails the ceremony.
|
||||||
|
func TestEscrowCeremony_R978_OutputWithoutPendingPackageFails(t *testing.T) {
|
||||||
|
srv, _ := newEscrowTestServerHub(t, func(context.Context) ([]byte, []byte, int, error) {
|
||||||
|
return []byte(`{"version":2,"recovery_code":"` + firstR + `","uploaded":true}`), nil, 0, nil
|
||||||
|
}, &fakeEscrowHub{})
|
||||||
|
h := srv.Handler()
|
||||||
|
startAndWait(t, srv, h)
|
||||||
|
st := do(t, h, "GET", "/escrow/ceremony/status", "A", "").Body.String()
|
||||||
|
if !strings.Contains(st, `"phase":"failed"`) || !strings.Contains(st, "no pending package") {
|
||||||
|
t.Fatalf("want failed: %s", st)
|
||||||
|
}
|
||||||
|
if code, _ := claimCode(t, h); code != http.StatusConflict {
|
||||||
|
t.Fatalf("claim on a failed ceremony: got %d, want 409", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertNoRIn(t *testing.T, where, body, r string) {
|
||||||
|
t.Helper()
|
||||||
|
if strings.Contains(body, r) {
|
||||||
|
t.Fatalf("%s carries the recovery code", where)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,207 @@
|
|||||||
|
package localapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The internal data place („Belső tárhely", agent v0.158.0) — felhom.eu architecture/16-storage-physical-identity.md
|
||||||
|
// §5, §7. On a one-disk box the root helper configs/felhom-internal-place makes a second thin volume in local-lvm
|
||||||
|
// (<vg>/felhom-belso, ext4, label felhom-belso). The installer (a new box) or the signed add_internal_place job (an old
|
||||||
|
// box) then writes InternalPlaceRecord to the agent's state dir. From then on the agent treats the volume exactly like
|
||||||
|
// an enrolled drive: a Felhom mount unit by fs-UUID at /mnt/belso (EnsureMount), the `enrolled` intent, and — when the
|
||||||
|
// controller asks with its own token (POST /disks/guest-attach) — the bind at /mnt/felhom-drives/belso. So it is a
|
||||||
|
// HOST mount under the guest's mp8 parent: not in the guest config, not in vzdump, not in PBS (`16` §5).
|
||||||
|
//
|
||||||
|
// What it is NOT: a drive the household can format, wipe, eject or decommission. Its device is a device-mapper volume
|
||||||
|
// on the system disk, so the role gate (roleForMountPath → system) and the format path (ValidateBlockDevice refuses
|
||||||
|
// dm) refuse all four — pinned by internalplace_test.go. Its identity is the system disk's (build A), so a Tier-2 copy
|
||||||
|
// between it and sys_drive is never a second copy.
|
||||||
|
const (
|
||||||
|
InternalPlaceName = "belso"
|
||||||
|
InternalPlaceWhere = "/mnt/belso"
|
||||||
|
// InternalPlaceFile is the record's name inside the agent's state dir.
|
||||||
|
InternalPlaceFile = "internal-place.json"
|
||||||
|
)
|
||||||
|
|
||||||
|
var reFSUUID = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
||||||
|
|
||||||
|
// InternalPlaceRecord is the state file the maker writes once the volume exists and is formatted.
|
||||||
|
type InternalPlaceRecord struct {
|
||||||
|
FSUUID string `json:"fs_uuid"`
|
||||||
|
GiB int `json:"gib,omitempty"`
|
||||||
|
MadeBy string `json:"made_by,omitempty"` // "installer" | "signed-job"
|
||||||
|
MadeAt string `json:"made_at,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadInternalPlace reads the record; ok=false when there is none (the common case: a box with no internal place).
|
||||||
|
func ReadInternalPlace(file string) (InternalPlaceRecord, bool, error) {
|
||||||
|
var rec InternalPlaceRecord
|
||||||
|
b, err := os.ReadFile(file)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return rec, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return rec, false, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(b, &rec); err != nil {
|
||||||
|
return rec, false, fmt.Errorf("internal place record %s: %w", file, err)
|
||||||
|
}
|
||||||
|
if !reFSUUID.MatchString(rec.FSUUID) {
|
||||||
|
return rec, false, fmt.Errorf("internal place record %s: fs_uuid %q is not a filesystem UUID", file, rec.FSUUID)
|
||||||
|
}
|
||||||
|
return rec, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteInternalPlace writes the record atomically (0600).
|
||||||
|
func WriteInternalPlace(file string, rec InternalPlaceRecord) error {
|
||||||
|
if !reFSUUID.MatchString(rec.FSUUID) {
|
||||||
|
return fmt.Errorf("internal place: fs_uuid %q is not a filesystem UUID", rec.FSUUID)
|
||||||
|
}
|
||||||
|
b, err := json.MarshalIndent(rec, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp := file + ".tmp"
|
||||||
|
if err := os.WriteFile(tmp, append(b, '\n'), 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp, file)
|
||||||
|
}
|
||||||
|
|
||||||
|
// internalPlaceState memoises what EnsureInternalPlace last saw, so the 20-second tick logs a state CHANGE once and
|
||||||
|
// does no work on the steady state.
|
||||||
|
type internalPlaceState struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
done string // the fs-UUID that is mounted and recorded enrolled
|
||||||
|
lastWhy string // the last reason it could not be ensured (logged once per change)
|
||||||
|
}
|
||||||
|
|
||||||
|
// internalPlaceUUID returns the configured internal place's fs-UUID, or "" when this box has none (or the record is
|
||||||
|
// unreadable — logged by EnsureInternalPlace).
|
||||||
|
func (s *Server) internalPlaceUUID() string {
|
||||||
|
if s.internalPlaceFile == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
rec, ok, err := ReadInternalPlace(s.internalPlaceFile)
|
||||||
|
if err != nil || !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return rec.FSUUID
|
||||||
|
}
|
||||||
|
|
||||||
|
// isInternalPlace reports whether a host path is this box's internal place: its raw mount (/mnt/belso) or its bind
|
||||||
|
// under the guests' shared parent (/mnt/felhom-drives/belso — the path POST /disks/same-disk judges).
|
||||||
|
func (s *Server) isInternalPlace(hostPath string) bool {
|
||||||
|
p := path.Clean(hostPath)
|
||||||
|
return (p == InternalPlaceWhere || p == StableParentDir+"/"+InternalPlaceName) && s.internalPlaceUUID() != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnsureInternalPlace mounts the internal place on the host at /mnt/belso by fs-UUID and records it `enrolled`,
|
||||||
|
// when this box has one. Idempotent and cheap on the steady state; safe on the 20-second tick. It never formats,
|
||||||
|
// never unmounts, and never mounts anything at /mnt/belso that is not the recorded UUID: a household drive that
|
||||||
|
// happens to be named „belso" keeps its mount, and the internal place waits (logged once).
|
||||||
|
func (s *Server) EnsureInternalPlace(ctx context.Context) {
|
||||||
|
if s.internalPlaceFile == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rec, ok, err := ReadInternalPlace(s.internalPlaceFile)
|
||||||
|
if err != nil {
|
||||||
|
s.internalPlaceNote("record unreadable: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
uuid := rec.FSUUID
|
||||||
|
st := &s.ipState
|
||||||
|
st.mu.Lock()
|
||||||
|
done := st.done == uuid
|
||||||
|
st.mu.Unlock()
|
||||||
|
|
||||||
|
// What is at /mnt/belso now, if anything?
|
||||||
|
mountedUUID, mounted := "", false
|
||||||
|
if mounts, merr := s.hostReader().Mounts(); merr == nil {
|
||||||
|
for _, m := range mounts {
|
||||||
|
if path.Clean(m.MountPoint) == InternalPlaceWhere {
|
||||||
|
mounted = true
|
||||||
|
if u, okU := s.hostReader().ResolveUUID(m.Device); okU {
|
||||||
|
mountedUUID = u
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mounted && mountedUUID != "" && mountedUUID != uuid {
|
||||||
|
s.internalPlaceNote("another filesystem (uuid " + mountedUUID + ") is mounted at " + InternalPlaceWhere + " — the internal place waits")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// A Felhom unit for /mnt/belso that names another UUID (a household drive called „belso", unplugged): never
|
||||||
|
// overwrite it — that would silently take the drive's name and mount point.
|
||||||
|
if s.driveTargets != nil {
|
||||||
|
if drives, derr := s.driveTargets.Known(ctx); derr == nil {
|
||||||
|
for _, d := range drives {
|
||||||
|
if path.Clean(d.MountPath) == InternalPlaceWhere && d.UUID != "" && d.UUID != uuid {
|
||||||
|
s.internalPlaceNote("a drive unit for " + InternalPlaceWhere + " names another uuid (" + d.UUID + ") — the internal place waits")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if mounted && mountedUUID == uuid {
|
||||||
|
if !done {
|
||||||
|
s.recordIntent(ctx, InternalPlaceWhere, "enrolled")
|
||||||
|
st.mu.Lock()
|
||||||
|
st.done, st.lastWhy = uuid, ""
|
||||||
|
st.mu.Unlock()
|
||||||
|
s.logger.Info("internal place: mounted and enrolled", "where", InternalPlaceWhere, "fs_uuid", uuid, "gib", rec.GiB)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if s.disks == nil {
|
||||||
|
s.internalPlaceNote("disk management is not configured on this host")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if s.resolveStorageDevice != nil {
|
||||||
|
if _, rerr := s.resolveStorageDevice("uuid:" + uuid); rerr != nil {
|
||||||
|
s.internalPlaceNote("its volume is not present (uuid " + uuid + ")")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
if err := s.disks.EnsureMount(ctx, storage.MountSpec{Name: InternalPlaceName, UUID: uuid, Where: InternalPlaceWhere, FSType: "ext4"}); err != nil {
|
||||||
|
s.internalPlaceNote("mount failed: " + err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.recordIntent(ctx, InternalPlaceWhere, "enrolled")
|
||||||
|
st.mu.Lock()
|
||||||
|
st.done, st.lastWhy = uuid, ""
|
||||||
|
st.mu.Unlock()
|
||||||
|
s.logger.Info("internal place: mounted on the host", "where", InternalPlaceWhere, "fs_uuid", uuid, "gib", rec.GiB,
|
||||||
|
"duration_ms", time.Since(start).Milliseconds())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) internalPlaceNote(why string) {
|
||||||
|
st := &s.ipState
|
||||||
|
st.mu.Lock()
|
||||||
|
changed := st.lastWhy != why
|
||||||
|
st.lastWhy, st.done = why, ""
|
||||||
|
st.mu.Unlock()
|
||||||
|
if changed {
|
||||||
|
s.logger.Warn("internal place: not ensured", "why", why)
|
||||||
|
} else {
|
||||||
|
s.logger.Debug("internal place: still not ensured", "why", why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// InternalPlaceFileIn is the record's path inside a state dir.
|
||||||
|
func InternalPlaceFileIn(stateDir string) string { return filepath.Join(stateDir, InternalPlaceFile) }
|
||||||
@@ -0,0 +1,249 @@
|
|||||||
|
package localapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The internal data place (felhom.eu architecture/16 §5, agent v0.158.0): mounted like an enrolled drive, but never
|
||||||
|
// at the cost of a household drive, never formatted, ejected or decommissioned, and the same disk as sys_drive.
|
||||||
|
|
||||||
|
const ipUUID = "0b6e4c5a-1d2e-4f3a-9b8c-7d6e5f4a3b2c"
|
||||||
|
const ipDev = "/dev/mapper/pve-felhom--belso"
|
||||||
|
|
||||||
|
// ipHost is a HostReader whose mount table the test changes, and whose by-uuid map it controls.
|
||||||
|
type ipHost struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
mounts []storage.Mount
|
||||||
|
uuids map[string]string // device → fs-UUID
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *ipHost) Mounts() ([]storage.Mount, error) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
return append([]storage.Mount(nil), h.mounts...), nil
|
||||||
|
}
|
||||||
|
func (h *ipHost) ResolveUUID(dev string) (string, bool) { u, ok := h.uuids[dev]; return u, ok }
|
||||||
|
func (h *ipHost) DeviceExists(string) bool { return true }
|
||||||
|
func (h *ipHost) Rotational(string) (bool, bool) { return false, false }
|
||||||
|
func (h *ipHost) Removable(string) (bool, bool) { return false, false }
|
||||||
|
func (h *ipHost) BlockSlaves(string) ([]string, bool) { return nil, false }
|
||||||
|
|
||||||
|
func newIPHost(extra ...storage.Mount) *ipHost {
|
||||||
|
return &ipHost{
|
||||||
|
// root on a whole-disk partition (as sysOnSDA): the system disk resolves, so a USB drive is user-data.
|
||||||
|
mounts: append([]storage.Mount{{Device: "/dev/sda1", MountPoint: "/", FSType: "ext4"}}, extra...),
|
||||||
|
uuids: map[string]string{ipDev: ipUUID, "/dev/sdb1": "99999999-aaaa-bbbb-cccc-dddddddddddd"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type ipRig struct {
|
||||||
|
srv *Server
|
||||||
|
disks *fakeDiskOps
|
||||||
|
intent *fakeIntent
|
||||||
|
}
|
||||||
|
|
||||||
|
func newIPRig(t *testing.T, host *ipHost, known []storage.KnownTarget, record bool) ipRig {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := filepath.Join(dir, InternalPlaceFile)
|
||||||
|
if record {
|
||||||
|
if err := WriteInternalPlace(file, InternalPlaceRecord{FSUUID: ipUUID, GiB: 183, MadeBy: "installer"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d, in := &fakeDiskOps{}, newFakeIntent()
|
||||||
|
srv, err := NewServer(Options{
|
||||||
|
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
|
||||||
|
Storage: fakeStorage{}, Tokens: staticTokens{"A": 8200}, Disks: d, DiskGate: &fakeGate{},
|
||||||
|
HostReader: host, Intent: in, DriveTargets: fakeKnownTargets{drives: known},
|
||||||
|
InternalPlaceFile: file, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv.baseCtx = context.Background()
|
||||||
|
srv.resolveStorageDevice = func(id string) (string, error) {
|
||||||
|
if id == "uuid:"+ipUUID {
|
||||||
|
return "/dev/dm-7", nil
|
||||||
|
}
|
||||||
|
return "", errors.New("absent")
|
||||||
|
}
|
||||||
|
return ipRig{srv, d, in}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInternalPlace_NoRecord_DoesNothing(t *testing.T) {
|
||||||
|
r := newIPRig(t, newIPHost(), nil, false)
|
||||||
|
r.srv.EnsureInternalPlace(context.Background())
|
||||||
|
if len(r.disks.mountCalls) != 0 {
|
||||||
|
t.Fatalf("no record: no mount may be made, got %+v", r.disks.mountCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInternalPlace_Record_MountsByUUIDAtMntBelso(t *testing.T) {
|
||||||
|
r := newIPRig(t, newIPHost(), nil, true)
|
||||||
|
r.srv.EnsureInternalPlace(context.Background())
|
||||||
|
if len(r.disks.mountCalls) != 1 {
|
||||||
|
t.Fatalf("want exactly one EnsureMount, got %+v", r.disks.mountCalls)
|
||||||
|
}
|
||||||
|
got := r.disks.mountCalls[0]
|
||||||
|
if got.UUID != ipUUID || got.Where != InternalPlaceWhere || got.FSType != "ext4" || got.Name != InternalPlaceName {
|
||||||
|
t.Fatalf("EnsureMount spec: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInternalPlace_AlreadyMounted_EnrolledOnceNoRemount(t *testing.T) {
|
||||||
|
host := newIPHost(storage.Mount{Device: ipDev, MountPoint: InternalPlaceWhere, FSType: "ext4"})
|
||||||
|
r := newIPRig(t, host, nil, true)
|
||||||
|
r.srv.EnsureInternalPlace(context.Background())
|
||||||
|
r.srv.EnsureInternalPlace(context.Background())
|
||||||
|
if len(r.disks.mountCalls) != 0 {
|
||||||
|
t.Fatalf("already mounted: no EnsureMount, got %+v", r.disks.mountCalls)
|
||||||
|
}
|
||||||
|
if r.intent.Get("uuid:"+ipUUID) != storage.IntentEnrolled {
|
||||||
|
t.Fatalf("the internal place must be recorded enrolled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A household drive called „belso" keeps its mount point: the internal place never takes it.
|
||||||
|
// Positive control: TestInternalPlace_Record_MountsByUUIDAtMntBelso (the same record, /mnt/belso free).
|
||||||
|
func TestInternalPlace_NeverTakesAHouseholdDriveNamedBelso(t *testing.T) {
|
||||||
|
mounted := newIPHost(storage.Mount{Device: "/dev/sdb1", MountPoint: InternalPlaceWhere, FSType: "ext4"})
|
||||||
|
r := newIPRig(t, mounted, nil, true)
|
||||||
|
r.srv.EnsureInternalPlace(context.Background())
|
||||||
|
if len(r.disks.mountCalls) != 0 || r.intent.Get("uuid:"+ipUUID) != "" {
|
||||||
|
t.Fatalf("a foreign filesystem at /mnt/belso: nothing may be mounted or recorded, got %+v", r.disks.mountCalls)
|
||||||
|
}
|
||||||
|
unplugged := []storage.KnownTarget{{Name: "belso", UUID: "99999999-aaaa-bbbb-cccc-dddddddddddd", MountPath: InternalPlaceWhere, MountBacked: true}}
|
||||||
|
r2 := newIPRig(t, newIPHost(), unplugged, true)
|
||||||
|
r2.srv.EnsureInternalPlace(context.Background())
|
||||||
|
if len(r2.disks.mountCalls) != 0 {
|
||||||
|
t.Fatalf("an unplugged household drive's unit for /mnt/belso must not be overwritten, got %+v", r2.disks.mountCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInternalPlace_VolumeAbsent_NoMount(t *testing.T) {
|
||||||
|
r := newIPRig(t, newIPHost(), nil, true)
|
||||||
|
r.srv.resolveStorageDevice = func(string) (string, error) { return "", errors.New("absent") }
|
||||||
|
r.srv.EnsureInternalPlace(context.Background())
|
||||||
|
if len(r.disks.mountCalls) != 0 {
|
||||||
|
t.Fatalf("absent volume: no mount, got %+v", r.disks.mountCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The household cannot eject or decommission it: its device is a device-mapper volume on the system disk.
|
||||||
|
// Positive control: the same calls for a whole-disk USB drive (/dev/sdb1) are not refused by role.
|
||||||
|
func TestInternalPlace_EjectAndDecommissionRefused(t *testing.T) {
|
||||||
|
host := newIPHost(
|
||||||
|
storage.Mount{Device: ipDev, MountPoint: InternalPlaceWhere, FSType: "ext4"},
|
||||||
|
storage.Mount{Device: "/dev/sdb1", MountPoint: "/mnt/hdd_1", FSType: "ext4"},
|
||||||
|
)
|
||||||
|
r := newIPRig(t, host, nil, true)
|
||||||
|
h := r.srv.Handler()
|
||||||
|
for _, route := range []string{"/disks/eject", "/disks/decommission"} {
|
||||||
|
if w := do(t, h, "POST", route, "A", `{"where":"/mnt/belso"}`); w.Code != http.StatusForbidden {
|
||||||
|
t.Fatalf("%s /mnt/belso: want 403, got %d %s", route, w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if w := do(t, h, "POST", "/disks/eject", "A", `{"where":"/mnt/hdd_1"}`); w.Code == http.StatusForbidden {
|
||||||
|
t.Fatalf("positive control: a USB drive's eject must not be refused by role, got %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
for _, u := range r.disks.unmountCalls {
|
||||||
|
if u == InternalPlaceWhere {
|
||||||
|
t.Fatalf("the internal place must never be unmounted, got %+v", r.disks.unmountCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consequence `16` §5 needs: a copy between the internal place and sys_drive is the SAME disk (build A then never
|
||||||
|
// counts it), and the place is reported as kind internal.
|
||||||
|
type ipIdentHost struct{ fakeIdentHost }
|
||||||
|
|
||||||
|
func (h *ipIdentHost) KernelName(dev string) (string, bool) {
|
||||||
|
if dev == ipDev {
|
||||||
|
return "dm-7", true
|
||||||
|
}
|
||||||
|
return h.fakeIdentHost.KernelName(dev)
|
||||||
|
}
|
||||||
|
func (h *ipIdentHost) Slaves(n string) []string {
|
||||||
|
if n == "dm-7" {
|
||||||
|
return []string{"dm-3"} // felhom-belso → the thin pool
|
||||||
|
}
|
||||||
|
if n == "dm-3" {
|
||||||
|
return []string{"sda3"} // the pool → the system disk's LVM partition
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInternalPlace_IsTheSystemDiskAndKindInternal(t *testing.T) {
|
||||||
|
ih := &ipIdentHost{fakeIdentHost{
|
||||||
|
virt: "none",
|
||||||
|
parent: map[string]string{"sda3": "sda"},
|
||||||
|
props: map[string]map[string]string{"sda": {"ID_BUS": "ata", "ID_MODEL": "SanDisk", "ID_SERIAL_SHORT": "1"}},
|
||||||
|
mounts: []storage.Mount{
|
||||||
|
{Device: "/dev/mapper/pve-root", MountPoint: "/", FSType: "ext4"},
|
||||||
|
{Device: "/dev/mapper/pve-root", MountPoint: "/mnt/felhom-drives", FSType: "ext4"},
|
||||||
|
{Device: ipDev, MountPoint: InternalPlaceWhere, FSType: "ext4"},
|
||||||
|
{Device: ipDev, MountPoint: "/mnt/felhom-drives/belso", FSType: "ext4"},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
fg := &fakeGuestsCfg{mounts: map[int]map[string]string{8200: {
|
||||||
|
"mp0": "local-lvm:vm-8200-disk-1,mp=/var/lib/felhom,backup=1,size=100G",
|
||||||
|
"mp8": "/mnt/felhom-drives,mp=/mnt/felhom-drives",
|
||||||
|
}}}
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := filepath.Join(dir, InternalPlaceFile)
|
||||||
|
if err := WriteInternalPlace(file, InternalPlaceRecord{FSUUID: ipUUID}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv, err := NewServer(Options{
|
||||||
|
ListenAddr: "127.0.0.1:0", Guests: fg, Backups: &fakeBackups{}, Store: &fakeStore{},
|
||||||
|
Storage: fakeStorage{targets: []hub.StorageTarget{provenSSD}}, Tokens: staticTokens{"A": 8200},
|
||||||
|
HostReader: fakeHostReader{mounts: ih.mounts}, Identity: &storage.IdentityResolver{Host: ih},
|
||||||
|
InternalPlaceFile: file, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv.baseCtx = context.Background()
|
||||||
|
got, code := sameDisk(t, srv.Handler(), "/mnt/felhom-drives/belso/felhom-data/jellyfin", "/mnt/sys_drive")
|
||||||
|
if code != http.StatusOK || got.Different || !got.Proven || got.Reason != "same_disk" {
|
||||||
|
t.Fatalf("internal place vs sys_drive: want same disk, proven; got %d %+v", code, got)
|
||||||
|
}
|
||||||
|
if got.A.Kind != storage.PlaceKindInternal {
|
||||||
|
t.Fatalf("kind: want %q, got %q", storage.PlaceKindInternal, got.A.Kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInternalPlaceRecord_RefusesANonUUID(t *testing.T) {
|
||||||
|
file := filepath.Join(t.TempDir(), InternalPlaceFile)
|
||||||
|
if err := WriteInternalPlace(file, InternalPlaceRecord{FSUUID: "../../etc"}); err == nil {
|
||||||
|
t.Fatal("a non-UUID must not be written")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The root helper's own tests (configs/test_felhom_internal_place.py — the ruled size table, never formatting what
|
||||||
|
// holds anything) run with the Go suite.
|
||||||
|
func TestInternalPlaceHelperPythonSuite(t *testing.T) {
|
||||||
|
py, err := exec.LookPath("python3")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("python3 not found")
|
||||||
|
}
|
||||||
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
script := filepath.Join(filepath.Dir(file), "..", "..", "configs", "test_felhom_internal_place.py")
|
||||||
|
out, err := exec.Command(py, script).CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("helper suite failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -135,6 +135,8 @@ func (s *Server) ownMountIdentity(hostPath string) (storage.PlaceIdentity, strin
|
|||||||
}
|
}
|
||||||
kind := storage.PlaceKindDisk
|
kind := storage.PlaceKindDisk
|
||||||
switch {
|
switch {
|
||||||
|
case s.isInternalPlace(hostPath): // v0.158.0 `16` §5: the one-disk box's data place, on the system disk
|
||||||
|
kind = storage.PlaceKindInternal
|
||||||
case id.Reason == storage.ReasonNetwork:
|
case id.Reason == storage.ReasonNetwork:
|
||||||
kind = storage.PlaceKindNetwork
|
kind = storage.PlaceKindNetwork
|
||||||
case strings.HasPrefix(m.Device, "/dev/") && !strings.HasPrefix(m.Device, "/dev/mapper/") && isPartitionPath(m.Device):
|
case strings.HasPrefix(m.Device, "/dev/") && !strings.HasPrefix(m.Device, "/dev/mapper/") && isPartitionPath(m.Device):
|
||||||
|
|||||||
+69
-43
@@ -90,6 +90,10 @@ type BackupStore interface {
|
|||||||
type LastKnownBackupStore interface {
|
type LastKnownBackupStore interface {
|
||||||
RecordBackupSuccess(target string, b hub.Backup) error
|
RecordBackupSuccess(target string, b hub.Backup) error
|
||||||
LastKnownSuccess(target string, vmid int) (time.Time, bool)
|
LastKnownSuccess(target string, vmid int) (time.Time, bool)
|
||||||
|
// R-973: the cadence cap's restart-proof history and its refusals.
|
||||||
|
RecentBackupSuccesses(target string, vmid int, since time.Time) []time.Time
|
||||||
|
RecordBackupRefusal(target string, vmid int, at time.Time) error
|
||||||
|
LastBackupRefusal() (target string, vmid int, at time.Time, ok bool)
|
||||||
}
|
}
|
||||||
|
|
||||||
// StorageView yields the host's observed storage targets (for mapping a mount's storage id →
|
// StorageView yields the host's observed storage targets (for mapping a mount's storage id →
|
||||||
@@ -208,6 +212,9 @@ type Options struct {
|
|||||||
// Intent records drive enroll/eject intent for the self-heal watchdog (slice 10 P3). OPTIONAL —
|
// Intent records drive enroll/eject intent for the self-heal watchdog (slice 10 P3). OPTIONAL —
|
||||||
// when nil, no intent is recorded (self-heal runs ungated).
|
// when nil, no intent is recorded (self-heal runs ungated).
|
||||||
Intent IntentRecorder
|
Intent IntentRecorder
|
||||||
|
// InternalPlaceFile (v0.158.0, felhom.eu `16` §5) is the path of the internal data place record
|
||||||
|
// (InternalPlaceRecord). OPTIONAL — "" means this host has no internal place and EnsureInternalPlace is a no-op.
|
||||||
|
InternalPlaceFile string
|
||||||
// GuestBinds persists which user-data drives (by durable-id) are enrolled into each guest, so the
|
// GuestBinds persists which user-data drives (by durable-id) are enrolled into each guest, so the
|
||||||
// startup re-assert (ReassertGuestBinds) can restore a bind that a re-provision dropped (F9).
|
// startup re-assert (ReassertGuestBinds) can restore a bind that a re-provision dropped (F9).
|
||||||
// OPTIONAL — when nil, guest binds are not recorded and the startup re-assert is a no-op.
|
// OPTIONAL — when nil, guest binds are not recorded and the startup re-assert is a no-op.
|
||||||
@@ -299,6 +306,11 @@ type Server struct {
|
|||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
|
|
||||||
|
// R-973 cadence cap: this process's successful runs per tier+guest, and the newest refusal.
|
||||||
|
capMu sync.Mutex
|
||||||
|
capRuns map[backupJobKey][]time.Time
|
||||||
|
capRefusal capRefusal
|
||||||
|
|
||||||
disks DiskOps // slice 8C (optional)
|
disks DiskOps // slice 8C (optional)
|
||||||
diskGate StorageGate // slice 8C (optional)
|
diskGate StorageGate // slice 8C (optional)
|
||||||
guestList GuestLister // slice 8C (optional)
|
guestList GuestLister // slice 8C (optional)
|
||||||
@@ -316,12 +328,14 @@ type Server struct {
|
|||||||
// identity blob from the hub, unseal it with the customer's recovery code, return ONLY the
|
// identity blob from the hub, unseal it with the customer's recovery code, return ONLY the
|
||||||
// offsite repository password. OPTIONAL — nil (no hub client configured) makes
|
// offsite repository password. OPTIONAL — nil (no hub client configured) makes
|
||||||
// POST /escrow/recover-offsite-password answer 503 rather than pretending.
|
// POST /escrow/recover-offsite-password answer 503 rather than pretending.
|
||||||
escrowRecovery EscrowRecoverer
|
escrowRecovery EscrowRecoverer
|
||||||
intent IntentRecorder // slice 10 P3 (optional)
|
intent IntentRecorder // slice 10 P3 (optional)
|
||||||
guestBinds *GuestBindStore // F9 startup bind re-assert record (optional)
|
internalPlaceFile string // v0.158.0 `16` §5 (optional)
|
||||||
formatJobs *FormatJobStore // F20-BUG3 detached-format job record (optional)
|
ipState internalPlaceState // EnsureInternalPlace's memo
|
||||||
staleLock StaleLockController // F2-b startup stale-lock recovery (optional)
|
guestBinds *GuestBindStore // F9 startup bind re-assert record (optional)
|
||||||
partialSweep func(ctx context.Context, vmid int) // R-1006 start-up sweep of dead vzdump leftovers (optional)
|
formatJobs *FormatJobStore // F20-BUG3 detached-format job record (optional)
|
||||||
|
staleLock StaleLockController // F2-b startup stale-lock recovery (optional)
|
||||||
|
partialSweep func(ctx context.Context, vmid int) // R-1006 start-up sweep of dead vzdump leftovers (optional)
|
||||||
// guestPower (F-REBOOT) is per-guest start-attempt state for the guest-power watchdog.
|
// guestPower (F-REBOOT) is per-guest start-attempt state for the guest-power watchdog.
|
||||||
// Guarded by guestPowerMu in guestpower.go; in-memory on purpose (see guestPowerState).
|
// Guarded by guestPowerMu in guestpower.go; in-memory on purpose (see guestPowerState).
|
||||||
guestPower map[int]guestPowerState
|
guestPower map[int]guestPowerState
|
||||||
@@ -421,6 +435,8 @@ type Server struct {
|
|||||||
escrowMu sync.Mutex
|
escrowMu sync.Mutex
|
||||||
escrowJob *escrowCeremonyJob
|
escrowJob *escrowCeremonyJob
|
||||||
escrowR []byte
|
escrowR []byte
|
||||||
|
escrowPending []byte // R-978: the ceremony's hub upload body, sent by the claim before R is handed out
|
||||||
|
escrowClaiming bool // R-978: a claim is uploading escrowPending (TTL and a new start wait)
|
||||||
escrowRClaimed bool
|
escrowRClaimed bool
|
||||||
escrowRExpiry time.Time
|
escrowRExpiry time.Time
|
||||||
escrowDone <-chan struct{} // closes when the detached job finishes (tests wait on it)
|
escrowDone <-chan struct{} // closes when the detached job finishes (tests wait on it)
|
||||||
@@ -455,43 +471,44 @@ func NewServer(o Options) (*Server, error) {
|
|||||||
cadence = defaultBackupCadence
|
cadence = defaultBackupCadence
|
||||||
}
|
}
|
||||||
s := &Server{
|
s := &Server{
|
||||||
addr: o.ListenAddr,
|
addr: o.ListenAddr,
|
||||||
cert: o.Cert,
|
cert: o.Cert,
|
||||||
guests: o.Guests,
|
guests: o.Guests,
|
||||||
backups: o.Backups,
|
backups: o.Backups,
|
||||||
store: o.Store,
|
store: o.Store,
|
||||||
storage: o.Storage,
|
storage: o.Storage,
|
||||||
driveTargets: o.DriveTargets,
|
driveTargets: o.DriveTargets,
|
||||||
smart: o.Smart,
|
smart: o.Smart,
|
||||||
tokens: o.Tokens,
|
tokens: o.Tokens,
|
||||||
cadence: cadence,
|
cadence: cadence,
|
||||||
logger: o.Logger,
|
logger: o.Logger,
|
||||||
now: func() time.Time { return time.Now().UTC() },
|
now: func() time.Time { return time.Now().UTC() },
|
||||||
disks: o.Disks,
|
disks: o.Disks,
|
||||||
diskGate: o.DiskGate,
|
diskGate: o.DiskGate,
|
||||||
guestList: o.Guests2,
|
guestList: o.Guests2,
|
||||||
guestAttach: o.GuestAttach,
|
guestAttach: o.GuestAttach,
|
||||||
mem: o.Memory,
|
mem: o.Memory,
|
||||||
netStorage: o.NetStorage,
|
netStorage: o.NetStorage,
|
||||||
privileged: o.Privileged,
|
privileged: o.Privileged,
|
||||||
configPath: o.ConfigPath,
|
configPath: o.ConfigPath,
|
||||||
stateDir: o.StateDir,
|
stateDir: o.StateDir,
|
||||||
netMountRoot: storage.NetworkMountRoot,
|
netMountRoot: storage.NetworkMountRoot,
|
||||||
smbCredsDir: o.SmbCredsDir,
|
smbCredsDir: o.SmbCredsDir,
|
||||||
escrowStagePath: o.EscrowStagePath,
|
escrowStagePath: o.EscrowStagePath,
|
||||||
escrowRecovery: o.EscrowRecovery,
|
escrowRecovery: o.EscrowRecovery,
|
||||||
intent: o.Intent,
|
intent: o.Intent,
|
||||||
guestBinds: o.GuestBinds,
|
internalPlaceFile: o.InternalPlaceFile,
|
||||||
formatJobs: o.FormatJobs,
|
guestBinds: o.GuestBinds,
|
||||||
staleLock: o.StaleLock,
|
formatJobs: o.FormatJobs,
|
||||||
host: o.HostReader,
|
staleLock: o.StaleLock,
|
||||||
ident: o.Identity,
|
host: o.HostReader,
|
||||||
hostMetrics: o.HostMetrics,
|
ident: o.Identity,
|
||||||
hostID: o.HostID,
|
hostMetrics: o.HostMetrics,
|
||||||
agentVersion: o.AgentVersion,
|
hostID: o.HostID,
|
||||||
logRing: o.LogRing,
|
agentVersion: o.AgentVersion,
|
||||||
jobs: map[backupJobKey]*backupJob{},
|
logRing: o.LogRing,
|
||||||
swapInFlight: map[int]bool{},
|
jobs: map[backupJobKey]*backupJob{},
|
||||||
|
swapInFlight: map[int]bool{},
|
||||||
}
|
}
|
||||||
// R-82 tier resolution. Options.BackupTiers is authoritative when supplied; otherwise ONE tier
|
// R-82 tier resolution. Options.BackupTiers is authoritative when supplied; otherwise ONE tier
|
||||||
// is synthesized from Backups + BackupCadence — the pre-R-82 shape, so every existing caller
|
// is synthesized from Backups + BackupCadence — the pre-R-82 shape, so every existing caller
|
||||||
@@ -872,6 +889,12 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int)
|
|||||||
"a backup is already in flight on target "+busyTarget+" (job "+busyJob+") — only one backup runs at a time per guest")
|
"a backup is already in flight on target "+busyTarget+" (job "+busyJob+") — only one backup runs at a time per guest")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// R-973: the cadence cap (backup_cap.go). Checked under jobsMu, after the in-flight answers, so two
|
||||||
|
// requests cannot both pass it (only one run per guest is ever in flight).
|
||||||
|
if s.capRefuses(w, key, tier) {
|
||||||
|
s.jobsMu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
// Job ids must be unique PER TIER, and by construction rather than by clock luck: two tiers
|
// Job ids must be unique PER TIER, and by construction rather than by clock luck: two tiers
|
||||||
// started inside the same nanosecond (the weekly both-due night, or any injected clock) would
|
// started inside the same nanosecond (the weekly both-due night, or any injected clock) would
|
||||||
// otherwise collide and hand the second caller the first tier's id. The PRIMARY keeps the
|
// otherwise collide and hand the second caller the first tier's id. The PRIMARY keeps the
|
||||||
@@ -935,6 +958,9 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int)
|
|||||||
s.logger.Info("local-api: backup job complete", "vmid", vmid, "target", tier.TargetID, "job", jobID, "archive", b.Archive)
|
s.logger.Info("local-api: backup job complete", "vmid", vmid, "target", tier.TargetID, "job", jobID, "archive", b.Archive)
|
||||||
}
|
}
|
||||||
s.store.RecordBackup(b)
|
s.store.RecordBackup(b)
|
||||||
|
if b.Success {
|
||||||
|
s.recordCapRun(key, b) // R-973
|
||||||
|
}
|
||||||
if b.Success && s.lastKnown != nil {
|
if b.Success && s.lastKnown != nil {
|
||||||
if err := s.lastKnown.RecordBackupSuccess(tier.TargetID, b); err != nil {
|
if err := s.lastKnown.RecordBackupSuccess(tier.TargetID, b); err != nil {
|
||||||
// Not fatal: the backup exists. Only the fallback after a restart loses this copy.
|
// Not fatal: the backup exists. Only the fallback after a restart loses this copy.
|
||||||
|
|||||||
@@ -64,6 +64,11 @@ const (
|
|||||||
// The config bundle (agent v0.143.0, R-840, `11` §5.4.2): the box's ROOT-OWNED files (sudoers, wrappers, units).
|
// The config bundle (agent v0.143.0, R-840, `11` §5.4.2): the box's ROOT-OWNED files (sudoers, wrappers, units).
|
||||||
// Destructive-class (signed, operational key) like agent_update; the root wrapper re-verifies the signature itself.
|
// Destructive-class (signed, operational key) like agent_update; the root wrapper re-verifies the signature itself.
|
||||||
ClassAgentConfigUpdate OpClass = "agent_config_update"
|
ClassAgentConfigUpdate OpClass = "agent_config_update"
|
||||||
|
|
||||||
|
// The internal data place on an OLD one-disk box (agent v0.158.0, felhom.eu `16` §9): thin space on the system
|
||||||
|
// disk, made by the root helper felhom-internal-place. Destructive-class (signed, operational key): it is an
|
||||||
|
// operator act, never a household button, and no agent-internal provenance may start it.
|
||||||
|
ClassAddInternalPlace OpClass = "add_internal_place"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Disposition is the classifier verdict.
|
// Disposition is the classifier verdict.
|
||||||
@@ -132,7 +137,7 @@ func Classify(class OpClass, prov Provenance) Disposition {
|
|||||||
return Destructive
|
return Destructive
|
||||||
case ClassKeyRotation:
|
case ClassKeyRotation:
|
||||||
return Destructive
|
return Destructive
|
||||||
case ClassAgentUpdate, ClassOSDockerStep, ClassOSPVEStep, ClassOSKernelStep, ClassAgentConfigUpdate:
|
case ClassAgentUpdate, ClassOSDockerStep, ClassOSPVEStep, ClassOSKernelStep, ClassAgentConfigUpdate, ClassAddInternalPlace:
|
||||||
// Never benign — no agent-internal provenance can make replacing the agent binary
|
// Never benign — no agent-internal provenance can make replacing the agent binary
|
||||||
// unsigned-safe (a compromised process must not be able to self-bless an update).
|
// unsigned-safe (a compromised process must not be able to self-bless an update).
|
||||||
return Destructive
|
return Destructive
|
||||||
|
|||||||
@@ -39,9 +39,21 @@ type decommissionParams struct {
|
|||||||
// none is wanted, because a drive can legitimately be decommissioned while physically ABSENT.
|
// none is wanted, because a drive can legitimately be decommissioned while physically ABSENT.
|
||||||
type DecommissionExecutor struct {
|
type DecommissionExecutor struct {
|
||||||
intent IntentDecommissioner
|
intent IntentDecommissioner
|
||||||
|
binds GuestBindPruner // optional (v0.159.1): drop the drive from every guest's bind record, as the self-serve path does
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GuestBindPruner is the per-guest enrolled-bind record (*localapi.GuestBindStore). The self-serve decommission prunes
|
||||||
|
// the drive from it; the signed one did not, so a forgotten drive (R-1124) left a record the 20-second guest-bind
|
||||||
|
// re-assert warned about forever ("skipping non-enrolled drive"), measured on demo-hp 2026-10-17.
|
||||||
|
type GuestBindPruner interface {
|
||||||
|
Guests() map[int][]string
|
||||||
|
Remove(vmid int, durableID string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetGuestBinds wires the bind record (nil-safe: no record, nothing pruned).
|
||||||
|
func (d *DecommissionExecutor) SetGuestBinds(b GuestBindPruner) { d.binds = b }
|
||||||
|
|
||||||
// NewDecommissionExecutor wires the executor to the agent's intent store.
|
// NewDecommissionExecutor wires the executor to the agent's intent store.
|
||||||
func NewDecommissionExecutor(intent IntentDecommissioner, logger *slog.Logger) *DecommissionExecutor {
|
func NewDecommissionExecutor(intent IntentDecommissioner, logger *slog.Logger) *DecommissionExecutor {
|
||||||
if logger == nil {
|
if logger == nil {
|
||||||
@@ -75,6 +87,20 @@ func (d *DecommissionExecutor) Execute(_ context.Context, op string, params json
|
|||||||
if err := d.intent.SetDecommissioned(p.DurableID); err != nil {
|
if err := d.intent.SetDecommissioned(p.DurableID); err != nil {
|
||||||
return fmt.Errorf("decommission: record intent for %q: %w", p.DurableID, err)
|
return fmt.Errorf("decommission: record intent for %q: %w", p.DurableID, err)
|
||||||
}
|
}
|
||||||
|
if d.binds != nil {
|
||||||
|
for vmid, ids := range d.binds.Guests() {
|
||||||
|
for _, id := range ids {
|
||||||
|
if id != p.DurableID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := d.binds.Remove(vmid, id); err != nil {
|
||||||
|
d.logger.Warn("decommission: guest-bind record not pruned", "vmid", vmid, "durable_id", id, "err", err)
|
||||||
|
} else {
|
||||||
|
d.logger.Info("decommission: drive removed from the guest's bind record", "vmid", vmid, "durable_id", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
d.logger.Warn("decommission: operator-signed decommission complete (drive will never auto-mount)", "durable_id", p.DurableID)
|
d.logger.Warn("decommission: operator-signed decommission complete (drive will never auto-mount)", "durable_id", p.DurableID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -89,3 +89,37 @@ func TestExecutorChain_DispatchesToOwner(t *testing.T) {
|
|||||||
t.Errorf("want ErrNoExecutor for an unowned op, got %v", err)
|
t.Errorf("want ErrNoExecutor for an unowned op, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeBinds struct{ m map[int][]string }
|
||||||
|
|
||||||
|
func (f *fakeBinds) Guests() map[int][]string { return f.m }
|
||||||
|
func (f *fakeBinds) Remove(vmid int, id string) error {
|
||||||
|
var kept []string
|
||||||
|
for _, x := range f.m[vmid] {
|
||||||
|
if x != id {
|
||||||
|
kept = append(kept, x)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.m[vmid] = kept
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// v0.159.1: the signed decommission drops the drive from EVERY guest's bind record, as the self-serve path does — and
|
||||||
|
// only that drive. RED-PROOF (felhom.eu audits/agent-allowlist-rotation-2026-10-17/P1/redproof-bindprune.txt): with
|
||||||
|
// the prune loop removed, the record still holds the drive and this test fails.
|
||||||
|
func TestDecommissionExecutor_PrunesTheGuestBindRecord(t *testing.T) {
|
||||||
|
fd := &fakeDecommissioner{}
|
||||||
|
fb := &fakeBinds{m: map[int][]string{9201: {"uuid:keep", "uuid:gone"}, 9202: {"uuid:gone"}}}
|
||||||
|
ex := NewDecommissionExecutor(fd, nil)
|
||||||
|
ex.SetGuestBinds(fb)
|
||||||
|
params, _ := json.Marshal(decommissionParams{DurableID: "uuid:gone"})
|
||||||
|
if err := ex.Execute(context.Background(), opDecommission, params); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := fb.m[9201]; len(got) != 1 || got[0] != "uuid:keep" {
|
||||||
|
t.Fatalf("9201 record = %v, want [uuid:keep]", got)
|
||||||
|
}
|
||||||
|
if got := fb.m[9202]; len(got) != 0 {
|
||||||
|
t.Fatalf("9202 record = %v, want empty", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package signedjobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
|
)
|
||||||
|
|
||||||
|
// opAddInternalPlace is the op class this executor serves (mirrors reconcile.ClassAddInternalPlace).
|
||||||
|
const opAddInternalPlace = "add_internal_place"
|
||||||
|
|
||||||
|
// InternalPlaceHelper is the root helper's path (configs/felhom-internal-place, from the config bundle).
|
||||||
|
const InternalPlaceHelper = "/usr/local/sbin/felhom-internal-place"
|
||||||
|
|
||||||
|
var reHelperResult = regexp.MustCompile(`^(created|existing|formatted) uuid=([0-9a-fA-F-]{36}) gib=([0-9]+)$`)
|
||||||
|
|
||||||
|
// InternalPlaceExecutor adds the internal data place to an OLD one-disk box (felhom.eu architecture/16 §9: „one-disk
|
||||||
|
// boxes get no internal place by themselves; adding one later is a signed operator job, not a household button").
|
||||||
|
// It runs the SAME root helper the installer runs — `felhom-internal-place create 0` (0 GiB planned: the guest's
|
||||||
|
// volumes already exist and are counted) — writes the record the agent mounts from, and asks the local API to mount
|
||||||
|
// it now. Every safety rule (the ruled size, never format what holds anything) lives in the helper, as root.
|
||||||
|
//
|
||||||
|
// The op carries no parameters: the size is the rule's, never the signer's. Destructive-class by default (the
|
||||||
|
// classifier's fail-safe; ClassAddInternalPlace documents it): it takes thin space on the system disk, so only an
|
||||||
|
// operator signature may start it.
|
||||||
|
type InternalPlaceExecutor struct {
|
||||||
|
runner proxmox.Runner
|
||||||
|
recordFile string
|
||||||
|
ensure func(context.Context) // late-bound: the local API server is built after the jobs runner
|
||||||
|
logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewInternalPlaceExecutor wires the executor. ensure may be nil at construction and set later through SetEnsure.
|
||||||
|
func NewInternalPlaceExecutor(runner proxmox.Runner, recordFile string, logger *slog.Logger) *InternalPlaceExecutor {
|
||||||
|
if logger == nil {
|
||||||
|
logger = slog.Default()
|
||||||
|
}
|
||||||
|
return &InternalPlaceExecutor{runner: runner, recordFile: recordFile, logger: logger}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetEnsure late-binds the mount step (localapi.Server.EnsureInternalPlace).
|
||||||
|
func (e *InternalPlaceExecutor) SetEnsure(f func(context.Context)) { e.ensure = f }
|
||||||
|
|
||||||
|
// Execute implements signedjobs.Executor for add_internal_place.
|
||||||
|
func (e *InternalPlaceExecutor) Execute(ctx context.Context, op string, params json.RawMessage) error {
|
||||||
|
if op != opAddInternalPlace {
|
||||||
|
return ErrNoExecutor
|
||||||
|
}
|
||||||
|
if p := bytes.TrimSpace(params); len(p) > 0 && !bytes.Equal(p, []byte("{}")) && !bytes.Equal(p, []byte("null")) {
|
||||||
|
return fmt.Errorf("add_internal_place: takes no parameters (the size is the ruled rule's), got %s", p)
|
||||||
|
}
|
||||||
|
if e.runner == nil || e.recordFile == "" {
|
||||||
|
return fmt.Errorf("add_internal_place: not wired on this host (no privileged runner or state dir)")
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
e.logger.Warn("add_internal_place: executing operator-signed job (the internal data place, `16` §9)")
|
||||||
|
out, stderr, err := e.runner.Run(ctx, InternalPlaceHelper, "create", "0")
|
||||||
|
line := strings.TrimSpace(string(out))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("add_internal_place: helper refused: %v: %s", err, strings.TrimSpace(string(stderr)))
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(line, "skipped ") {
|
||||||
|
// Not an error the operator can retry away: the pool has no room by the ruled rule. Fail the job with the
|
||||||
|
// helper's own numbers so the hub shows why.
|
||||||
|
return fmt.Errorf("add_internal_place: %s", line)
|
||||||
|
}
|
||||||
|
m := reHelperResult.FindStringSubmatch(line)
|
||||||
|
if m == nil {
|
||||||
|
return fmt.Errorf("add_internal_place: unexpected helper output %q", line)
|
||||||
|
}
|
||||||
|
gib, _ := strconv.Atoi(m[3])
|
||||||
|
rec := localapi.InternalPlaceRecord{FSUUID: m[2], GiB: gib, MadeBy: "signed-job", MadeAt: time.Now().UTC().Format(time.RFC3339)}
|
||||||
|
if err := localapi.WriteInternalPlace(e.recordFile, rec); err != nil {
|
||||||
|
return fmt.Errorf("add_internal_place: writing the record: %w", err)
|
||||||
|
}
|
||||||
|
if e.ensure != nil {
|
||||||
|
e.ensure(ctx)
|
||||||
|
}
|
||||||
|
e.logger.Warn("add_internal_place: complete", "result", m[1], "fs_uuid", m[2], "gib", gib,
|
||||||
|
"duration_ms", time.Since(start).Milliseconds())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package signedjobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||||
|
)
|
||||||
|
|
||||||
|
// add_internal_place (felhom.eu architecture/16 §9, agent v0.158.0): the operator's job for an OLD one-disk box.
|
||||||
|
|
||||||
|
type ipRunner struct {
|
||||||
|
out, stderr string
|
||||||
|
err error
|
||||||
|
calls [][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *ipRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
r.calls = append(r.calls, append([]string{name}, args...))
|
||||||
|
return []byte(r.out), []byte(r.stderr), r.err
|
||||||
|
}
|
||||||
|
func (r *ipRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
return r.Run(ctx, name, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ipExec(t *testing.T, r *ipRunner) (*InternalPlaceExecutor, string, *int) {
|
||||||
|
t.Helper()
|
||||||
|
file := filepath.Join(t.TempDir(), localapi.InternalPlaceFile)
|
||||||
|
e := NewInternalPlaceExecutor(r, file, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||||
|
n := 0
|
||||||
|
e.SetEnsure(func(context.Context) { n++ })
|
||||||
|
return e, file, &n
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAddInternalPlace_RunsTheHelperWritesTheRecordAndMounts(t *testing.T) {
|
||||||
|
r := &ipRunner{out: "created uuid=0b6e4c5a-1d2e-4f3a-9b8c-7d6e5f4a3b2c gib=33\n"}
|
||||||
|
e, file, ensured := ipExec(t, r)
|
||||||
|
if err := e.Execute(context.Background(), "add_internal_place", []byte(`{}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(r.calls) != 1 || strings.Join(r.calls[0], " ") != InternalPlaceHelper+" create 0" {
|
||||||
|
t.Fatalf("the helper must be called exactly as `create 0`, got %v", r.calls)
|
||||||
|
}
|
||||||
|
rec, ok, err := localapi.ReadInternalPlace(file)
|
||||||
|
if err != nil || !ok || rec.FSUUID != "0b6e4c5a-1d2e-4f3a-9b8c-7d6e5f4a3b2c" || rec.GiB != 33 || rec.MadeBy != "signed-job" {
|
||||||
|
t.Fatalf("record: %+v ok=%v err=%v", rec, ok, err)
|
||||||
|
}
|
||||||
|
if *ensured != 1 {
|
||||||
|
t.Fatalf("the mount must be asked for once, got %d", *ensured)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAddInternalPlace_NoRoomOrRefusalOrGarbage_WritesNothing(t *testing.T) {
|
||||||
|
for name, r := range map[string]*ipRunner{
|
||||||
|
"skipped": {out: "skipped only 7 GiB free for it (pool 53, reserve 16, promised 102, planned 0; needs 20)"},
|
||||||
|
"refused": {err: errors.New("exit status 1"), stderr: "felhom-internal-place: REFUSED: holds something"},
|
||||||
|
"garbage": {out: "created uuid=../../etc gib=1"},
|
||||||
|
} {
|
||||||
|
e, file, ensured := ipExec(t, r)
|
||||||
|
if err := e.Execute(context.Background(), "add_internal_place", nil); err == nil {
|
||||||
|
t.Fatalf("%s: want an error", name)
|
||||||
|
}
|
||||||
|
if _, statErr := os.Stat(file); !os.IsNotExist(statErr) || *ensured != 0 {
|
||||||
|
t.Fatalf("%s: no record and no mount may follow, stat=%v ensured=%d", name, statErr, *ensured)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAddInternalPlace_TakesNoParametersAndIgnoresOtherOps(t *testing.T) {
|
||||||
|
r := &ipRunner{out: "created uuid=0b6e4c5a-1d2e-4f3a-9b8c-7d6e5f4a3b2c gib=33"}
|
||||||
|
e, _, _ := ipExec(t, r)
|
||||||
|
if err := e.Execute(context.Background(), "add_internal_place", []byte(`{"gib":900}`)); err == nil || len(r.calls) != 0 {
|
||||||
|
t.Fatalf("a size from the signer must be refused before the helper runs, err=%v calls=%v", err, r.calls)
|
||||||
|
}
|
||||||
|
if err := e.Execute(context.Background(), "decommission", []byte(`{}`)); !errors.Is(err, ErrNoExecutor) {
|
||||||
|
t.Fatalf("another op: want ErrNoExecutor, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAddInternalPlace_IsSignatureOnly(t *testing.T) {
|
||||||
|
if got := reconcile.Classify(reconcile.ClassAddInternalPlace, reconcile.Provenance{}); got != reconcile.Destructive {
|
||||||
|
t.Fatalf("add_internal_place must need an operator signature, classified %v", got)
|
||||||
|
}
|
||||||
|
if string(reconcile.ClassAddInternalPlace) != opAddInternalPlace {
|
||||||
|
t.Fatalf("op string drift: %q vs %q", reconcile.ClassAddInternalPlace, opAddInternalPlace)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -13,14 +13,16 @@ import (
|
|||||||
// (guarded, Impl-1) and/or mounts + binds. Blank disks are "initialize"-only; disks already carrying a
|
// (guarded, Impl-1) and/or mounts + binds. Blank disks are "initialize"-only; disks already carrying a
|
||||||
// mountable FS are ALSO "attach" candidates.
|
// mountable FS are ALSO "attach" candidates.
|
||||||
type CandidateDisk struct {
|
type CandidateDisk struct {
|
||||||
Device string `json:"device"` // whole disk, e.g. /dev/sdd
|
Device string `json:"device"` // whole disk, e.g. /dev/sdd
|
||||||
SizeBytes int64 `json:"size_bytes"` // 0 when unreadable
|
SizeBytes int64 `json:"size_bytes"` // 0 when unreadable
|
||||||
Model string `json:"model,omitempty"`
|
Model string `json:"model,omitempty"`
|
||||||
FSType string `json:"fstype,omitempty"` // first filesystem found ("" = blank)
|
FSType string `json:"fstype,omitempty"` // first filesystem found ("" = blank)
|
||||||
DataBearing bool `json:"data_bearing"` // has any FS / partition (→ wipe-confirm downstream)
|
DataBearing bool `json:"data_bearing"` // has any FS / partition (→ wipe-confirm downstream)
|
||||||
Mountable bool `json:"mountable"` // carries an ext4/xfs FS the agent can mount as-is
|
Mountable bool `json:"mountable"` // carries an ext4/xfs FS the agent can mount as-is
|
||||||
MountSource string `json:"mount_source,omitempty"` // the node to mount for attach (e.g. /dev/sdd1)
|
MountSource string `json:"mount_source,omitempty"` // the node to mount for attach (e.g. /dev/sdd1)
|
||||||
DurableID string `json:"durable_id,omitempty"` // "uuid:<fs-uuid>" if it has an FS; "" for a blank disk
|
DurableID string `json:"durable_id,omitempty"` // "uuid:<fs-uuid>" if it has an FS; "" for a blank disk
|
||||||
|
// Partitions (v0.158.0, felhom.eu `16` §6): each partition of this disk as its own place — see partitions.go.
|
||||||
|
Partitions []CandidatePartition `json:"partitions,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// mountableFSTypes are the filesystems the agent can mount as-is (EnsureMount) → an "attach" candidate.
|
// mountableFSTypes are the filesystems the agent can mount as-is (EnsureMount) → an "attach" candidate.
|
||||||
@@ -83,6 +85,8 @@ func (h *SudoHostOps) buildCandidate(dev, name string) CandidateDisk {
|
|||||||
if len(nodes) > 1 {
|
if len(nodes) > 1 {
|
||||||
c.DataBearing = true
|
c.DataBearing = true
|
||||||
}
|
}
|
||||||
|
c.Partitions = candidatePartitions(lout, name, readSysPartSize, readSysPartNumber, h.host.ResolveUUID,
|
||||||
|
felhomOwnedMounts(dev, nodes, h.mountTable))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return c
|
return c
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
@@ -24,8 +25,13 @@ var devDiskRoot = "/dev/disk"
|
|||||||
const (
|
const (
|
||||||
durableByID = "byid:"
|
durableByID = "byid:"
|
||||||
durableByUUID = "byuuid:"
|
durableByUUID = "byuuid:"
|
||||||
|
// partFSUUIDSep joins a partition's by-id link and its fs-UUID in one wipe id (v0.158.0, `16` §6).
|
||||||
|
partFSUUIDSep = "+fsuuid:"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// rePartLink matches a by-id link of a PARTITION (udev appends -part<N> to the disk's link).
|
||||||
|
var rePartLink = regexp.MustCompile(`-part[1-9][0-9]{0,2}$`)
|
||||||
|
|
||||||
// byIDPriority ranks /dev/disk/by-id link prefixes most-stable-first: wwn (hardware world-wide
|
// byIDPriority ranks /dev/disk/by-id link prefixes most-stable-first: wwn (hardware world-wide
|
||||||
// name) > nvme-eui (NVMe EUI) > nvme-/ata-/scsi-/usb- (model+serial). dm-/lvm-/md- names are
|
// name) > nvme-eui (NVMe EUI) > nvme-/ata-/scsi-/usb- (model+serial). dm-/lvm-/md- names are
|
||||||
// excluded (they are mapper constructs, not the physical disk we want to bind a wipe to).
|
// excluded (they are mapper constructs, not the physical disk we want to bind a wipe to).
|
||||||
@@ -41,6 +47,14 @@ func DeviceDurableID(device string) (string, error) {
|
|||||||
}
|
}
|
||||||
// 1. best by-id link pointing at this device.
|
// 1. best by-id link pointing at this device.
|
||||||
if name := bestByIDLink(target); name != "" {
|
if name := bestByIDLink(target); name != "" {
|
||||||
|
// v0.158.0 (felhom.eu `16` §6): a PARTITION's link already carries the disk id + the part number
|
||||||
|
// (wwn-…-part2). When the part holds a filesystem, its fs-UUID is bound too, so a part that was reformatted —
|
||||||
|
// or another disk's part2 that took the link — derives a different id and the wipe refuses.
|
||||||
|
if rePartLink.MatchString(name) {
|
||||||
|
if uuid := byUUIDFor(target); uuid != "" {
|
||||||
|
return durableByID + name + partFSUUIDSep + uuid, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
return durableByID + name, nil
|
return durableByID + name, nil
|
||||||
}
|
}
|
||||||
// 2. fallback: a filesystem UUID symlink.
|
// 2. fallback: a filesystem UUID symlink.
|
||||||
@@ -56,10 +70,29 @@ func ResolveDurableDevice(durableID string) (string, error) {
|
|||||||
switch {
|
switch {
|
||||||
case strings.HasPrefix(durableID, durableByID):
|
case strings.HasPrefix(durableID, durableByID):
|
||||||
name := strings.TrimPrefix(durableID, durableByID)
|
name := strings.TrimPrefix(durableID, durableByID)
|
||||||
|
fsUUID := ""
|
||||||
|
if i := strings.Index(name, partFSUUIDSep); i >= 0 {
|
||||||
|
name, fsUUID = name[:i], name[i+len(partFSUUIDSep):]
|
||||||
|
if !rePartLink.MatchString(name) || !safeLinkName(fsUUID) {
|
||||||
|
return "", fmt.Errorf("storage: unsafe durable id %q", durableID)
|
||||||
|
}
|
||||||
|
}
|
||||||
if !safeLinkName(name) {
|
if !safeLinkName(name) {
|
||||||
return "", fmt.Errorf("storage: unsafe durable id %q", durableID)
|
return "", fmt.Errorf("storage: unsafe durable id %q", durableID)
|
||||||
}
|
}
|
||||||
return filepath.EvalSymlinks(filepath.Join(devDiskRoot, "by-id", name))
|
dev, err := filepath.EvalSymlinks(filepath.Join(devDiskRoot, "by-id", name))
|
||||||
|
if err != nil || fsUUID == "" {
|
||||||
|
return dev, err
|
||||||
|
}
|
||||||
|
// Both halves must name the SAME device now (`16` §6): the part by its disk id + number, and its filesystem.
|
||||||
|
byFS, ferr := filepath.EvalSymlinks(filepath.Join(devDiskRoot, "by-uuid", fsUUID))
|
||||||
|
if ferr != nil {
|
||||||
|
return "", fmt.Errorf("storage: durable id %q: its filesystem %s no longer resolves: %w", durableID, fsUUID, ferr)
|
||||||
|
}
|
||||||
|
if byFS != dev {
|
||||||
|
return "", fmt.Errorf("storage: durable id %q: the part (%s) and its filesystem (%s) are different devices now", durableID, dev, byFS)
|
||||||
|
}
|
||||||
|
return dev, nil
|
||||||
case strings.HasPrefix(durableID, durableByUUID):
|
case strings.HasPrefix(durableID, durableByUUID):
|
||||||
uuid := strings.TrimPrefix(durableID, durableByUUID)
|
uuid := strings.TrimPrefix(durableID, durableByUUID)
|
||||||
if !safeLinkName(uuid) {
|
if !safeLinkName(uuid) {
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ForgetRetiredDrives (R-1124) removes the host mount unit of every DECOMMISSIONED drive that is physically gone, so a
|
||||||
|
// later host boot no longer waits for it and the agent stops listing it (RegistryKnownTargets and
|
||||||
|
// ReassertEnrolledMounts both enumerate the units — no unit, no entry, no „absent by UUID" line every 20 s).
|
||||||
|
//
|
||||||
|
// Only a drive that is BOTH decommissioned (the operator's or the customer's permanent removal) AND absent (its
|
||||||
|
// /dev/disk/by-uuid link does not resolve) AND not mounted is forgotten. A decommissioned drive that is still plugged in
|
||||||
|
// keeps its unit and its raw host mount — that is what makes the one-click re-enroll (H3) work, and decommission stays a
|
||||||
|
// logical retire (`03` §storage). The intent record stays `decommissioned`: a forgotten drive that comes back is a new
|
||||||
|
// candidate, never auto-mounted. The root half (`felhom-priv-apply unit-forget`) re-checks every one of these conditions
|
||||||
|
// and refuses otherwise. Called at agent start, on the 20-second storage tick, and right after a decommission. Best
|
||||||
|
// effort per unit; returns how many units it removed. Pinned by TestForgetRetiredDrives_*.
|
||||||
|
func (h *SudoHostOps) ForgetRetiredDrives(ctx context.Context, intent IntentReader) int {
|
||||||
|
if intent == nil {
|
||||||
|
return 0 // no intent store → we cannot know a drive was retired; forget nothing
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(h.unitDir)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Warn("storage: forget retired drives — cannot read unit dir", "dir", h.unitDir, "err", err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
mounted := map[string]bool{}
|
||||||
|
if ms, merr := h.host.Mounts(); merr == nil {
|
||||||
|
for _, m := range ms {
|
||||||
|
mounted[m.MountPoint] = true
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
h.logger.Warn("storage: forget retired drives — mount table unreadable; forgetting nothing", "err", merr)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
removed := 0
|
||||||
|
for _, e := range entries {
|
||||||
|
name := e.Name()
|
||||||
|
if !strings.HasSuffix(name, ".mount") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
data, rerr := os.ReadFile(filepath.Join(h.unitDir, name))
|
||||||
|
if rerr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
spec, ok := parseFelhomMountUnit(string(data))
|
||||||
|
if !ok {
|
||||||
|
continue // not one of our by-UUID drive units (network units are refused by the parser)
|
||||||
|
}
|
||||||
|
durableID := "uuid:" + spec.UUID
|
||||||
|
if intent.Get(durableID) != IntentDecommissioned {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if mounted[spec.Where] {
|
||||||
|
h.logger.Debug("storage: decommissioned drive still mounted — keeping its unit", "where", spec.Where)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, derr := ResolveStorageDevice(durableID); derr == nil {
|
||||||
|
h.logger.Debug("storage: decommissioned drive is present — keeping its unit (re-enroll stays one click)", "where", spec.Where)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.unitEnabled(name) {
|
||||||
|
if err := h.run(ctx, h.bins.Systemctl, "disable", "--", name); err != nil {
|
||||||
|
h.logger.Warn("storage: forget retired drive — disable failed", "where", spec.Where, "unit", name, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := h.run(ctx, h.bins.PrivApply, "unit-forget", name); err != nil {
|
||||||
|
h.logger.Warn("storage: forget retired drive — unit removal refused or failed", "where", spec.Where, "unit", name, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
removed++
|
||||||
|
h.logger.Info("storage: forgot a decommissioned drive that is gone — its mount unit is removed",
|
||||||
|
"name", spec.Name, "where", spec.Where, "uuid", spec.UUID, "unit", name)
|
||||||
|
}
|
||||||
|
if removed > 0 {
|
||||||
|
if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil {
|
||||||
|
h.logger.Warn("storage: daemon-reload after forgetting retired drives failed", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return removed
|
||||||
|
}
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// forgetRunner records every privileged call and plays the root half of `felhom-priv-apply unit-forget`: the unit file
|
||||||
|
// disappears from the unit dir, so the CONSEQUENCE (the drive is no longer listed) is observable, not just the calls.
|
||||||
|
type forgetRunner struct {
|
||||||
|
unitDir string
|
||||||
|
calls [][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *forgetRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
return r.Run(ctx, name, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *forgetRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
r.calls = append(r.calls, append([]string{name}, args...))
|
||||||
|
if len(args) == 2 && args[0] == "unit-forget" {
|
||||||
|
_ = os.Remove(filepath.Join(r.unitDir, args[1]))
|
||||||
|
}
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type forgetCase struct {
|
||||||
|
intent DriveIntent
|
||||||
|
present bool // the by-uuid link resolves
|
||||||
|
mounted bool
|
||||||
|
enabled bool // the wants-symlink exists
|
||||||
|
mountErr bool
|
||||||
|
}
|
||||||
|
|
||||||
|
const forgetUUID = "4ebfd36d-87d3-4b63-839e-a5a710fc799a"
|
||||||
|
|
||||||
|
func runForget(t *testing.T, c forgetCase, intent IntentReader) (int, *forgetRunner, *RegistryKnownTargets, string) {
|
||||||
|
t.Helper()
|
||||||
|
unitDir := t.TempDir()
|
||||||
|
unit := "mnt-media_a.mount"
|
||||||
|
writeUnit(t, unitDir, unit, renderMountUnit(MountSpec{Name: forgetUUID, UUID: forgetUUID, Where: "/mnt/media_a", FSType: "ext4"}))
|
||||||
|
if c.enabled {
|
||||||
|
if err := os.MkdirAll(filepath.Join(unitDir, "multi-user.target.wants"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Join(unitDir, unit), filepath.Join(unitDir, "multi-user.target.wants", unit)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
devRoot := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(devRoot, "by-uuid"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c.present {
|
||||||
|
dev := filepath.Join(devRoot, "sdz1")
|
||||||
|
if err := os.WriteFile(dev, nil, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(dev, filepath.Join(devRoot, "by-uuid", forgetUUID)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
withDevDiskRoot(t, devRoot)
|
||||||
|
hr := &fakeHostReader{}
|
||||||
|
if c.mounted {
|
||||||
|
hr.mounts = []Mount{{MountPoint: "/mnt/media_a", FSType: "ext4"}}
|
||||||
|
}
|
||||||
|
if c.mountErr {
|
||||||
|
hr.mountsErr = os.ErrPermission
|
||||||
|
}
|
||||||
|
rr := &forgetRunner{unitDir: unitDir}
|
||||||
|
ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir,
|
||||||
|
StageDir: t.TempDir(), Host: hr, Logger: quietLogger()})
|
||||||
|
if intent == nil {
|
||||||
|
intent = &fakeIntent{m: map[string]DriveIntent{"uuid:" + forgetUUID: c.intent}}
|
||||||
|
}
|
||||||
|
n := ops.ForgetRetiredDrives(context.Background(), intent)
|
||||||
|
return n, rr, NewRegistryKnownTargets(unitDir, intent, quietLogger()), unitDir
|
||||||
|
}
|
||||||
|
|
||||||
|
// The R-1124 case: decommissioned, then unplugged. The unit is disabled and removed through the root checker, systemd
|
||||||
|
// is reloaded, and the CONSEQUENCE — the agent no longer lists the drive — holds. RED-PROOF (P1/redproof-forget.txt):
|
||||||
|
// with ForgetRetiredDrives' body replaced by `return 0` this test fails (the drive is still listed).
|
||||||
|
func TestForgetRetiredDrives_DecommissionedAndGone_UnitRemovedAndNotListed(t *testing.T) {
|
||||||
|
n, rr, known, _ := runForget(t, forgetCase{intent: IntentDecommissioned, enabled: true}, nil)
|
||||||
|
if n != 1 {
|
||||||
|
t.Fatalf("removed = %d, want 1 (calls %v)", n, rr.calls)
|
||||||
|
}
|
||||||
|
pa := Binaries{}.withDefaults()
|
||||||
|
want := [][]string{
|
||||||
|
{pa.Systemctl, "disable", "--", "mnt-media_a.mount"},
|
||||||
|
{pa.PrivApply, "unit-forget", "mnt-media_a.mount"},
|
||||||
|
{pa.Systemctl, "daemon-reload"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(rr.calls, want) {
|
||||||
|
t.Fatalf("calls = %v\nwant %v", rr.calls, want)
|
||||||
|
}
|
||||||
|
got, err := known.Known(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("the forgotten drive is still listed: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Already disabled (the Tester 1 shape after a manual disable): no disable call, the removal still runs.
|
||||||
|
func TestForgetRetiredDrives_AlreadyDisabled_NoDisableCall(t *testing.T) {
|
||||||
|
n, rr, _, _ := runForget(t, forgetCase{intent: IntentDecommissioned}, nil)
|
||||||
|
pa := Binaries{}.withDefaults()
|
||||||
|
want := [][]string{{pa.PrivApply, "unit-forget", "mnt-media_a.mount"}, {pa.Systemctl, "daemon-reload"}}
|
||||||
|
if n != 1 || !reflect.DeepEqual(rr.calls, want) {
|
||||||
|
t.Fatalf("n=%d calls=%v want %v", n, rr.calls, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Everything that must KEEP its unit: a decommissioned drive still plugged in (re-enroll is one click — H3), one still
|
||||||
|
// mounted, an enrolled or ejected drive that is merely absent (it comes back), a drive with no recorded intent, and an
|
||||||
|
// unreadable mount table. No privileged call at all, and the drive stays listed where it was listed.
|
||||||
|
// RED-PROOF: with the present-drive check removed, the "present" case calls unit-forget and fails.
|
||||||
|
func TestForgetRetiredDrives_KeepsEveryOtherUnit(t *testing.T) {
|
||||||
|
for name, c := range map[string]forgetCase{
|
||||||
|
"decommissioned but present": {intent: IntentDecommissioned, present: true, enabled: true},
|
||||||
|
"decommissioned but mounted": {intent: IntentDecommissioned, mounted: true},
|
||||||
|
"enrolled and absent": {intent: IntentEnrolled},
|
||||||
|
"ejected and absent": {intent: IntentEjected},
|
||||||
|
"no intent": {intent: IntentNew},
|
||||||
|
"mount table unreadable": {intent: IntentDecommissioned, mountErr: true},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
n, rr, _, unitDir := runForget(t, c, nil)
|
||||||
|
if n != 0 || len(rr.calls) != 0 {
|
||||||
|
t.Fatalf("n=%d calls=%v — this unit must be kept", n, rr.calls)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(unitDir, "mnt-media_a.mount")); err != nil {
|
||||||
|
t.Fatalf("unit file gone: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No intent store on the host (open failure): nothing is known to be retired, so nothing is forgotten.
|
||||||
|
func TestForgetRetiredDrives_NilIntent_ForgetsNothing(t *testing.T) {
|
||||||
|
unitDir := t.TempDir()
|
||||||
|
writeUnit(t, unitDir, "mnt-media_a.mount", renderMountUnit(MountSpec{Name: "x", UUID: forgetUUID, Where: "/mnt/media_a"}))
|
||||||
|
rr := &forgetRunner{unitDir: unitDir}
|
||||||
|
ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, UnitDir: unitDir, StageDir: t.TempDir(), Host: &fakeHostReader{}, Logger: quietLogger()})
|
||||||
|
if n := ops.ForgetRetiredDrives(context.Background(), nil); n != 0 || len(rr.calls) != 0 {
|
||||||
|
t.Fatalf("n=%d calls=%v", n, rr.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
+32
-10
@@ -419,8 +419,9 @@ func (h *SudoHostOps) InspectDevice(ctx context.Context, device string) (DeviceP
|
|||||||
// any TYPE/PTTYPE/USAGE line is positive data-bearing evidence. Its exit code is NOT relied
|
// any TYPE/PTTYPE/USAGE line is positive data-bearing evidence. Its exit code is NOT relied
|
||||||
// on (blkid exits 2 on a blank device) — output presence is what matters. A broken/empty
|
// on (blkid exits 2 on a blank device) — output presence is what matters. A broken/empty
|
||||||
// blkid simply adds no positive evidence; lsblk (below) is the read-success authority.
|
// blkid simply adds no positive evidence; lsblk (below) is the read-success authority.
|
||||||
bout, _, _ := h.runner.Run(ctx, h.bins.Blkid, "-p", "-o", "export", device)
|
bout, _, berr := h.runner.Run(ctx, h.bins.Blkid, "-p", "-o", "export", device)
|
||||||
for k, v := range parseBlkidExport(bout) {
|
blk := parseBlkidExport(bout)
|
||||||
|
for k, v := range blk {
|
||||||
switch k {
|
switch k {
|
||||||
case "TYPE":
|
case "TYPE":
|
||||||
probe.HasFilesystem = true
|
probe.HasFilesystem = true
|
||||||
@@ -439,11 +440,11 @@ func (h *SudoHostOps) InspectDevice(ctx context.Context, device string) (DeviceP
|
|||||||
// lsblk -J is the READ-SUCCESS authority + the partition/mount view. It exits 0 on any valid
|
// lsblk -J is the READ-SUCCESS authority + the partition/mount view. It exits 0 on any valid
|
||||||
// device (blank or not), so a clean parse means the agent reliably read the device. If lsblk
|
// device (blank or not), so a clean parse means the agent reliably read the device. If lsblk
|
||||||
// fails, Probed stays false → DataBearing()=true (fail-safe — never call a device blank on a
|
// fails, Probed stays false → DataBearing()=true (fail-safe — never call a device blank on a
|
||||||
// failed read).
|
// failed read). An exit 0 whose output does not parse into a device is NOT a read either (R-1093
|
||||||
|
// review): it stays unprobed, never "blank".
|
||||||
lout, _, lerr := h.runner.Run(ctx, h.bins.Lsblk, "-J", "-o", "NAME,FSTYPE,PTTYPE,MOUNTPOINT", device)
|
lout, _, lerr := h.runner.Run(ctx, h.bins.Lsblk, "-J", "-o", "NAME,FSTYPE,PTTYPE,MOUNTPOINT", device)
|
||||||
if lerr == nil {
|
if hasChildren, mounted, fstype, pttype, parsed := parseLsblkDevice(lout); lerr == nil && parsed {
|
||||||
probe.Probed = true
|
probe.Probed = true
|
||||||
hasChildren, mounted, fstype, pttype := parseLsblkDevice(lout)
|
|
||||||
if hasChildren {
|
if hasChildren {
|
||||||
probe.HasPartitions = true
|
probe.HasPartitions = true
|
||||||
}
|
}
|
||||||
@@ -456,13 +457,33 @@ func (h *SudoHostOps) InspectDevice(ctx context.Context, device string) (DeviceP
|
|||||||
probe.FSType = fstype
|
probe.FSType = fstype
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if pttype != "" {
|
if pttype != "" && !inheritedPartitionTable(blk, berr, pttype) {
|
||||||
probe.HasPartitionTable = true
|
probe.HasPartitionTable = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return probe, nil
|
return probe, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// inheritedPartitionTable reports whether lsblk's PTTYPE for the probed node is only the PARENT disk's
|
||||||
|
// table (R-1093). lsblk prints the disk's table type on every one of its partitions, so an EMPTY
|
||||||
|
// partition read as "device has a partition table" and asked for the typed erase confirmation.
|
||||||
|
//
|
||||||
|
// The partition's OWN signatures come from `blkid -p` on the partition itself (above): a nested table
|
||||||
|
// inside it is its PTTYPE, a filesystem is its TYPE. So lsblk's value is set aside ONLY when blkid
|
||||||
|
// positively proved two things — that it completed its probe of the device (exit 0: PART_ENTRY_* alone
|
||||||
|
// comes from the PARENT's table, so output without a clean exit does not prove the partition's contents
|
||||||
|
// were read), and that the device is a partition of a table of exactly that type (PART_ENTRY_NUMBER +
|
||||||
|
// PART_ENTRY_SCHEME == lsblk's PTTYPE). Anything else keeps the old, safe reading: a blkid that failed,
|
||||||
|
// exited non-zero or printed nothing, a whole disk, or a scheme that differs from lsblk's (then lsblk sees
|
||||||
|
// a table blkid did not attribute to the parent). Pinned by TestInspect_R1093_* (hostops_disk_test.go).
|
||||||
|
func inheritedPartitionTable(blkid map[string]string, blkidErr error, lsblkPTType string) bool {
|
||||||
|
if blkidErr != nil || blkid["PART_ENTRY_NUMBER"] == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
scheme := blkid["PART_ENTRY_SCHEME"]
|
||||||
|
return scheme != "" && strings.EqualFold(scheme, lsblkPTType)
|
||||||
|
}
|
||||||
|
|
||||||
// Format runs mkfs.<fstype> on a validated device. The caller is responsible for authorization
|
// Format runs mkfs.<fstype> on a validated device. The caller is responsible for authorization
|
||||||
// (8C: only after classifying the device non-data-bearing, or via a slice-10 operator signature).
|
// (8C: only after classifying the device non-data-bearing, or via a slice-10 operator signature).
|
||||||
func (h *SudoHostOps) Format(ctx context.Context, device, fstype string) error {
|
func (h *SudoHostOps) Format(ctx context.Context, device, fstype string) error {
|
||||||
@@ -541,13 +562,14 @@ type lsblkDevice struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parseLsblkDevice parses `lsblk -J -o NAME,FSTYPE,PTTYPE,MOUNTPOINT <device>` for the top device:
|
// parseLsblkDevice parses `lsblk -J -o NAME,FSTYPE,PTTYPE,MOUNTPOINT <device>` for the top device:
|
||||||
// whether it has child partitions, is mounted (itself or any child), and its fstype/pttype.
|
// whether it has child partitions, is mounted (itself or any child), and its fstype/pttype. parsed=false
|
||||||
func parseLsblkDevice(out []byte) (hasChildren, mounted bool, fstype, pttype string) {
|
// when the output is not a device listing — the caller must then treat the device as unprobed.
|
||||||
|
func parseLsblkDevice(out []byte) (hasChildren, mounted bool, fstype, pttype string, parsed bool) {
|
||||||
var doc struct {
|
var doc struct {
|
||||||
BlockDevices []lsblkDevice `json:"blockdevices"`
|
BlockDevices []lsblkDevice `json:"blockdevices"`
|
||||||
}
|
}
|
||||||
if json.Unmarshal(out, &doc) != nil || len(doc.BlockDevices) == 0 {
|
if json.Unmarshal(out, &doc) != nil || len(doc.BlockDevices) == 0 {
|
||||||
return false, false, "", ""
|
return false, false, "", "", false
|
||||||
}
|
}
|
||||||
d := doc.BlockDevices[0]
|
d := doc.BlockDevices[0]
|
||||||
fstype, pttype = d.FSType, d.PTType
|
fstype, pttype = d.FSType, d.PTType
|
||||||
@@ -558,7 +580,7 @@ func parseLsblkDevice(out []byte) (hasChildren, mounted bool, fstype, pttype str
|
|||||||
mounted = true
|
mounted = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return hasChildren, mounted, fstype, pttype
|
return hasChildren, mounted, fstype, pttype, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// NoopHostOps is the safe fallback when the privileged surface is unavailable or declined
|
// NoopHostOps is the safe fallback when the privileged surface is unavailable or declined
|
||||||
|
|||||||
@@ -222,3 +222,114 @@ func TestInspect_ReadsFilesystemUUID(t *testing.T) {
|
|||||||
t.Fatalf("FSUUID = %q, want the blkid UUID", p.FSUUID)
|
t.Fatalf("FSUUID = %q, want the blkid UUID", p.FSUUID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- R-1093: an EMPTY partition is blank; lsblk's PTTYPE on a partition is the PARENT disk's ----------
|
||||||
|
|
||||||
|
// blkidEmptyGPTPart is `blkid -p -o export /dev/sdb1` for an empty GPT partition: only the partition's
|
||||||
|
// own entry in the parent table (PART_ENTRY_*), no TYPE, no PTTYPE — nothing inside the partition.
|
||||||
|
const blkidEmptyGPTPart = "DEVNAME=/dev/sdb1\nPART_ENTRY_SCHEME=gpt\nPART_ENTRY_UUID=3b0c1a52-55a4-4c4e-9d0b-7a1f0e0f7c11\n" +
|
||||||
|
"PART_ENTRY_TYPE=0fc63daf-8483-4772-8e79-3d69d8477de4\nPART_ENTRY_NUMBER=1\nPART_ENTRY_OFFSET=2048\n" +
|
||||||
|
"PART_ENTRY_SIZE=2097152\nPART_ENTRY_DISK=8:16\n"
|
||||||
|
|
||||||
|
// lsblkPartInheritsGPT is `lsblk -J -o NAME,FSTYPE,PTTYPE,MOUNTPOINT /dev/sdb1`: lsblk prints the DISK's
|
||||||
|
// table type on the partition — the input that made the agent read an empty partition as data-bearing.
|
||||||
|
const lsblkPartInheritsGPT = `{"blockdevices":[{"name":"sdb1","fstype":null,"pttype":"gpt","mountpoint":null}]}`
|
||||||
|
|
||||||
|
// TestInspect_R1093_PartitionVerdicts pins the CONSEQUENCE (DataBearing + Reason, what decides whether
|
||||||
|
// the typed erase confirmation is asked) for a partition, in both directions.
|
||||||
|
//
|
||||||
|
// COMPANION RED-PROOFS (observed 2026-10-18): (1) restore the pre-fix `if pttype != "" {` (drop the
|
||||||
|
// inheritedPartitionTable call) → BOTH empty-partition rows (GPT and dos) FAIL with
|
||||||
|
// `DataBearing=true reason="device has a partition table", want blank`. (2) drop the `blkidErr != nil`
|
||||||
|
// check → the "exit 2" and "I/O error" rows FAIL with `DataBearing=false reason="device is blank"`.
|
||||||
|
// (3) drop `&& parsed` → both lsblk-exit-0 rows FAIL with `Probed=true … reason="device is blank"`
|
||||||
|
// (the unparsable-output hole predates R-1093; closed here). Restored.
|
||||||
|
func TestInspect_R1093_PartitionVerdicts(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
dev string
|
||||||
|
blkid string
|
||||||
|
blkidErr error
|
||||||
|
lsblk string
|
||||||
|
lsblkErr error
|
||||||
|
wantBear bool
|
||||||
|
wantReason string
|
||||||
|
unprobed bool // the verdict must be "could not be reliably inspected"
|
||||||
|
}{
|
||||||
|
{name: "empty GPT partition is blank (the R-1093 case)", dev: "/dev/sdb1",
|
||||||
|
blkid: blkidEmptyGPTPart, lsblk: lsblkPartInheritsGPT,
|
||||||
|
wantBear: false, wantReason: "device is blank"},
|
||||||
|
{name: "empty MBR (dos) partition is blank", dev: "/dev/sdb2",
|
||||||
|
blkid: "DEVNAME=/dev/sdb2\nPART_ENTRY_SCHEME=dos\nPART_ENTRY_TYPE=0x83\nPART_ENTRY_NUMBER=2\nPART_ENTRY_DISK=8:16\n",
|
||||||
|
lsblk: `{"blockdevices":[{"name":"sdb2","fstype":null,"pttype":"dos","mountpoint":null}]}`,
|
||||||
|
wantBear: false, wantReason: "device is blank"},
|
||||||
|
{name: "partition with a filesystem stays data-bearing", dev: "/dev/sdb1",
|
||||||
|
blkid: "DEVNAME=/dev/sdb1\nUUID=8c1e\nTYPE=ext4\nUSAGE=filesystem\n" + blkidEmptyGPTPart[len("DEVNAME=/dev/sdb1\n"):],
|
||||||
|
lsblk: `{"blockdevices":[{"name":"sdb1","fstype":"ext4","pttype":"gpt","mountpoint":null}]}`,
|
||||||
|
wantBear: true, wantReason: "device has a ext4 filesystem"},
|
||||||
|
{name: "partition holding its OWN (nested) table stays data-bearing", dev: "/dev/sdb1",
|
||||||
|
blkid: "DEVNAME=/dev/sdb1\nPTUUID=0b8d4358\nPTTYPE=dos\n" + blkidEmptyGPTPart[len("DEVNAME=/dev/sdb1\n"):],
|
||||||
|
lsblk: lsblkPartInheritsGPT,
|
||||||
|
wantBear: true, wantReason: "device has a partition table"},
|
||||||
|
{name: "blkid printed nothing: lsblk's PTTYPE stands (fail-safe)", dev: "/dev/sdb1",
|
||||||
|
blkid: "", blkidErr: errors.New("sudo: a password is required"), lsblk: lsblkPartInheritsGPT,
|
||||||
|
wantBear: true, wantReason: "device has a partition table"},
|
||||||
|
// Fail-closed paths (the 2026-10-18 review): PART_ENTRY_* is read from the PARENT's table, so it
|
||||||
|
// proves nothing about the partition's contents unless blkid also completed (exit 0).
|
||||||
|
{name: "blkid exit 2 with only PART_ENTRY output: lsblk stands", dev: "/dev/sdb1",
|
||||||
|
blkid: blkidEmptyGPTPart, blkidErr: errors.New("exit status 2"), lsblk: lsblkPartInheritsGPT,
|
||||||
|
wantBear: true, wantReason: "device has a partition table"},
|
||||||
|
{name: "blkid I/O error after PART_ENTRY output: lsblk stands", dev: "/dev/sdb1",
|
||||||
|
blkid: blkidEmptyGPTPart, blkidErr: errors.New("exit status 1"), lsblk: lsblkPartInheritsGPT,
|
||||||
|
wantBear: true, wantReason: "device has a partition table"},
|
||||||
|
{name: "blkid binary missing: lsblk stands", dev: "/dev/sdb1",
|
||||||
|
blkid: "", blkidErr: errors.New(`exec: "/usr/sbin/blkid": file does not exist`), lsblk: lsblkPartInheritsGPT,
|
||||||
|
wantBear: true, wantReason: "device has a partition table"},
|
||||||
|
{name: "lsblk exit 0 with unparsable output: unprobed", dev: "/dev/sdb1",
|
||||||
|
blkid: blkidEmptyGPTPart, lsblk: `lsblk: something odd`,
|
||||||
|
wantBear: true, unprobed: true},
|
||||||
|
{name: "lsblk exit 0 with no device listed: unprobed", dev: "/dev/sdb1",
|
||||||
|
blkid: blkidEmptyGPTPart, lsblk: `{"blockdevices":[]}`,
|
||||||
|
wantBear: true, unprobed: true},
|
||||||
|
{name: "lsblk failed: unprobed", dev: "/dev/sdb1",
|
||||||
|
blkid: blkidEmptyGPTPart, lsblk: "", lsblkErr: errors.New("exit status 32"),
|
||||||
|
wantBear: true, unprobed: true},
|
||||||
|
{name: "scheme differs from lsblk's PTTYPE: lsblk stands", dev: "/dev/sdb1",
|
||||||
|
blkid: strings.Replace(blkidEmptyGPTPart, "PART_ENTRY_SCHEME=gpt", "PART_ENTRY_SCHEME=dos", 1),
|
||||||
|
lsblk: lsblkPartInheritsGPT,
|
||||||
|
wantBear: true, wantReason: "device has a partition table"},
|
||||||
|
{name: "whole disk with a GPT stays data-bearing", dev: "/dev/sdb",
|
||||||
|
blkid: "DEVNAME=/dev/sdb\nPTUUID=a07f88ff\nPTTYPE=gpt\n",
|
||||||
|
lsblk: `{"blockdevices":[{"name":"sdb","fstype":null,"pttype":"gpt","mountpoint":null}]}`,
|
||||||
|
wantBear: true, wantReason: "device has a partition table"},
|
||||||
|
{name: "whole disk: blkid silent, lsblk GPT stands", dev: "/dev/sdb",
|
||||||
|
blkid: "", lsblk: `{"blockdevices":[{"name":"sdb","fstype":null,"pttype":"gpt","mountpoint":null}]}`,
|
||||||
|
wantBear: true, wantReason: "device has a partition table"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
r := &scriptedRunner{
|
||||||
|
outputs: map[string][]byte{"blkid": []byte(c.blkid), "lsblk": []byte(c.lsblk)},
|
||||||
|
errs: map[string]error{"blkid": c.blkidErr, "lsblk": c.lsblkErr},
|
||||||
|
}
|
||||||
|
p, err := newSudo(r).InspectDevice(context.Background(), c.dev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c.unprobed {
|
||||||
|
if p.Probed || !p.DataBearing() || p.Reason() != "device could not be reliably inspected" {
|
||||||
|
t.Fatalf("Probed=%v DataBearing=%v reason=%q, want unprobed + data-bearing (probe %+v)",
|
||||||
|
p.Probed, p.DataBearing(), p.Reason(), p)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !p.Probed {
|
||||||
|
t.Fatal("expected a clean probe (lsblk read cleanly)")
|
||||||
|
}
|
||||||
|
if p.DataBearing() != c.wantBear || p.Reason() != c.wantReason {
|
||||||
|
t.Fatalf("DataBearing=%v reason=%q, want DataBearing=%v reason=%q (probe %+v)",
|
||||||
|
p.DataBearing(), p.Reason(), c.wantBear, c.wantReason, p)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The root mkfs wrapper's own tests (configs/test_felhom_mkfs_guarded.py, agent v0.158.0 — a sibling partition that
|
||||||
|
// is Felhom's own drive no longer blocks formatting the part next to it; every other mount still refuses) run with
|
||||||
|
// the Go suite.
|
||||||
|
func TestMkfsGuardedPythonSuite(t *testing.T) {
|
||||||
|
py, err := exec.LookPath("python3")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("python3 not found")
|
||||||
|
}
|
||||||
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
script := filepath.Join(filepath.Dir(file), "..", "..", "configs", "test_felhom_mkfs_guarded.py")
|
||||||
|
out, err := exec.Command(py, script).CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("wrapper suite failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The data-bearing partition path binds the disk id + the part number + the fs-UUID (felhom.eu architecture/16 §6,
|
||||||
|
// agent v0.158.0). The wipe re-resolve (localapi antiRetargetResolveExpect) formats only when resolve(id) succeeds AND
|
||||||
|
// DeviceDurableID(that device) == id — the round trip these tests pin, in the three ways a target can move.
|
||||||
|
|
||||||
|
type partFixture struct{ base, root string }
|
||||||
|
|
||||||
|
func newPartFixture(t *testing.T) partFixture {
|
||||||
|
t.Helper()
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("symlink fixtures run on Linux")
|
||||||
|
}
|
||||||
|
f := partFixture{base: t.TempDir()}
|
||||||
|
f.root = filepath.Join(f.base, "disk")
|
||||||
|
for _, d := range []string{"sdb", "sdb1", "sdb2", "sdc2"} {
|
||||||
|
if err := os.WriteFile(filepath.Join(f.base, d), []byte("x"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
os.MkdirAll(filepath.Join(f.root, "by-id"), 0o755)
|
||||||
|
os.MkdirAll(filepath.Join(f.root, "by-uuid"), 0o755)
|
||||||
|
withDevDiskRoot(t, f.root)
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f partFixture) link(t *testing.T, sub, name, dev string) {
|
||||||
|
t.Helper()
|
||||||
|
p := filepath.Join(f.root, sub, name)
|
||||||
|
os.Remove(p)
|
||||||
|
if err := os.Symlink(filepath.Join(f.base, dev), p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f partFixture) dev(name string) string { return filepath.Join(f.base, name) }
|
||||||
|
|
||||||
|
func (f partFixture) standard(t *testing.T) {
|
||||||
|
f.link(t, "by-id", "wwn-0x5000aaaa", "sdb")
|
||||||
|
f.link(t, "by-id", "ata-TOSHIBA_MQ04_65NOP-part1", "sdb1")
|
||||||
|
f.link(t, "by-id", "wwn-0x5000aaaa-part1", "sdb1")
|
||||||
|
f.link(t, "by-id", "wwn-0x5000aaaa-part2", "sdb2")
|
||||||
|
f.link(t, "by-uuid", "1111-films", "sdb2")
|
||||||
|
}
|
||||||
|
|
||||||
|
func roundTrip(id string) (string, bool) {
|
||||||
|
dev, err := ResolveDurableDevice(id)
|
||||||
|
if err != nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
again, err := DeviceDurableID(dev)
|
||||||
|
return dev, err == nil && again == id
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPartitionWipeID_BindsDiskPartAndFilesystem(t *testing.T) {
|
||||||
|
f := newPartFixture(t)
|
||||||
|
f.standard(t)
|
||||||
|
id, err := DeviceDurableID(f.dev("sdb2"))
|
||||||
|
if err != nil || id != "byid:wwn-0x5000aaaa-part2+fsuuid:1111-films" {
|
||||||
|
t.Fatalf("a part with a filesystem: want disk id + part + fs-UUID, got %q %v", id, err)
|
||||||
|
}
|
||||||
|
if dev, ok := roundTrip(id); !ok || dev != f.dev("sdb2") {
|
||||||
|
t.Fatalf("the unchanged part must round-trip, got %q %v", dev, ok)
|
||||||
|
}
|
||||||
|
// A blank part (no filesystem) and a whole disk keep the plain by-id binding.
|
||||||
|
if id1, _ := DeviceDurableID(f.dev("sdb1")); id1 != "byid:wwn-0x5000aaaa-part1" {
|
||||||
|
t.Fatalf("a blank part: want the plain part link, got %q", id1)
|
||||||
|
}
|
||||||
|
if idDisk, _ := DeviceDurableID(f.dev("sdb")); idDisk != "byid:wwn-0x5000aaaa" {
|
||||||
|
t.Fatalf("a whole disk keeps today's id, got %q", idDisk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPartitionWipeID_ReformattedPartRefuses(t *testing.T) {
|
||||||
|
f := newPartFixture(t)
|
||||||
|
f.standard(t)
|
||||||
|
id, _ := DeviceDurableID(f.dev("sdb2"))
|
||||||
|
os.Remove(filepath.Join(f.root, "by-uuid", "1111-films"))
|
||||||
|
f.link(t, "by-uuid", "2222-other", "sdb2") // someone reformatted the part since the household confirmed
|
||||||
|
if _, ok := roundTrip(id); ok {
|
||||||
|
t.Fatalf("a reformatted part must not round-trip (the wipe would hit a different filesystem)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPartitionWipeID_LinkMovedToAnotherDiskRefuses(t *testing.T) {
|
||||||
|
f := newPartFixture(t)
|
||||||
|
f.standard(t)
|
||||||
|
id, _ := DeviceDurableID(f.dev("sdb2"))
|
||||||
|
f.link(t, "by-id", "wwn-0x5000aaaa-part2", "sdc2") // a different node took the part link; the fs is still sdb2
|
||||||
|
if _, err := ResolveDurableDevice(id); err == nil || !strings.Contains(err.Error(), "different devices") {
|
||||||
|
t.Fatalf("the part link and its filesystem on different devices must refuse, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPartitionWipeID_RefusesAForgedCompound(t *testing.T) {
|
||||||
|
f := newPartFixture(t)
|
||||||
|
f.standard(t)
|
||||||
|
f.link(t, "by-uuid", "3333-whole", "sdb") // a filesystem straight on the whole disk: both halves name sdb
|
||||||
|
for _, id := range []string{"byid:wwn-0x5000aaaa+fsuuid:3333-whole", "byid:wwn-0x5000aaaa-part2+fsuuid:../x"} {
|
||||||
|
if _, err := ResolveDurableDevice(id); err == nil {
|
||||||
|
t.Fatalf("%q must be refused", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Partitions as places (agent v0.158.0) — felhom.eu architecture/16-storage-physical-identity.md §6.
|
||||||
|
//
|
||||||
|
// A disk that passed the WHOLE-disk unclaimed filter (classifyClaim: not the system disk, no LVM/ZFS/RAID/LUKS/swap
|
||||||
|
// member on any node, no foreign mount on any node) also lists its partitions, each with its own state:
|
||||||
|
//
|
||||||
|
// empty no filesystem, no children → „format this part" (self-serve: the agent's probe says blank) —
|
||||||
|
// only while NO part of the disk is mounted (Formattable)
|
||||||
|
// filesystem ext4/xfs (mountable) or another fs → „use this part" when mountable; „format this part" only through
|
||||||
|
// the customer-confirmed data-bearing path (the agent's own probe)
|
||||||
|
// in_use already Felhom's own drive → shown, never offered (it is a registered drive already)
|
||||||
|
//
|
||||||
|
// A partition with children (LVM, a mapper) is not listed. The design's separate „a filesystem with no files" state is
|
||||||
|
// folded into `filesystem` on purpose: telling the two apart needs a root read-only mount, and the data-bearing path's
|
||||||
|
// confirmation is the safer default for both (`16` §13). Creating, deleting or resizing a partition: never here (D5).
|
||||||
|
// The format binding is the partition's own by-id link (…-partN: the disk id + the partition number) via
|
||||||
|
// DeviceDurableID, re-resolved before mkfs (wipe_reresolve.go); the attach key is its fs-UUID.
|
||||||
|
type CandidatePartition struct {
|
||||||
|
Device string `json:"device"` // e.g. /dev/sdb2
|
||||||
|
Number int `json:"number,omitempty"`
|
||||||
|
SizeBytes int64 `json:"size_bytes"`
|
||||||
|
FSType string `json:"fstype,omitempty"`
|
||||||
|
State string `json:"state"` // empty | filesystem | in_use
|
||||||
|
Mountable bool `json:"mountable"`
|
||||||
|
DurableID string `json:"durable_id,omitempty"` // uuid:<fs-uuid> — the attach key ("" when empty)
|
||||||
|
MountedAt string `json:"mounted_at,omitempty"` // in_use: Felhom's own mount
|
||||||
|
// Formattable is false when ANY node of the disk is mounted (another part in use): the root mkfs wrapper refuses
|
||||||
|
// every mount outside /mnt/felhom-drives on the disk and deliberately does not trust a sibling that claims to be
|
||||||
|
// Felhom's own (a compromised agent could fake that). Such a part can still be USED (attach erases nothing).
|
||||||
|
Formattable bool `json:"formattable"`
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
PartEmpty = "empty"
|
||||||
|
PartFilesystem = "filesystem"
|
||||||
|
PartInUse = "in_use"
|
||||||
|
)
|
||||||
|
|
||||||
|
// candidatePartitions is the pure lister: the whole disk's lsblk tree → its partitions. sizeOf/numberOf read /sys,
|
||||||
|
// uuidOf the by-uuid links, owned is the set of mountpoints corroborated as Felhom's own (felhomOwnedMounts).
|
||||||
|
func candidatePartitions(lsblkJSON []byte, wholeName string, sizeOf func(string) int64, numberOf func(string) int,
|
||||||
|
uuidOf func(string) (string, bool), owned map[string]bool) []CandidatePartition {
|
||||||
|
var doc struct {
|
||||||
|
BlockDevices []lsblkDev `json:"blockdevices"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(lsblkJSON, &doc) != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []CandidatePartition
|
||||||
|
for _, d := range doc.BlockDevices {
|
||||||
|
if d.Name != wholeName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
diskMounted := d.MountPoint != ""
|
||||||
|
var walk func(n lsblkDev)
|
||||||
|
walk = func(n lsblkDev) {
|
||||||
|
if n.MountPoint != "" {
|
||||||
|
diskMounted = true
|
||||||
|
}
|
||||||
|
for _, c := range n.Children {
|
||||||
|
walk(c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range d.Children {
|
||||||
|
walk(c)
|
||||||
|
}
|
||||||
|
for _, p := range d.Children {
|
||||||
|
if len(p.Children) > 0 || memberFSTypes[p.FSType] {
|
||||||
|
continue // LVM/mapper on it, or a member: not a place (the disk filter refuses these anyway)
|
||||||
|
}
|
||||||
|
c := CandidatePartition{Device: "/dev/" + p.Name, Number: numberOf(p.Name), SizeBytes: sizeOf(p.Name), FSType: p.FSType}
|
||||||
|
switch {
|
||||||
|
case p.MountPoint != "" && (underFelhomDrives(p.MountPoint) || owned[p.MountPoint]):
|
||||||
|
c.State, c.MountedAt = PartInUse, p.MountPoint
|
||||||
|
case p.MountPoint != "":
|
||||||
|
continue // mounted by something else: never offered (the disk filter refuses the disk anyway)
|
||||||
|
case p.FSType == "":
|
||||||
|
c.State = PartEmpty
|
||||||
|
default:
|
||||||
|
c.State = PartFilesystem
|
||||||
|
c.Mountable = mountableFSTypes[p.FSType]
|
||||||
|
if u, ok := uuidOf(c.Device); ok && u != "" {
|
||||||
|
c.DurableID = "uuid:" + u
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.Formattable = c.State != PartInUse && !diskMounted
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// readSysPartNumber reads /sys/class/block/<part>/partition (0 when unreadable).
|
||||||
|
func readSysPartNumber(name string) int {
|
||||||
|
b, err := os.ReadFile(filepath.Join("/sys/class/block", name, "partition"))
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
n, _ := strconv.Atoi(strings.TrimSpace(string(b)))
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// readSysPartSize reads a partition's size in bytes from /sys/class/block/<part>/size (512-byte sectors).
|
||||||
|
func readSysPartSize(name string) int64 {
|
||||||
|
b, err := os.ReadFile(filepath.Join("/sys/class/block", name, "size"))
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
s, err := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return s * 512
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Partitions as places (felhom.eu architecture/16 §6, agent v0.158.0): every partition of a free disk with its own
|
||||||
|
// state; Felhom's own drive shown but never offered; anything foreign, layered or a member never listed.
|
||||||
|
|
||||||
|
const partTree = `{"blockdevices":[
|
||||||
|
{"name":"sdb","fstype":null,"mountpoint":null,"children":[
|
||||||
|
{"name":"sdb1","fstype":"ext4","mountpoint":"/mnt/felhom-drives/hdd_1"},
|
||||||
|
{"name":"sdb2","fstype":null,"mountpoint":null},
|
||||||
|
{"name":"sdb3","fstype":"ntfs","mountpoint":null},
|
||||||
|
{"name":"sdb4","fstype":"xfs","mountpoint":null},
|
||||||
|
{"name":"sdb5","fstype":"ext4","mountpoint":"/mnt/films"},
|
||||||
|
{"name":"sdb6","fstype":"ext4","mountpoint":"/srv/data"},
|
||||||
|
{"name":"sdb7","fstype":"LVM2_member","mountpoint":null,"children":[{"name":"vg-lv","fstype":"ext4","mountpoint":null}]},
|
||||||
|
{"name":"sdb8","fstype":"crypto_LUKS","mountpoint":null}
|
||||||
|
]},
|
||||||
|
{"name":"sdc","fstype":null,"mountpoint":null,"children":[{"name":"sdc1","fstype":null,"mountpoint":null}]}
|
||||||
|
]}`
|
||||||
|
|
||||||
|
func listParts(owned map[string]bool) map[string]CandidatePartition {
|
||||||
|
got := candidatePartitions([]byte(partTree), "sdb",
|
||||||
|
func(n string) int64 { return int64(len(n)) << 30 },
|
||||||
|
func(n string) int { return int(n[len(n)-1] - '0') },
|
||||||
|
func(dev string) (string, bool) { return "uuid-of-" + strings.TrimPrefix(dev, "/dev/"), true },
|
||||||
|
owned)
|
||||||
|
m := map[string]CandidatePartition{}
|
||||||
|
for _, p := range got {
|
||||||
|
m[p.Device] = p
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPartitions_StatesPerPartition(t *testing.T) {
|
||||||
|
m := listParts(map[string]bool{"/mnt/films": true})
|
||||||
|
want := map[string]string{"/dev/sdb1": PartInUse, "/dev/sdb2": PartEmpty, "/dev/sdb3": PartFilesystem,
|
||||||
|
"/dev/sdb4": PartFilesystem, "/dev/sdb5": PartInUse}
|
||||||
|
if len(m) != len(want) {
|
||||||
|
t.Fatalf("want exactly %d listed partitions, got %+v", len(want), m)
|
||||||
|
}
|
||||||
|
for dev, st := range want {
|
||||||
|
if m[dev].State != st {
|
||||||
|
t.Fatalf("%s: want %s, got %+v", dev, st, m[dev])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m["/dev/sdb2"].DurableID != "" || m["/dev/sdb2"].Mountable {
|
||||||
|
t.Fatalf("an empty part has no attach key: %+v", m["/dev/sdb2"])
|
||||||
|
}
|
||||||
|
if !m["/dev/sdb4"].Mountable || m["/dev/sdb4"].DurableID != "uuid:uuid-of-sdb4" || m["/dev/sdb4"].Number != 4 {
|
||||||
|
t.Fatalf("an xfs part is usable by its fs-UUID: %+v", m["/dev/sdb4"])
|
||||||
|
}
|
||||||
|
if m["/dev/sdb3"].Mountable {
|
||||||
|
t.Fatalf("ntfs is a filesystem the agent does not mount as-is: %+v", m["/dev/sdb3"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A raw /mnt/<name> mount counts as Felhom's own ONLY when corroborated (felhomOwnedMounts); else it is foreign and
|
||||||
|
// the part is not listed. Positive control: TestPartitions_StatesPerPartition (the same mount, corroborated).
|
||||||
|
func TestPartitions_UncorroboratedMountIsNotOffered(t *testing.T) {
|
||||||
|
m := listParts(nil)
|
||||||
|
if _, ok := m["/dev/sdb5"]; ok {
|
||||||
|
t.Fatalf("an uncorroborated /mnt/films mount must not be listed, got %+v", m["/dev/sdb5"])
|
||||||
|
}
|
||||||
|
for _, dev := range []string{"/dev/sdb6", "/dev/sdb7", "/dev/sdb8", "/dev/sdc1"} {
|
||||||
|
if _, ok := m[dev]; ok {
|
||||||
|
t.Fatalf("%s must never be listed (foreign mount, LVM, LUKS, another disk)", dev)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The root wrapper refuses any format on a disk with a mounted node (and trusts no „it is Felhom's own" claim), so a
|
||||||
|
// part next to a mounted one is never offered for formatting — only for use. Positive control: an unmounted disk.
|
||||||
|
func TestPartitions_FormattableOnlyWhenNothingOnTheDiskIsMounted(t *testing.T) {
|
||||||
|
m := listParts(map[string]bool{"/mnt/films": true})
|
||||||
|
for dev, p := range m {
|
||||||
|
if p.Formattable {
|
||||||
|
t.Fatalf("%s: a part of a disk with a mounted part must not be formattable: %+v", dev, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free := `{"blockdevices":[{"name":"sdc","fstype":null,"mountpoint":null,"children":[
|
||||||
|
{"name":"sdc1","fstype":null,"mountpoint":null},{"name":"sdc2","fstype":"ext4","mountpoint":null}]}]}`
|
||||||
|
got := candidatePartitions([]byte(free), "sdc", func(string) int64 { return 1 << 30 }, func(string) int { return 1 },
|
||||||
|
func(string) (string, bool) { return "u", true }, nil)
|
||||||
|
if len(got) != 2 || !got[0].Formattable || !got[1].Formattable {
|
||||||
|
t.Fatalf("parts of an unmounted disk are formattable, got %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,243 @@
|
|||||||
|
// Package thincheck finds an ORPHAN thin device: one the LVM thin pool holds that LVM itself no longer knows (R-1012).
|
||||||
|
//
|
||||||
|
// A full host root while vzdump takes its storage snapshot can leave such a device behind (demo-hp 2026-10-14: the
|
||||||
|
// pool held devices 1, 2 and 4; LVM knew 1 and 2). From then on every snapshot backup of that guest fails at the
|
||||||
|
// snapshot step („Creation of new snapshot 4 of device 2 failed") and nothing in the product said why. This package
|
||||||
|
// NAMES the orphan, in plain words, in the backup's failure — it never removes one: the repair
|
||||||
|
// (`dmsetup message <pool>-tpool 0 "delete <id>"`) stays a human step, decided by the operator (`03` whole-guest
|
||||||
|
// backup; runbook `whole-guest-backup-orphan-thin.md`).
|
||||||
|
//
|
||||||
|
// The device list comes from the root helper `felhom-thin-check <vg> <pool>` (config bundle), which reads the pool
|
||||||
|
// through a metadata snapshot and LVM's list before and after. An id is an orphan only when the pool holds it and
|
||||||
|
// NEITHER LVM list names it, on two reads a few seconds apart — so a volume being created at that moment is never
|
||||||
|
// reported.
|
||||||
|
package thincheck
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HelperPath is where the config bundle installs the root helper.
|
||||||
|
const HelperPath = "/usr/local/sbin/felhom-thin-check"
|
||||||
|
|
||||||
|
// Pool names one LVM thin pool.
|
||||||
|
type Pool struct{ VG, Pool string }
|
||||||
|
|
||||||
|
func (p Pool) String() string { return p.VG + "/" + p.Pool }
|
||||||
|
|
||||||
|
// Reading is one helper run, parsed.
|
||||||
|
type Reading struct {
|
||||||
|
PoolIDs []int
|
||||||
|
LVMBefore map[int]string
|
||||||
|
LVMAfter map[int]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse reads the helper's output. ok=false unless the closing "ok" line is there (a cut-off run is no reading).
|
||||||
|
func Parse(out string) (Reading, bool) {
|
||||||
|
r := Reading{LVMBefore: map[int]string{}, LVMAfter: map[int]string{}}
|
||||||
|
done := false
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
f := strings.Fields(line)
|
||||||
|
switch {
|
||||||
|
case len(f) == 1 && f[0] == "ok":
|
||||||
|
done = true
|
||||||
|
case len(f) == 2 && f[0] == "pool":
|
||||||
|
if id, err := strconv.Atoi(f[1]); err == nil {
|
||||||
|
r.PoolIDs = append(r.PoolIDs, id)
|
||||||
|
}
|
||||||
|
case len(f) == 3 && (f[0] == "lvm-before" || f[0] == "lvm-after"):
|
||||||
|
id, err := strconv.Atoi(f[1])
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if f[0] == "lvm-before" {
|
||||||
|
r.LVMBefore[id] = f[2]
|
||||||
|
} else {
|
||||||
|
r.LVMAfter[id] = f[2]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r, done
|
||||||
|
}
|
||||||
|
|
||||||
|
// Orphans returns the ids the pool holds that neither LVM list names, sorted.
|
||||||
|
func (r Reading) Orphans() []int {
|
||||||
|
var out []int
|
||||||
|
for _, id := range r.PoolIDs {
|
||||||
|
_, b := r.LVMBefore[id]
|
||||||
|
_, a := r.LVMAfter[id]
|
||||||
|
if !b && !a {
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Ints(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runner is the privileged exec surface (proxmox.ExecRunner in sudo mode).
|
||||||
|
type Runner interface {
|
||||||
|
Run(ctx context.Context, name string, args ...string) ([]byte, []byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// StorageLister is the read the checker needs to map a guest's volumes to their thin pools (*proxmox.Client).
|
||||||
|
type StorageLister interface {
|
||||||
|
GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
|
||||||
|
ListStorage(ctx context.Context) ([]proxmox.Storage, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checker runs the check for one guest's pools.
|
||||||
|
type Checker struct {
|
||||||
|
API StorageLister
|
||||||
|
Runner Runner
|
||||||
|
Helper string // "" → HelperPath
|
||||||
|
Pause time.Duration // between the two reads when the first finds an orphan; 0 → 3 s
|
||||||
|
Logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finding is one pool with orphans.
|
||||||
|
type Finding struct {
|
||||||
|
Pool Pool
|
||||||
|
Orphans []int
|
||||||
|
}
|
||||||
|
|
||||||
|
// PoolsOf maps the guest's volume-backed disks (rootfs, mpN) to the lvmthin pools they live on.
|
||||||
|
func (c *Checker) PoolsOf(ctx context.Context, vmid int) ([]Pool, error) {
|
||||||
|
cfg, err := c.API.GuestConfig(ctx, vmid)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
stores, err := c.API.ListStorage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
thin := map[string]Pool{}
|
||||||
|
for _, s := range stores {
|
||||||
|
if s.Type == "lvmthin" && s.VGName != "" && s.ThinPool != "" {
|
||||||
|
thin[s.Storage] = Pool{VG: s.VGName, Pool: s.ThinPool}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
vols := []string{cfg.RootFS}
|
||||||
|
mps := cfg.MountPoints()
|
||||||
|
keys := make([]string, 0, len(mps))
|
||||||
|
for k := range mps {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, k := range keys {
|
||||||
|
vols = append(vols, mps[k])
|
||||||
|
}
|
||||||
|
seen := map[Pool]bool{}
|
||||||
|
var out []Pool
|
||||||
|
for _, v := range vols {
|
||||||
|
store, _, ok := strings.Cut(strings.Split(v, ",")[0], ":")
|
||||||
|
if !ok || strings.HasPrefix(store, "/") {
|
||||||
|
continue // a bind mount (a path) is no volume
|
||||||
|
}
|
||||||
|
if p, isThin := thin[store]; isThin && !seen[p] {
|
||||||
|
seen[p] = true
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// read runs the helper once.
|
||||||
|
func (c *Checker) read(ctx context.Context, p Pool) (Reading, error) {
|
||||||
|
helper := c.Helper
|
||||||
|
if helper == "" {
|
||||||
|
helper = HelperPath
|
||||||
|
}
|
||||||
|
out, stderr, err := c.Runner.Run(ctx, helper, p.VG, p.Pool)
|
||||||
|
if err != nil {
|
||||||
|
return Reading{}, fmt.Errorf("felhom-thin-check %s: %w: %s", p, err, strings.TrimSpace(string(stderr)))
|
||||||
|
}
|
||||||
|
r, ok := Parse(string(out))
|
||||||
|
if !ok {
|
||||||
|
return Reading{}, fmt.Errorf("felhom-thin-check %s: the output has no closing ok line", p)
|
||||||
|
}
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check returns the pools of vmid that hold an orphan (two reads, intersected). A pool that cannot be read is logged
|
||||||
|
// and left out: the check FAILS OPEN — it explains a failure, it never stops a backup.
|
||||||
|
func (c *Checker) Check(ctx context.Context, vmid int) []Finding {
|
||||||
|
logger := c.Logger
|
||||||
|
if logger == nil {
|
||||||
|
logger = slog.Default()
|
||||||
|
}
|
||||||
|
pools, err := c.PoolsOf(ctx, vmid)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("thincheck: cannot map the guest's disks to thin pools — not checked", "vmid", vmid, "err", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
pause := c.Pause
|
||||||
|
if pause <= 0 {
|
||||||
|
pause = 3 * time.Second
|
||||||
|
}
|
||||||
|
var found []Finding
|
||||||
|
for _, p := range pools {
|
||||||
|
first, err := c.read(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("thincheck: thin pool not read — not checked", "vmid", vmid, "pool", p.String(), "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
o1 := first.Orphans()
|
||||||
|
if len(o1) == 0 {
|
||||||
|
logger.Debug("thincheck: thin pool and LVM agree", "vmid", vmid, "pool", p.String(), "devices", len(first.PoolIDs))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return found
|
||||||
|
case <-time.After(pause):
|
||||||
|
}
|
||||||
|
second, err := c.read(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("thincheck: second read failed — not reported", "vmid", vmid, "pool", p.String(), "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
in2 := map[int]bool{}
|
||||||
|
for _, id := range second.Orphans() {
|
||||||
|
in2[id] = true
|
||||||
|
}
|
||||||
|
var both []int
|
||||||
|
for _, id := range o1 {
|
||||||
|
if in2[id] {
|
||||||
|
both = append(both, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(both) > 0 {
|
||||||
|
found = append(found, Finding{Pool: p, Orphans: both})
|
||||||
|
logger.Warn("thincheck: ORPHAN thin device(s) in the pool — LVM does not know them; NOT removed (a human step)",
|
||||||
|
"vmid", vmid, "pool", p.String(), "orphans", both)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
// Describe is the operator sentence for a backup failure. Plain words; names the pool and the ids; says nothing was
|
||||||
|
// removed and where the repair is written down.
|
||||||
|
func Describe(f []Finding) string {
|
||||||
|
if len(f) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var parts []string
|
||||||
|
for _, x := range f {
|
||||||
|
ids := make([]string, len(x.Orphans))
|
||||||
|
for i, id := range x.Orphans {
|
||||||
|
ids[i] = strconv.Itoa(id)
|
||||||
|
}
|
||||||
|
parts = append(parts, fmt.Sprintf("thin pool %s holds device %s that LVM does not know", x.Pool, strings.Join(ids, ", ")))
|
||||||
|
}
|
||||||
|
return "probable cause: " + strings.Join(parts, "; ") +
|
||||||
|
" (an orphan snapshot, usually left when the host disk filled during a backup). Snapshot backups on this pool " +
|
||||||
|
"fail until an operator removes it; Felhom did not remove anything. Repair: runbook whole-guest-backup-orphan-thin.md"
|
||||||
|
}
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
package thincheck
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The helper's output on demo-hp 2026-10-14, before the repair: the pool held 1, 2 and 4; LVM knew 1 and 2
|
||||||
|
// (audits/power-cut-fixes-2026-10-14/E/thin-ls.txt).
|
||||||
|
const orphanOut = `lvm-before 1 vm-9201-disk-0
|
||||||
|
lvm-before 2 vm-9201-disk-1
|
||||||
|
pool 1
|
||||||
|
pool 2
|
||||||
|
pool 4
|
||||||
|
lvm-after 1 vm-9201-disk-0
|
||||||
|
lvm-after 2 vm-9201-disk-1
|
||||||
|
ok
|
||||||
|
`
|
||||||
|
|
||||||
|
const healthyOut = `lvm-before 1 vm-9201-disk-0
|
||||||
|
lvm-before 2 vm-9201-disk-1
|
||||||
|
pool 1
|
||||||
|
pool 2
|
||||||
|
lvm-after 1 vm-9201-disk-0
|
||||||
|
lvm-after 2 vm-9201-disk-1
|
||||||
|
ok
|
||||||
|
`
|
||||||
|
|
||||||
|
func quiet() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }
|
||||||
|
|
||||||
|
func TestParse_TheOrphanOf20261014(t *testing.T) {
|
||||||
|
r, ok := Parse(orphanOut)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("a complete reading was not accepted")
|
||||||
|
}
|
||||||
|
if got := r.Orphans(); !reflect.DeepEqual(got, []int{4}) {
|
||||||
|
t.Fatalf("orphans = %v, want [4]", got)
|
||||||
|
}
|
||||||
|
h, _ := Parse(healthyOut)
|
||||||
|
if got := h.Orphans(); len(got) != 0 {
|
||||||
|
t.Fatalf("a healthy pool reported orphans %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A volume LVM created WHILE the pool was read appears only in the after-list: never an orphan.
|
||||||
|
func TestOrphans_AVolumeCreatedDuringTheReadIsNotAnOrphan(t *testing.T) {
|
||||||
|
r, _ := Parse(strings.Replace(orphanOut, "lvm-after 2 vm-9201-disk-1\n", "lvm-after 2 vm-9201-disk-1\nlvm-after 4 snap_vm-9201-disk-1_vzdump\n", 1))
|
||||||
|
if got := r.Orphans(); len(got) != 0 {
|
||||||
|
t.Fatalf("orphans = %v, want none", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A run cut off before its closing "ok" (the helper died, the output is partial) is no reading at all.
|
||||||
|
func TestParse_NoClosingLineIsNoReading(t *testing.T) {
|
||||||
|
if _, ok := Parse(strings.TrimSuffix(orphanOut, "ok\n")); ok {
|
||||||
|
t.Fatal("a cut-off run was accepted as a reading")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeAPI struct {
|
||||||
|
cfg proxmox.GuestConfig
|
||||||
|
stores []proxmox.Storage
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeAPI) GuestConfig(context.Context, int) (proxmox.GuestConfig, error) { return f.cfg, nil }
|
||||||
|
func (f *fakeAPI) ListStorage(context.Context) ([]proxmox.Storage, error) { return f.stores, nil }
|
||||||
|
|
||||||
|
func guest9201() *fakeAPI {
|
||||||
|
extra := map[string]json.RawMessage{
|
||||||
|
"mp0": json.RawMessage(`"local-lvm:vm-9201-disk-1,mp=/var/lib/felhom,backup=1,size=20G"`),
|
||||||
|
"mp8": json.RawMessage(`"/mnt/felhom-drives,mp=/mnt/felhom-drives"`),
|
||||||
|
"mp9": json.RawMessage(`"local:9201/vm-9201-disk-2.raw,mp=/x"`),
|
||||||
|
}
|
||||||
|
return &fakeAPI{
|
||||||
|
cfg: proxmox.GuestConfig{RootFS: "local-lvm:vm-9201-disk-0,size=8G", Extra: extra},
|
||||||
|
stores: []proxmox.Storage{
|
||||||
|
{Storage: "local", Type: "dir", Path: "/var/lib/vz"},
|
||||||
|
{Storage: "local-lvm", Type: "lvmthin", VGName: "pve", ThinPool: "data"},
|
||||||
|
{Storage: "felhom-pbs", Type: "pbs"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type seqRunner struct {
|
||||||
|
outs []string
|
||||||
|
errs []error
|
||||||
|
calls [][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *seqRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
r.calls = append(r.calls, append([]string{name}, args...))
|
||||||
|
i := len(r.calls) - 1
|
||||||
|
var err error
|
||||||
|
if i < len(r.errs) {
|
||||||
|
err = r.errs[i]
|
||||||
|
}
|
||||||
|
if i < len(r.outs) {
|
||||||
|
return []byte(r.outs[i]), nil, err
|
||||||
|
}
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolsOf_RootfsAndVolumeMountpointsOnly(t *testing.T) {
|
||||||
|
c := &Checker{API: guest9201(), Runner: &seqRunner{}, Logger: quiet()}
|
||||||
|
pools, err := c.PoolsOf(context.Background(), 9201)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(pools, []Pool{{VG: "pve", Pool: "data"}}) {
|
||||||
|
t.Fatalf("pools = %v, want [pve/data] once (bind mounts and dir volumes are not thin)", pools)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consequence: an orphan seen on BOTH reads becomes a finding whose sentence names the pool and the id, and says
|
||||||
|
// nothing was removed. The only command ever run is the read-only helper with <vg> <pool>.
|
||||||
|
// RED-PROOF (P1/redproof-thincheck.txt): with Orphans() returning nil this test fails.
|
||||||
|
func TestCheck_OrphanOnBothReads_IsNamed(t *testing.T) {
|
||||||
|
rr := &seqRunner{outs: []string{orphanOut, orphanOut}}
|
||||||
|
c := &Checker{API: guest9201(), Runner: rr, Pause: 1, Logger: quiet()}
|
||||||
|
f := c.Check(context.Background(), 9201)
|
||||||
|
if len(f) != 1 || f[0].Pool.String() != "pve/data" || !reflect.DeepEqual(f[0].Orphans, []int{4}) {
|
||||||
|
t.Fatalf("findings = %+v", f)
|
||||||
|
}
|
||||||
|
want := [][]string{{HelperPath, "pve", "data"}, {HelperPath, "pve", "data"}}
|
||||||
|
if !reflect.DeepEqual(rr.calls, want) {
|
||||||
|
t.Fatalf("calls = %v, want only the read-only helper twice", rr.calls)
|
||||||
|
}
|
||||||
|
d := Describe(f)
|
||||||
|
for _, s := range []string{"thin pool pve/data holds device 4 that LVM does not know", "Felhom did not remove anything",
|
||||||
|
"whole-guest-backup-orphan-thin.md"} {
|
||||||
|
if !strings.Contains(d, s) {
|
||||||
|
t.Fatalf("Describe = %q, missing %q", d, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An id that is gone on the second read (a volume mid-creation) is not reported; a healthy pool is read ONCE.
|
||||||
|
func TestCheck_TransientOrHealthy_NothingReported(t *testing.T) {
|
||||||
|
rr := &seqRunner{outs: []string{orphanOut, healthyOut}}
|
||||||
|
if f := (&Checker{API: guest9201(), Runner: rr, Pause: 1, Logger: quiet()}).Check(context.Background(), 9201); len(f) != 0 {
|
||||||
|
t.Fatalf("a transient id was reported: %+v", f)
|
||||||
|
}
|
||||||
|
rr = &seqRunner{outs: []string{healthyOut}}
|
||||||
|
if f := (&Checker{API: guest9201(), Runner: rr, Logger: quiet()}).Check(context.Background(), 9201); len(f) != 0 || len(rr.calls) != 1 {
|
||||||
|
t.Fatalf("healthy: findings %+v, calls %v", f, rr.calls)
|
||||||
|
}
|
||||||
|
if Describe(nil) != "" {
|
||||||
|
t.Fatal("no finding must describe as empty")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The helper refusing (a metadata snapshot held by someone else, exit 3) or failing: no finding, no panic — the check
|
||||||
|
// fails open; it explains a failure, it never stops a backup.
|
||||||
|
func TestCheck_HelperRefuses_FailsOpen(t *testing.T) {
|
||||||
|
rr := &seqRunner{errs: []error{errors.New("exit status 3")}}
|
||||||
|
if f := (&Checker{API: guest9201(), Runner: rr, Logger: quiet()}).Check(context.Background(), 9201); len(f) != 0 {
|
||||||
|
t.Fatalf("findings = %+v", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The root helper's own tests (configs/test_felhom_thin_check.py) run with the Go suite.
|
||||||
|
func TestRootHelperPythonSuite(t *testing.T) {
|
||||||
|
py, err := exec.LookPath("python3")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("python3 not found")
|
||||||
|
}
|
||||||
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
script := filepath.Join(filepath.Dir(file), "..", "..", "configs", "test_felhom_thin_check.py")
|
||||||
|
out, err := exec.Command(py, script).CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("helper suite failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,6 +15,9 @@ Gates (all must pass; **non-zero exit on any failure**):
|
|||||||
or because someone is offline on a train. CI runs the FULL set for exactly this
|
or because someone is offline on a train. CI runs the FULL set for exactly this
|
||||||
reason: it is the machine that can afford a network check, and it is the half
|
reason: it is the machine that can afford a network check, and it is the half
|
||||||
that emails when something is wrong.
|
that emails when something is wrong.
|
||||||
|
… gate-decoys the decoy suite (test_gate_decoys.py): every gate here still convicts its decoys
|
||||||
|
(R-1116). ~20 s in the workspace, so the full run only: not the hook, but CI
|
||||||
|
runs the full set on every push. Scratch clones and a local fake Gitea only.
|
||||||
|
|
||||||
WHY THIS FILE EXISTS, WITH ONE GATE (2026-08-02, R-29 leg (b)).
|
WHY THIS FILE EXISTS, WITH ONE GATE (2026-08-02, R-29 leg (b)).
|
||||||
|
|
||||||
@@ -58,6 +61,12 @@ GATES = [
|
|||||||
("release-complete", os.path.join(ROOT, "scripts", "check-release-complete.py"), [], True),
|
("release-complete", os.path.join(ROOT, "scripts", "check-release-complete.py"), [], True),
|
||||||
# R-389 — a REPORT.md observation with no register row behind it. Fast: stdlib file reads.
|
# R-389 — a REPORT.md observation with no register row behind it. Fast: stdlib file reads.
|
||||||
("observations", SHARED_OBSERVATIONS, [ROOT], True),
|
("observations", SHARED_OBSERVATIONS, [ROOT], True),
|
||||||
|
# R-1116 (2026-10-17) — the gates' OWN decoy suite. It was in no runner, the shape that let the catalog's
|
||||||
|
# R-1115 fail unseen for a night. Full run only: ~20 s in the workspace (measured, 30 cases; ~5 s in a
|
||||||
|
# CI-shaped clone, where the shared gates have fewer siblings to search), too slow for the hook — and CI
|
||||||
|
# runs the FULL set, so every push is still judged by it. It plants only in scratch clones and asks a fake
|
||||||
|
# Gitea on 127.0.0.1 (never the network, never the real tree); it does not call this runner (no recursion).
|
||||||
|
("gate-decoys", os.path.join(ROOT, "scripts", "test_gate_decoys.py"), [], False),
|
||||||
]
|
]
|
||||||
|
|
||||||
VERDICT = {0: "OK", 1: "FAILED", 2: "INCONCLUSIVE"}
|
VERDICT = {0: "OK", 1: "FAILED", 2: "INCONCLUSIVE"}
|
||||||
|
|||||||
@@ -69,6 +69,10 @@ fi
|
|||||||
existing="$(curl -fsS -o /dev/null -w '%{http_code}' \
|
existing="$(curl -fsS -o /dev/null -w '%{http_code}' \
|
||||||
"$GITEA_BASE/api/packages/$GITEA_OWNER/generic/felhom-agent/$VERSION/felhom-agent" 2>/dev/null || true)"
|
"$GITEA_BASE/api/packages/$GITEA_OWNER/generic/felhom-agent/$VERSION/felhom-agent" 2>/dev/null || true)"
|
||||||
[[ "$existing" != "200" ]] || die "version $VERSION is ALREADY PUBLISHED — bump the version instead"
|
[[ "$existing" != "200" ]] || die "version $VERSION is ALREADY PUBLISHED — bump the version instead"
|
||||||
|
# R-1098 (2026-10-16): only a clean 404 proves the version is new. Until now a registry that could not be reached
|
||||||
|
# (curl prints 000) or answered anything else passed this check — „I could not ask" is not that proof.
|
||||||
|
# Test: scripts/test_release_push_once.sh.
|
||||||
|
[[ "$existing" == "404" ]] || die "could not establish whether $VERSION is published (registry answered HTTP '${existing:-none}') — refusing"
|
||||||
|
|
||||||
# ── 3. Build ────────────────────────────────────────────────────────────────────────────────────
|
# ── 3. Build ────────────────────────────────────────────────────────────────────────────────────
|
||||||
BIN="$(mktemp -t felhom-agent-XXXXXX)"
|
BIN="$(mktemp -t felhom-agent-XXXXXX)"
|
||||||
|
|||||||
@@ -57,6 +57,20 @@ class AgentGatesTest(unittest.TestCase):
|
|||||||
finally:
|
finally:
|
||||||
shutil.rmtree(tmp, ignore_errors=True)
|
shutil.rmtree(tmp, ignore_errors=True)
|
||||||
|
|
||||||
|
def test_decoy_suite_is_a_full_run_gate(self):
|
||||||
|
"""R-1116: the decoy suite is a gate of the FULL run (which CI runs on every push), and --fast
|
||||||
|
names it as skipped (~20 s, too slow for the hook)."""
|
||||||
|
sys.dont_write_bytecode = True # no __pycache__ left in scripts/ (not gitignored here)
|
||||||
|
sys.path.insert(0, os.path.join(ROOT, "scripts"))
|
||||||
|
import agent_gates as mod
|
||||||
|
rows = [g for g in mod.GATES if g[0] == "gate-decoys"]
|
||||||
|
self.assertEqual(len(rows), 1, "gate-decoys is not registered exactly once")
|
||||||
|
self.assertEqual(os.path.basename(rows[0][1]), "test_gate_decoys.py")
|
||||||
|
self.assertFalse(rows[0][3], "gate-decoys must not be --fast")
|
||||||
|
self.assertEqual([g[0] for g in mod.GATES if not g[3]], ["published", "gate-decoys"])
|
||||||
|
self.assertIn("--fast SKIPPED (deliberate periodic runs, never in a hook): published, gate-decoys",
|
||||||
|
self.out)
|
||||||
|
|
||||||
def test_unknown_argument_is_rejected(self):
|
def test_unknown_argument_is_rejected(self):
|
||||||
p = subprocess.run([sys.executable, ENTRY, "--nope"], cwd=ROOT,
|
p = subprocess.run([sys.executable, ENTRY, "--nope"], cwd=ROOT,
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
|
|||||||
@@ -66,6 +66,9 @@ COVERS = {
|
|||||||
"inside an HTML comment (R-426)",
|
"inside an HTML comment (R-426)",
|
||||||
"observations": "R-419 in THIS repo's REPORT.md: an Observations note SAYING it carries no marker - vs the "
|
"observations": "R-419 in THIS repo's REPORT.md: an Observations note SAYING it carries no marker - vs the "
|
||||||
"two genuine markers (R-426)",
|
"two genuine markers (R-426)",
|
||||||
|
"gate-decoys": "this suite, run as a gate of agent_gates.py's full run, which CI runs on every push (R-1116): "
|
||||||
|
"every case above is a decoy of its own gate, and ONE case that stops convicting turns the run "
|
||||||
|
"red - red-proved 2026-10-17 by flipping a reuse-refs FACT case to expect rc 0",
|
||||||
}
|
}
|
||||||
|
|
||||||
fails = []
|
fails = []
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# test_release_push_once.sh — release-agent.sh releases a version only when the registry says it is NEW (R-1098).
|
||||||
|
#
|
||||||
|
# The REAL release-agent.sh runs from a copy inside a scratch git repo (so its tag check reads scratch tags), with
|
||||||
|
# RELEASE_ALLOW_DIRTY=1 (the clean-tree gate has its own history) and a fake `curl` and `go` first on PATH: curl answers
|
||||||
|
# the package probe as FAKE_MODE says, and `go` stops the run right after the check, so nothing is built or published.
|
||||||
|
# published (200) -> refused, "ALREADY PUBLISHED"
|
||||||
|
# absent (404) -> passes the check, reaches the build
|
||||||
|
# unreachable (000) -> refused, "could not establish"
|
||||||
|
# server error (500) -> refused, "could not establish"
|
||||||
|
# the tag already exists -> refused, "already exists"
|
||||||
|
# Run: bash scripts/test_release_push_once.sh (exit 0 all cases right · 1 a case wrong)
|
||||||
|
set -uo pipefail
|
||||||
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
WS="$(mktemp -d)"; trap 'rm -rf "${WS:?}"' EXIT
|
||||||
|
fails=0; ran=0
|
||||||
|
mkdir -p "$WS/bin" "$WS/repo/scripts"
|
||||||
|
cp "$HERE/release-agent.sh" "$WS/repo/scripts/release-agent.sh"
|
||||||
|
(cd "$WS/repo" && git init -q -b main && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m init)
|
||||||
|
cat > "$WS/bin/curl" <<'FAKE'
|
||||||
|
#!/bin/bash
|
||||||
|
case "${FAKE_MODE}" in
|
||||||
|
published) printf 200; exit 0 ;;
|
||||||
|
absent) printf 404; exit 22 ;;
|
||||||
|
down) printf 000; exit 7 ;;
|
||||||
|
error) printf 500; exit 22 ;;
|
||||||
|
esac
|
||||||
|
FAKE
|
||||||
|
printf '#!/bin/sh\necho "FAKE-GO-REACHED $*"; exit 1\n' > "$WS/bin/go"
|
||||||
|
chmod +x "$WS/bin/curl" "$WS/bin/go"
|
||||||
|
run() { # name, mode, expected rc, required fragment, forbidden fragment
|
||||||
|
local out rc
|
||||||
|
out=$(cd "$WS/repo" && FAKE_MODE="$2" RELEASE_ALLOW_DIRTY=1 GITEA_USER=u GITEA_TOKEN=t PATH="$WS/bin:$PATH" \
|
||||||
|
bash scripts/release-agent.sh 9.9.9 2>&1); rc=$?
|
||||||
|
ran=$((ran+1))
|
||||||
|
if [[ "$rc" != "$3" ]] || ! grep -qi -- "$4" <<<"$out" || { [[ -n "$5" ]] && grep -qi -- "$5" <<<"$out"; }; then
|
||||||
|
fails=$((fails+1)); echo " FAIL $1 (exit $rc)"; echo "$out" | tail -4 | sed 's/^/ /'; return
|
||||||
|
fi
|
||||||
|
echo " ok $1"
|
||||||
|
}
|
||||||
|
echo "release-agent.sh — a version is released once, and only when the registry says it is new"
|
||||||
|
run "published (200) -> refused" published 1 "ALREADY PUBLISHED" "FAKE-GO-REACHED"
|
||||||
|
run "absent (404) -> proceeds" absent 1 "FAKE-GO-REACHED" "could not establish"
|
||||||
|
run "unreachable (000) -> refused" down 1 "could not establish" "FAKE-GO-REACHED"
|
||||||
|
run "server error (500) -> refused" error 1 "could not establish" "FAKE-GO-REACHED"
|
||||||
|
(cd "$WS/repo" && git tag v9.9.9)
|
||||||
|
run "the tag exists -> refused" absent 1 "already exists" "FAKE-GO-REACHED"
|
||||||
|
echo
|
||||||
|
if [[ $fails -eq 0 ]]; then echo "all $ran cases behaved"; exit 0; fi
|
||||||
|
echo "$fails of $ran cases FAILED"; exit 1
|
||||||
Reference in New Issue
Block a user