Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 65c1994d3b | |||
| 6545cccff7 | |||
| eb38d78945 | |||
| 4a1c07b288 | |||
| 5c3b218822 | |||
| e15d1d30ea | |||
| 344c4f9030 | |||
| c52d591851 | |||
| 37b99d73b3 | |||
| dd8b2ace02 | |||
| 383522450a | |||
| 1a8bed11e7 |
@@ -16,7 +16,16 @@
|
|||||||
# A failing run must reach a person — a detector nobody hears is the defect R-29 filed, rebuilt one
|
# A failing run must reach a person — a detector nobody hears is the defect R-29 filed, rebuilt one
|
||||||
# layer up. That is the last step, and it runs ONLY on failure.
|
# layer up. That is the last step, and it runs ONLY on failure.
|
||||||
name: gates
|
name: gates
|
||||||
on: [push]
|
# R-1129: `workflow_dispatch` is the manual PROOF that the failure mail can be sent. A push run is
|
||||||
|
# unchanged by it: the proof step below is skipped on a push, and `force_alarm` is then empty.
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
force_alarm:
|
||||||
|
description: "Fail this run on purpose so the alarm step sends its real mail (a proof, not a gate result)"
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
gates:
|
gates:
|
||||||
@@ -57,6 +66,22 @@ jobs:
|
|||||||
GITEA_BASE: http://gitea.gitea-system.svc.cluster.local:3000
|
GITEA_BASE: http://gitea.gitea-system.svc.cluster.local:3000
|
||||||
run: cd ws/felhom-agent && python3 scripts/agent_gates.py
|
run: cd ws/felhom-agent && python3 scripts/agent_gates.py
|
||||||
|
|
||||||
|
- name: Manual proof - alarm key present, optional forced failure (dispatch only, R-1129)
|
||||||
|
# Runs ONLY on a manual dispatch, never on a push. It prints the key's LENGTH, never the key,
|
||||||
|
# so "CI can see the secret" is an observable rather than an assumption. With
|
||||||
|
# force_alarm=true it then fails on purpose so the alarm step below sends its real mail;
|
||||||
|
# the commit itself stays green, because its push run skips this step.
|
||||||
|
if: github.event_name == 'workflow_dispatch'
|
||||||
|
env:
|
||||||
|
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
||||||
|
FORCE_ALARM: ${{ github.event.inputs.force_alarm }}
|
||||||
|
run: |
|
||||||
|
python3 -c 'import os;print("RESEND_API_KEY length: %d" % len(os.environ.get("RESEND_API_KEY","")))'
|
||||||
|
if [ "$FORCE_ALARM" = "true" ]; then
|
||||||
|
echo "force_alarm=true - failing on purpose so the alarm step runs (R-1129 proof)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Alarm on failure
|
- name: Alarm on failure
|
||||||
# THE POINT OF THE WHOLE THING. Probe P5 measured that a failed run produces NO mail, NO
|
# THE POINT OF THE WHOLE THING. Probe P5 measured that a failed run produces NO mail, NO
|
||||||
# notification row and NO log line from Gitea itself — a red tick in a web UI nobody watches
|
# notification row and NO log line from Gitea itself — a red tick in a web UI nobody watches
|
||||||
@@ -70,6 +95,8 @@ jobs:
|
|||||||
if: failure()
|
if: failure()
|
||||||
env:
|
env:
|
||||||
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
||||||
|
# Empty on a push, so a push run's mail is exactly what it was before R-1129.
|
||||||
|
FORCE_ALARM: ${{ github.event.inputs.force_alarm }}
|
||||||
run: |
|
run: |
|
||||||
python3 - <<'PY'
|
python3 - <<'PY'
|
||||||
import json, os, sys, urllib.request, urllib.error
|
import json, os, sys, urllib.request, urllib.error
|
||||||
@@ -87,7 +114,9 @@ jobs:
|
|||||||
body = json.dumps({
|
body = json.dumps({
|
||||||
"from": "Felhom CI <monitoring@felhom.eu>",
|
"from": "Felhom CI <monitoring@felhom.eu>",
|
||||||
"to": ["admin@felhom.eu"],
|
"to": ["admin@felhom.eu"],
|
||||||
"subject": "[felhom CI] gates FAILED in %s" % repo,
|
"subject": ("[felhom CI] FORCED TEST, not a real failure - gates FAILED in %s"
|
||||||
|
if os.environ.get("FORCE_ALARM") == "true"
|
||||||
|
else "[felhom CI] gates FAILED in %s") % repo,
|
||||||
"text": (
|
"text": (
|
||||||
"The gate entry point exited non-zero.\n\n"
|
"The gate entry point exited non-zero.\n\n"
|
||||||
"Repository : %s\n"
|
"Repository : %s\n"
|
||||||
|
|||||||
+113
-4
@@ -1,3 +1,112 @@
|
|||||||
|
## Unreleased — the orphan thin check runs only after a failed backup (R-1012 follow-up) (2026-10-17)
|
||||||
|
|
||||||
|
**Not released, not vouched; the next vouched agent release carries it.** v0.159.0/v0.159.1 ran the R-1012 orphan check
|
||||||
|
before EVERY snapshot-mode whole-guest backup, and that check takes a short metadata snapshot of the live thin pool.
|
||||||
|
Now `BackupRunner` runs it only when the snapshot step has failed (the vzdump start or its task) — the moment the
|
||||||
|
answer is needed. The finding is still appended in plain words to the failed backup's error, exactly as before; a
|
||||||
|
successful backup never runs the check and keeps a clean record; stop mode and an unwired runner are unchanged. The
|
||||||
|
`thin-orphan-check` capability text says „after a failed whole-guest backup". Tests
|
||||||
|
`TestBackup_SnapshotFailure_NamesTheOrphan` (now: exactly 1 call, after the failure) and
|
||||||
|
`TestBackup_ThinCheck_SilentWhenNotApplicable` (now: 0 calls on a successful backup), red-proved against the v0.159.1
|
||||||
|
`runner.go`: failure legs „thin check ran 2 times, want 1", success leg `calls=1`. No root file, sudoers or bundle
|
||||||
|
change. (Installer side, felhom.eu, same night: `felhom-host-install.sh --rotate-recovery-only`, R-1141.)
|
||||||
|
**Owed by the release that carries it (not yet proven live):** one green snapshot backup with NO `felhom-thin-check`
|
||||||
|
call in the journal, and one failed snapshot backup whose error carries the finding.
|
||||||
|
|
||||||
|
## v0.159.1 — the signed decommission prunes the guest-bind record (found by v0.159.0's own live proof) (2026-10-17)
|
||||||
|
|
||||||
|
**v0.159.0 is superseded — do not vouch it.** Everything of v0.159.0 below, plus: `signedjobs.DecommissionExecutor`
|
||||||
|
drops the drive from every guest's bind record (`SetGuestBinds`, wired to the agent's `GuestBindStore`), as the
|
||||||
|
self-serve decommission always did. Without it a drive decommissioned by an operator-signed job and then forgotten
|
||||||
|
(R-1124) left a record the 20-second guest-bind re-assert warned about every tick („reconcile: skipping non-enrolled
|
||||||
|
drive") — seen on demo-hp minutes after the R-1124 proof. Test `TestDecommissionExecutor_PrunesTheGuestBindRecord`,
|
||||||
|
red-proved. Released: tag `v0.159.1`, sha256 `de8f6ad9…aff9`, bundle `2af3d27d…7cd6` (the same root files as v0.159.0).
|
||||||
|
**Not vouched.** Delivery to a box on the v0.158.1 bundle: `agent_update 0.159.1` → `agent_config_update 0.159.0-step1`
|
||||||
|
(sha `1d65edb3…d6cf`, published) → `agent_config_update 0.159.1` (the bundle adds `felhom-thin-check`, R16).
|
||||||
|
|
||||||
|
## v0.159.0 — forget a decommissioned drive that is gone (R-1124); name an orphan thin device in a backup failure (R-1012); Go 1.27.2 (R-972) (2026-10-17)
|
||||||
|
|
||||||
|
**Superseded by v0.159.1 — do not vouch.** Task: „agent release, the catalog allow-list, break-glass rotation, off-site
|
||||||
|
history check" (felhom.eu `REPORT-agent-allowlist-rotation-2026-10-17.md`).
|
||||||
|
|
||||||
|
- **Go 1.27.2 + x/crypto v0.58.0** (R-972, `37b99d7`): the first release that carries them (v0.158.1 was built with
|
||||||
|
go1.26.0).
|
||||||
|
- **R-1124 — a decommissioned drive that is gone is forgotten.** `SudoHostOps.ForgetRetiredDrives` (agent start and the
|
||||||
|
20-second storage tick): for a Felhom drive unit whose intent is `decommissioned`, whose `/dev/disk/by-uuid` link is
|
||||||
|
gone and which is not mounted, `systemctl disable` + the new root verb `felhom-priv-apply unit-forget <unit>` +
|
||||||
|
`daemon-reload`. The unit is gone, so a later boot no longer waits for the drive and the agent stops listing it (no
|
||||||
|
„absent by UUID" line every 20 s). A decommissioned drive still plugged in KEEPS its unit and raw mount, so the
|
||||||
|
one-click re-enroll (H3) still works. `unit-forget` re-checks as root: a felhom by-UUID drive unit (not a network
|
||||||
|
unit), disabled, its `Where` not mounted, its by-UUID link absent — otherwise refused. Sudoers `FELHOM_MOUNT` gains the
|
||||||
|
one verb. Tests `TestForgetRetiredDrives_*` and `configs/test_felhom_priv_apply.py` `UnitForget`, red-proved.
|
||||||
|
**Live on demo-hp:** a loop-backed test drive enrolled through the local API, decommissioned by a signed job (kept
|
||||||
|
its unit while present), unplugged → 8 s later the unit and its wants link were removed and `GET /disks` dropped it.
|
||||||
|
- **R-1012 — an orphan thin device is NAMED in the backup failure, never removed.** New READ-ONLY root helper
|
||||||
|
`felhom-thin-check <vg> <pool>` (config bundle): LVM's thin ids before and after, the pool's ids through a metadata
|
||||||
|
snapshot it reserves and releases (refuses — exit 3 — when someone else holds one; releases on every exit; sends no
|
||||||
|
other pool message). `internal/thincheck`: an orphan = in the pool and in neither LVM list, on two reads 3 s apart;
|
||||||
|
fails open. `BackupRunner.SetThinCheck`: before a snapshot-mode backup (a WARN when found) and again after a failure —
|
||||||
|
the finding is appended in plain words to the error the controller mails to the operator, pointing at the new
|
||||||
|
runbook `whole-guest-backup-orphan-thin.md`. Sudoers `FELHOM_THINCHECK`, capability `thin-orphan-check`. Tests
|
||||||
|
`internal/thincheck`, `TestBackup_SnapshotFailure_NamesTheOrphan`, `configs/test_felhom_thin_check.py`, red-proved.
|
||||||
|
**Live on demo-hp:** on a scratch thin pool (a loop file on `/mnt/hdd_1`, never `pve/data`) the helper listed a made
|
||||||
|
orphan (device 7) and refused a snapshot held by someone else; on `pve/data` it read 1, 2 = LVM's and released; a
|
||||||
|
normal whole-guest backup of 9201 ran the check first and finished green.
|
||||||
|
- Released: tag `v0.159.0`, sha256 `3cfd0a25…edfed5`, bundle `40663c4f…6d681`; step bundle `0.159.0-step1`.
|
||||||
|
|
||||||
|
## CI only — a manual proof that the failure mail can be sent (R-1129, 2026-10-17)
|
||||||
|
|
||||||
|
No agent release; no binary changes.
|
||||||
|
|
||||||
|
`.gitea/workflows/gates.yml` gains a manual `workflow_dispatch` trigger with one input, `force_alarm` (default false), and
|
||||||
|
one step that runs ONLY on a dispatch: it prints the length of `RESEND_API_KEY` (never the key) and, with
|
||||||
|
`force_alarm=true`, fails on purpose so the existing alarm step sends its real mail; the mail's subject then starts
|
||||||
|
„FORCED TEST, not a real failure". A push run is unchanged: the new step is skipped and the subject is the old one. Same
|
||||||
|
change in all four repos' workflows. Purpose: prove the failure mail now goes out after the operator set the key
|
||||||
|
(every failed run until today ended „ALARM FAILED: RESEND_API_KEY is empty").
|
||||||
|
|
||||||
|
## scripts only — the decoy suite is a gate of the full run (R-1116, 2026-10-17)
|
||||||
|
|
||||||
|
No agent release; no binary changes. `scripts/test_gate_decoys.py` is now the `gate-decoys` row of `agent_gates.py`:
|
||||||
|
the full run (which CI runs on every push), not `--fast` — it takes ~20 s in the workspace (measured: 30 cases; ~5 s in a
|
||||||
|
CI-shaped shallow clone, also green). It was in no runner, the shape that let the catalog's R-1115 fail unseen for a
|
||||||
|
night. It plants only in scratch clones and asks a fake Gitea on 127.0.0.1; it does not call the runner (no
|
||||||
|
recursion). Declared in its own `COVERS`; `test_agent_gates.py` pins it (registered once, full run only, named by
|
||||||
|
`--fast`'s SKIPPED line). Red-proved: a reuse-refs FACT case flipped to expect rc 0 turned the full run red
|
||||||
|
(`CONVICTED: gate-decoys`); restored, green. CLAUDE.md's `--fast` sentence names it.
|
||||||
|
|
||||||
|
## unreleased — Go 1.27.2 and golang.org/x/crypto v0.58.0 (R-972, 2026-10-16)
|
||||||
|
|
||||||
|
**Not released, not deployed** — agent releases wait for the operator's vouch; the next vouched release carries this.
|
||||||
|
`go.mod`: `go 1.27.2` (was 1.25.0; the shipped 0.158.1 binary was built with go1.26.0), x/crypto v0.58.0 (was v0.52.0),
|
||||||
|
x/sys v0.49.0. `go build`, `go vet`, the full `go test ./...` and `agent_gates.py --fast` green on Go 1.27.2.
|
||||||
|
`release-agent.sh` builds with the toolchain `go.mod` names (GOTOOLCHAIN=auto), so no script change.
|
||||||
|
|
||||||
|
## scripts only — `release-agent.sh` refuses when the registry cannot say the version is new (R-1098, 2026-10-16)
|
||||||
|
|
||||||
|
No agent release; no binary changes. `scripts/release-agent.sh` already refused a version whose tag exists or whose
|
||||||
|
package answers 200. It let everything else through — a registry it could not reach (curl prints `000`) or a 500 read
|
||||||
|
as „new". Now only a clean `404` proves the version is new; anything else refuses with the code it got. New
|
||||||
|
`scripts/test_release_push_once.sh` (the real script, a fake `curl` and `go` on PATH; published / absent / unreachable /
|
||||||
|
server error / tag exists), red-proved: with the 404 rule removed, the unreachable and server-error cases fail. Same rule
|
||||||
|
as the controller `build.sh` and, since today, felhom.eu `scripts/build-hub.sh`.
|
||||||
|
|
||||||
|
## v0.158.1 — security review of v0.158.0: the mkfs wrapper trusts no sibling claim; the internal-place helper fails closed (2026-10-16)
|
||||||
|
|
||||||
|
**v0.158.0 is superseded — do not vouch it.** A security review of `912f7c1` found two defects; both fixed here, each test
|
||||||
|
seen failing against the 0.158.0 script:
|
||||||
|
- **`felhom-mkfs-guarded`**: 0.158.0 let a format through when a SIBLING part was mounted at /mnt/<name> AND under
|
||||||
|
/mnt/felhom-drives („Felhom's own drive"). That corroboration is a mount the agent can make itself, so a compromised agent
|
||||||
|
could unlock a host owner's disk — the exact case this root wrapper exists for. Removed: any mount of any node outside
|
||||||
|
/mnt/felhom-drives refuses again (now read from the kernel's mount table for every node; an unlistable disk refuses).
|
||||||
|
Consequence: the free part next to an in-use Felhom drive cannot be formatted — `CandidatePartition.Formattable` is false
|
||||||
|
for every part of a disk with a mounted node, and the controller does not offer it (a part with a filesystem can still be
|
||||||
|
USED). Pinned by configs/test_felhom_mkfs_guarded.py + partitions_test.go.
|
||||||
|
- **`felhom-internal-place`**: a failed `lvs` listing read as „nothing promised" (a too-large place) and a `blkid` error read
|
||||||
|
as „no signature" (the only gate before mkfs). Now: a failed listing, a listing without the pool, or any blkid exit other
|
||||||
|
than 0/2 REFUSES. Three new tests, all failing against 0.158.0.
|
||||||
|
- Released: tag `v0.158.1` (code `98f3cdd`), sha256 `20264df6…84b76a1`, bundle `c23c3d35…cb1c41b`. **Not vouched.**
|
||||||
|
|
||||||
## v0.158.0 — the internal data place; partitions as places; add_internal_place (2026-10-16)
|
## v0.158.0 — the internal data place; partitions as places; add_internal_place (2026-10-16)
|
||||||
|
|
||||||
Design: felhom.eu `architecture/16-storage-physical-identity.md` §5, §6, §7, §9 (operator rulings D3–D5; the D3 sizes
|
Design: felhom.eu `architecture/16-storage-physical-identity.md` §5, §6, §7, §9 (operator rulings D3–D5; the D3 sizes
|
||||||
@@ -17,12 +126,12 @@ ruled 2026-10-16 after the space check, felhom.eu `audits/storage-identity-build
|
|||||||
- **Partitions as places** (`internal/storage/partitions.go`): `GET /disks/candidates` lists each partition of a free
|
- **Partitions as places** (`internal/storage/partitions.go`): `GET /disks/candidates` lists each partition of a free
|
||||||
disk as empty / filesystem / in_use. A data-bearing part's wipe id binds disk id + part number + fs-UUID
|
disk as empty / filesystem / in_use. A data-bearing part's wipe id binds disk id + part number + fs-UUID
|
||||||
(`byid:<…-partN>+fsuuid:<uuid>`); both must name the same device at format time.
|
(`byid:<…-partN>+fsuuid:<uuid>`); both must name the same device at format time.
|
||||||
- **`felhom-mkfs-guarded`**: a SIBLING part that is Felhom's own drive (mounted raw at /mnt/<name> AND under
|
- **`felhom-mkfs-guarded`** (REVERTED in v0.158.1): a SIBLING part that is Felhom's own drive no longer blocked formatting
|
||||||
/mnt/felhom-drives) no longer blocks formatting the free part next to it; every other mount still refuses; an
|
the free part next to it. First tests of the wrapper: `configs/test_felhom_mkfs_guarded.py`.
|
||||||
unlistable disk now refuses (it passed before). First tests of the wrapper: `configs/test_felhom_mkfs_guarded.py`.
|
|
||||||
- `configs/` changed (sudoers `FELHOM_INTERNALPLACE`, the helper, the wrapper, the bundle list) — **a config-bundle job
|
- `configs/` changed (sudoers `FELHOM_INTERNALPLACE`, the helper, the wrapper, the bundle list) — **a config-bundle job
|
||||||
is needed** with the agent update. **Not vouched** (the operator's act).
|
is needed** with the agent update. **Not vouched** (the operator's act).
|
||||||
- Released: tag `v0.158.0` (code `912f7c1`), sha256 `a3589d1d…2caba`, bundle `17066273…fb5c76`.
|
- Released: tag `v0.158.0` (code `912f7c1`), sha256 `a3589d1d…2caba`, bundle `17066273…fb5c76`. **Superseded by v0.158.1 —
|
||||||
|
do not vouch** (its mkfs wrapper trusted a sibling claim the agent can fake; its helper failed open).
|
||||||
|
|
||||||
## v0.157.0 — every storage place knows its physical disk(s); POST /disks/same-disk (2026-10-15)
|
## v0.157.0 — every storage place knows its physical disk(s); POST /disks/same-disk (2026-10-15)
|
||||||
|
|
||||||
|
|||||||
@@ -56,7 +56,8 @@ This is in the core because breaching it is how this component stops being audit
|
|||||||
gate in its `GATES` table (that table is the list); the shared ones — `reuse_refs_check`,
|
gate in its `GATES` table (that table is the list); the shared ones — `reuse_refs_check`,
|
||||||
`instructions_gate`, `observations_gate` — are the copies in `felhom.eu/scripts/`, never copied into
|
`instructions_gate`, `observations_gate` — are the copies in `felhom.eu/scripts/`, never copied into
|
||||||
this repo (a copy recreates the drift they detect; an absent sibling clone FAILS). `--fast` selects the
|
this repo (a copy recreates the drift they detect; an absent sibling clone FAILS). `--fast` selects the
|
||||||
gates touching no network and no container runtime, and skips `published` (network), naming it. **A missing gate is a FAILURE, never a skip.**
|
gates touching no network and no container runtime, and skips `published` (network) and `gate-decoys` (the decoy
|
||||||
|
suite, ~20 s; R-1116), naming them — CI runs the full set. **A missing gate is a FAILURE, never a skip.**
|
||||||
|
|
||||||
**The pre-push hook** (`.githooks/pre-push`) runs it with `--fast` and refuses a failing push. It is
|
**The pre-push hook** (`.githooks/pre-push`) runs it with `--fast` and refuses a failing push. It is
|
||||||
**per-clone** — switch it on once with `git config core.hooksPath .githooks`, and a manual run WARNS
|
**per-clone** — switch it on once with `git config core.hooksPath .githooks`, and a manual run WARNS
|
||||||
|
|||||||
+11
@@ -1,5 +1,16 @@
|
|||||||
# CONTEXT — felhom-agent working state
|
# CONTEXT — felhom-agent working state
|
||||||
|
|
||||||
|
> **2026-10-17 — v0.159.1 RELEASED (0.159.0 superseded, do not vouch).** Go 1.27.2; R-1124 a decommissioned drive that is
|
||||||
|
> gone is forgotten (`ForgetRetiredDrives`, root verb `unit-forget`); R-1012 the read-only `felhom-thin-check` names an
|
||||||
|
> orphan thin device in a failed backup's text; the signed decommission prunes the guest-bind record. Bundle adds a path →
|
||||||
|
> step bundle `0.159.0-step1`. Proven on demo-hp; delivered by signed jobs on the operator's yes. Vouch: the operator's.
|
||||||
|
|
||||||
|
> **2026-10-16 — v0.158.1 RELEASED and VOUCHED (storage identity build B; 0.158.0 superseded, do not vouch).** The internal
|
||||||
|
> data place (root helper `felhom-internal-place`, the record in the state dir, mounted at /mnt/belso like a drive), the
|
||||||
|
> signed `add_internal_place`, partitions in `/disks/candidates` (`formattable` only while nothing on the disk is mounted),
|
||||||
|
> the partition wipe id `byid:<…-partN>+fsuuid:<uuid>`, the strict mkfs wrapper. Delivered through step bundle
|
||||||
|
> `0.158.1-step1` (a bundle that adds a path, R16). Next: R-1093 (an empty part's probe reads the parent's partition table).
|
||||||
|
|
||||||
> **2026-10-15 (night) — v0.157.0 RELEASED, NOT VOUCHED (storage identity build A, felhom.eu `16` §13).** Every storage
|
> **2026-10-15 (night) — v0.157.0 RELEASED, NOT VOUCHED (storage identity build A, felhom.eu `16` §13).** Every storage
|
||||||
> place carries `kind`, `physical_disks`, `identity` (+ reason) in `GET /disks` and the hub report; `POST /disks/same-disk`
|
> place carries `kind`, `physical_disks`, `identity` (+ reason) in `GET /disks` and the hub report; `POST /disks/same-disk`
|
||||||
> answers the controller (v0.317.0, MinAgent 0.157.0). Computed on every read, never cached. Delivered by signed jobs to
|
> answers the controller (v0.317.0, MinAgent 0.157.0). Computed on every read, never cached. Delivered by signed jobs to
|
||||||
|
|||||||
@@ -1,43 +1,41 @@
|
|||||||
# REPORT — v0.157.0: every storage place knows its physical disk(s); POST /disks/same-disk (2026-10-15)
|
# REPORT — the orphan thin check runs only after a failed backup (unreleased) (2026-10-17, night)
|
||||||
|
|
||||||
Written in ASD-STE100 Simplified Technical English. Design read first: felhom.eu `architecture/16-storage-physical-identity.md`
|
Written in ASD-STE100 Simplified Technical English. Architecture read: felhom.eu `documentation/architecture/03-host-agent.md`
|
||||||
(§3, §4, §8, §10; ACCEPTED 2026-10-15, §13 = what was built), `03-host-agent.md`. Task: storage identity build A (operator
|
(the `FELHOM_THINCHECK` row), `runbooks/whole-guest-backup-orphan-thin.md`, R-1012's closing note.
|
||||||
rulings D1–D6). Controller counterpart: felhom-controller v0.317.0 (MinAgent 0.157.0); hub v0.154.0 shows the fields.
|
|
||||||
|
## In plain words
|
||||||
|
|
||||||
|
- Before tonight the agent looked for an orphan thin device before **every** whole-guest backup. To look, it takes a
|
||||||
|
short metadata snapshot of the live disk pool. Now it looks **only when the backup's snapshot step has failed** — the
|
||||||
|
only moment somebody needs the answer. The operator's failure mail still names the orphan, in the same words.
|
||||||
|
- **Not released. Not vouched.** The next agent release that the operator vouches carries it. Nothing was copied to a
|
||||||
|
box.
|
||||||
|
|
||||||
## What changed
|
## What changed
|
||||||
|
|
||||||
- `internal/storage/identity.go`: the physical disk(s) under a place, from `/sys`, `/run/udev/data`, `/proc/mounts` only
|
- `internal/backup/runner.go`: the pre-run check (and its WARN line) is gone; `explainFailure` (after a vzdump start
|
||||||
(no block I/O). `proven`, or `unknown` with `vm` / `network` / `usb_bridge` / `weak_wwn` / `no_serial`.
|
failure or a task failure, snapshot mode only) is now the only caller. Comments say so.
|
||||||
- `GET /disks` rows and the hub report's `storage_targets` gain `kind`, `physical_disks`, `identity`, `identity_reason`.
|
- `internal/capability/manifest.go`: the `thin-orphan-check` text says „after a failed whole-guest backup".
|
||||||
- `POST /disks/same-disk` `{a, b}` → `{different, proven, reason, a, b}`; scoped like every `/disks` call; a path the agent
|
- `internal/backup/r1012_thincheck_test.go`: failure → exactly ONE call and the finding in the error; success → ZERO
|
||||||
did not mount itself is `unmapped`.
|
calls and a clean record; stop mode → zero calls; unwired → error unchanged.
|
||||||
- `configs/` unchanged.
|
|
||||||
|
|
||||||
## Tests and red-proofs
|
## Red-proof
|
||||||
|
|
||||||
Fixture `/sys` + udev trees for every spike case and five same-disk handler tests. Seen failing first: the LVM thin
|
The new tests, run against the v0.159.1 `runner.go` (`git show HEAD:…`), then the fix restored:
|
||||||
„diamond" (tmeta/tdata under one pool) read as a failure — fixed by counting a seen node once. Broken on purpose and seen
|
|
||||||
failing: the VM rule, the vendor-less WWN rule, the own-mount rule (an absent drive), the `sys_drive` mapping.
|
|
||||||
`python3 scripts/agent_gates.py`: all OK.
|
|
||||||
|
|
||||||
## Release and delivery
|
| test | failed with |
|
||||||
|
|---|---|
|
||||||
|
| `TestBackup_SnapshotFailure_NamesTheOrphan/{task_fails,start_fails}` | `thin check ran 2 times, want 1 (after the failure only)` |
|
||||||
|
| `TestBackup_ThinCheck_SilentWhenNotApplicable` | `success: … Success:true … calls=1 (want no check on a successful backup, clean record)` |
|
||||||
|
|
||||||
- `scripts/release-agent.sh 0.157.0`: tag `v0.157.0`, sha256 `addd4041…0aef`, bundle `0604bc60…397a`.
|
Restored → both PASS; `git diff` on `runner.go` = the fix only.
|
||||||
- Delivered by signed `agent_update` jobs to Tester 1, demo-hp and the N100 (the three test boxes). **NOT VOUCHED** — the
|
|
||||||
operator's act. Until the vouch, the global floor cannot declare MinAgent 0.157.0.
|
|
||||||
|
|
||||||
## Live proof (method: the agent's local API on each PVE host, with the guest's own token)
|
## Gates
|
||||||
|
|
||||||
| Box | `GET /disks` | same-disk: drive vs system data |
|
`go build ./... && go vet ./... && go test ./...` and `python3 scripts/agent_gates.py` — results in the session report.
|
||||||
|---|---|---|
|
|
||||||
| demo-hp | `local`/`local-lvm` proven (SATA SSD), NVMe drive proven (`eui:`), PBS `network` | different, proven |
|
|
||||||
| N100 | system proven (SATA SSD serial), USB drive proven (the bridge passes the real serial) | different, proven |
|
|
||||||
| Tester 1 (VM) | every place `unknown/vm` | not proven (`vm`) |
|
|
||||||
|
|
||||||
On all three: system data vs `/var/lib/docker` → same disk, proven; a drive name that is not mounted → `unmapped`. The hub
|
## Not live-validated
|
||||||
stored the same fields (read from its database with the WAL). Evidence: felhom.eu `audits/storage-identity-buildA-2026-10-15/`.
|
|
||||||
|
|
||||||
## Observations
|
No agent was built or delivered (no release tonight). The changed path is the failure path; the next release's live
|
||||||
|
proof should make one snapshot backup succeed (helper NOT called: no `felhom-thin-check` line in the agent journal
|
||||||
1. FILED: R-1083 — scratch box 9202 has no `local_api` in its `bootstrap.json`, so it cannot use any agent call.
|
and none in the sudo log for that run) and one fail (the finding in the failure text).
|
||||||
2. NOT-A-FINDING: `usb_bridge` is built from the design's rule and was not seen on a real enclosure; that is R-1077, already open.
|
|
||||||
|
|||||||
@@ -15,13 +15,14 @@
|
|||||||
| `SudoHostOps.run` | internal/storage/hostops.go | `run(ctx, name, args...) error` | allowlisted exec with stderr-wrapped error | Every arg pre-validated via validate.go before this is called |
|
| `SudoHostOps.run` | internal/storage/hostops.go | `run(ctx, name, args...) error` | allowlisted exec with stderr-wrapped error | Every arg pre-validated via validate.go before this is called |
|
||||||
| `Prober.Probe` | internal/capability/probe.go | `Probe(ctx) []Status` | live sudo-policy capability check (`sudo -n -l --`) | Needs a DIRECT runner (never the sudo-prefixing one — double-sudo); never executes probed cmds. v0.86.0: config-gated caps (`Capability.GatedBy` + `Prober.GateActive`) report `inactive`/"disabled by configuration" ONLY when healthy — broken plumbing stays degraded; the pbsdr-* gate answers from `pbsdr.Manager.DRConfigured` (marker-backed across restarts) |
|
| `Prober.Probe` | internal/capability/probe.go | `Probe(ctx) []Status` | live sudo-policy capability check (`sudo -n -l --`) | Needs a DIRECT runner (never the sudo-prefixing one — double-sudo); never executes probed cmds. v0.86.0: config-gated caps (`Capability.GatedBy` + `Prober.GateActive`) report `inactive`/"disabled by configuration" ONLY when healthy — broken plumbing stays degraded; the pbsdr-* gate answers from `pbsdr.Manager.DRConfigured` (marker-backed across restarts) |
|
||||||
| ~~`stageTemp`~~ (REMOVED v0.146.0, R-861) | — | — | — | Nothing the agent writes is `install`ed where root reads it any more: use `felhom-priv-apply` (below) or ship a fixed file in the bundle |
|
| ~~`stageTemp`~~ (REMOVED v0.146.0, R-861) | — | — | — | Nothing the agent writes is `install`ed where root reads it any more: use `felhom-priv-apply` (below) or ship a fixed file in the bundle |
|
||||||
| `felhom-priv-apply` (v0.146.0, R-861) | configs/felhom-priv-apply | `felhom-priv-apply unit <name> \| dnsmasq <tmp> <name> \| wg \| sshd-config \| sshd-key \| controller-image <vmid>` (the last reads the ref on stdin, R-861 (a) A1) | ANY agent-rendered file a root program reads (systemd unit, dnsmasq drop-in, wg-quick conf, OOB sshd) — fixed source + destination, CONTENT checked against the agent's own renderers | A new renderer needs a verb + a contract test (`internal/privapplytest.Check`) feeding its REAL output; never a new `install` sudoers line |
|
| `felhom-priv-apply` (v0.146.0, R-861) | configs/felhom-priv-apply | `felhom-priv-apply unit <name> \| unit-forget <name> (v0.159.0, R-1124) \| dnsmasq <tmp> <name> \| wg \| sshd-config \| sshd-key \| controller-image <vmid>` (the last reads the ref on stdin, R-861 (a) A1) | ANY agent-rendered file a root program reads (systemd unit, dnsmasq drop-in, wg-quick conf, OOB sshd) — fixed source + destination, CONTENT checked against the agent's own renderers | A new renderer needs a verb + a contract test (`internal/privapplytest.Check`) feeding its REAL output; never a new `install` sudoers line |
|
||||||
| `privapplytest.Check` | internal/privapplytest/check.go | `Check(t, verb, name, content) string` | the Go↔root-checker contract: a renderer's real output must read `OK` | Skips without python3; one call per rendered shape + one refused control |
|
| `privapplytest.Check` | internal/privapplytest/check.go | `Check(t, verb, name, content) string` | the Go↔root-checker contract: a renderer's real output must read `OK` | Skips without python3; one call per rendered shape + one refused control |
|
||||||
| `BUNDLE_FILES` + `Bundle` (mode `bundle`, `--install-bundle`; mode `agent_update` v0.146.0, R-861) | configs/felhom-os-apply | the ONE table of root-owned paths + the installer of them | ANY new root-owned file the installer writes (sudoers line, wrapper, unit) — add it to the table, never a new installer fetch (R-840) | The builder (`scripts/build-config-bundle.py`) and the installer read the same table; `test_every_root_file_the_installer_writes_is_in_the_bundle` fails on a path the bundle lacks. Trust files (`/etc/felhom/os-trust.json`, `operator-signers`) are NEVER bundle paths (R17) |
|
| `BUNDLE_FILES` + `Bundle` (mode `bundle`, `--install-bundle`; mode `agent_update` v0.146.0, R-861) | configs/felhom-os-apply | the ONE table of root-owned paths + the installer of them | ANY new root-owned file the installer writes (sudoers line, wrapper, unit) — add it to the table, never a new installer fetch (R-840) | The builder (`scripts/build-config-bundle.py`) and the installer read the same table; `test_every_root_file_the_installer_writes_is_in_the_bundle` fails on a path the bundle lacks. Trust files (`/etc/felhom/os-trust.json`, `operator-signers`) are NEVER bundle paths (R17) |
|
||||||
| `osupdate.ConfigUpdateExecutor` | internal/osupdate/bundle.go | signed op `agent_config_update` {agent_version, bundle_sha256} | delivering the bundle to an installed box | a courier only: the root wrapper re-verifies signature, host, nonce and sha itself |
|
| `osupdate.ConfigUpdateExecutor` | internal/osupdate/bundle.go | signed op `agent_config_update` {agent_version, bundle_sha256} | delivering the bundle to an installed box | a courier only: the root wrapper re-verifies signature, host, nonce and sha itself |
|
||||||
| `osupdate.Leg.SendUnsent` / `lockPass` (v0.144.0, R-868) | internal/osupdate/unsent.go | `(ctx) int` | an OS-pass report the agent never sent (killed mid-pass): the wrapper keeps `report-<run>-<layer>-apply.json` beside the plan; the agent deletes it once the hub has it | any new caller that runs an apply pass must hold `lockPass` (flock, across processes) — the sender must never take a running pass's copy |
|
| `osupdate.Leg.SendUnsent` / `lockPass` (v0.144.0, R-868) | internal/osupdate/unsent.go | `(ctx) int` | an OS-pass report the agent never sent (killed mid-pass): the wrapper keeps `report-<run>-<layer>-apply.json` beside the plan; the agent deletes it once the hub has it | any new caller that runs an apply pass must hold `lockPass` (flock, across processes) — the sender must never take a running pass's copy |
|
||||||
| `osupdate.LoadSavedBlock` (v0.144.0, R-866) | internal/osupdate/leg.go | `(planDir) (block, savedAt, ok)` | the hub's newest os_update block as the daemon last received it (`os-update-block.json`) | the debug pass uses it ONLY when the hub cannot be reached, and says so in its header; no saved block → no pass |
|
| `osupdate.LoadSavedBlock` (v0.144.0, R-866) | internal/osupdate/leg.go | `(planDir) (block, savedAt, ok)` | the hub's newest os_update block as the daemon last received it (`os-update-block.json`) | the debug pass uses it ONLY when the hub cannot be reached, and says so in its header; no saved block → no pass |
|
||||||
| `dpkg_state()` / `DPKG_STATE_SCRIPT` (v0.145.0, R-876) | configs/felhom-os-apply | `audit, journal = self.dpkg_state()` | dpkg's state in ONE call: `--audit` AND the update journal | never gate a repair on `--audit` alone — a crash leaves only the journal (measured); keep it one call (R-845) |
|
| `dpkg_state()` / `DPKG_STATE_SCRIPT` (v0.145.0, R-876) | configs/felhom-os-apply | `audit, journal = self.dpkg_state()` | dpkg's state in ONE call: `--audit` AND the update journal | never gate a repair on `--audit` alone — a crash leaves only the journal (measured); keep it one call (R-845) |
|
||||||
|
| `thincheck.Checker.Check` + `Describe` (v0.159.0, R-1012) | internal/thincheck/thincheck.go | `Check(ctx, vmid) []Finding` | name an ORPHAN thin device (pool holds it, LVM does not) in a whole-guest backup failure — `BackupRunner.SetThinCheck` | READ-ONLY by design: the root helper `felhom-thin-check <vg> <pool>` sends only reserve/release_metadata_snap; an orphan needs two reads; fails open (never stops a backup). The repair stays human |
|
||||||
| `guesthook.SnippetReady` / `Register` (v0.146.0, R-861) | internal/guesthook/install.go | `SnippetReady(path) error` | is the pre-start hook (a FIXED file from the bundle) in place — register only then | The agent never installs the hook (Proxmox runs it as root); a missing hookscript stops a guest start, so never `Register` without `SnippetReady` |
|
| `guesthook.SnippetReady` / `Register` (v0.146.0, R-861) | internal/guesthook/install.go | `SnippetReady(path) error` | is the pre-start hook (a FIXED file from the bundle) in place — register only then | The agent never installs the hook (Proxmox runs it as root); a missing hookscript stops a guest start, so never `Register` without `SnippetReady` |
|
||||||
|
|
||||||
### Disk / format safety (role gates, durable IDs, format guards)
|
### Disk / format safety (role gates, durable IDs, format guards)
|
||||||
@@ -56,6 +57,7 @@
|
|||||||
| `SudoHostOps.EnsureMount` | internal/storage/hostops.go | `EnsureMount(ctx, MountSpec) error` | persistent by-UUID systemd .mount | Validate→render→stage→`install`→`enable --now`; idempotent |
|
| `SudoHostOps.EnsureMount` | internal/storage/hostops.go | `EnsureMount(ctx, MountSpec) error` | persistent by-UUID systemd .mount | Validate→render→stage→`install`→`enable --now`; idempotent |
|
||||||
| `SudoHostOps.Unmount` | internal/storage/hostops.go | `Unmount(ctx, where) error` | detach a mount unit | DESTRUCTIVE — caller MUST have gated it; does not self-authorize |
|
| `SudoHostOps.Unmount` | internal/storage/hostops.go | `Unmount(ctx, where) error` | detach a mount unit | DESTRUCTIVE — caller MUST have gated it; does not self-authorize |
|
||||||
| `SudoHostOps.ReassertEnrolledMounts` | internal/storage/hostops.go | `ReassertEnrolledMounts(ctx)` | reboot remount (re-resolve by UUID) | Re-asserts unless mounted AND enabled (`shouldReassertMount`); skips absent UUIDs |
|
| `SudoHostOps.ReassertEnrolledMounts` | internal/storage/hostops.go | `ReassertEnrolledMounts(ctx)` | reboot remount (re-resolve by UUID) | Re-asserts unless mounted AND enabled (`shouldReassertMount`); skips absent UUIDs |
|
||||||
|
| `SudoHostOps.ForgetRetiredDrives` (v0.159.0, R-1124) | internal/storage/forget_retired.go | `ForgetRetiredDrives(ctx, IntentReader) int` | drop the host mount unit of a DECOMMISSIONED drive that is GONE (start-up + the 20 s tick) | Only decommissioned + absent + unmounted; a present decommissioned drive keeps its unit (H3 re-enroll). The file goes only through `felhom-priv-apply unit-forget`, which re-checks all three as root |
|
||||||
| `GuestBinder.AttachDrive` / `DetachDrive` | internal/localapi/intermediary.go | `AttachDrive(ctx, vmid, where) (guestPath, err)` | live drive hot-swap under `/mnt/felhom-drives` | Normalizes to EXACTLY ONE bind via `countHostMounts` (converges double-binds); force re-bind when guest can't see it |
|
| `GuestBinder.AttachDrive` / `DetachDrive` | internal/localapi/intermediary.go | `AttachDrive(ctx, vmid, where) (guestPath, err)` | live drive hot-swap under `/mnt/felhom-drives` | Normalizes to EXACTLY ONE bind via `countHostMounts` (converges double-binds); force re-bind when guest can't see it |
|
||||||
| `GuestBinder.EnsureSharedParent` | internal/localapi/intermediary.go | `EnsureSharedParent(ctx) error` | shared-parent bind + boot unit | make-private+make-shared ONLY on first bind — re-running orphans the guest's slave; F2-a: compares script AND unit for staleness |
|
| `GuestBinder.EnsureSharedParent` | internal/localapi/intermediary.go | `EnsureSharedParent(ctx) error` | shared-parent bind + boot unit | make-private+make-shared ONLY on first bind — re-running orphans the guest's slave; F2-a: compares script AND unit for staleness |
|
||||||
| `StablePathForRaw` / `DriveNameFromRaw` | internal/localapi/intermediary.go | `StablePathForRaw("/mnt/<n>") string` | raw host mount → stable guest path | Single source of truth both repos derive from |
|
| `StablePathForRaw` / `DriveNameFromRaw` | internal/localapi/intermediary.go | `StablePathForRaw("/mnt/<n>") string` | raw host mount → stable guest path | Single source of truth both repos derive from |
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ import (
|
|||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/thincheck"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/vzpartial"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/vzpartial"
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/wgtunnel"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/wgtunnel"
|
||||||
)
|
)
|
||||||
@@ -1091,6 +1092,9 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
decommIntent = intentStore
|
decommIntent = intentStore
|
||||||
}
|
}
|
||||||
decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)
|
decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)
|
||||||
|
if guestBindStore != nil { // typed-nil guard, as above
|
||||||
|
decommExec.SetGuestBinds(guestBindStore) // v0.159.1: prune the bind record like the self-serve decommission
|
||||||
|
}
|
||||||
|
|
||||||
// Agent self-update (TASK D1): the agent_update executor downloads the operator-signed binary,
|
// Agent self-update (TASK D1): the agent_update executor downloads the operator-signed binary,
|
||||||
// verifies it against the SIGNED sha256, and hands it to the root guarded wrapper (A/B flip +
|
// verifies it against the SIGNED sha256, and hands it to the root guarded wrapper (A/B flip +
|
||||||
@@ -1425,6 +1429,9 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
// re-assert below sees its unit.
|
// re-assert below sees its unit.
|
||||||
localSrv.EnsureInternalPlace(ctx)
|
localSrv.EnsureInternalPlace(ctx)
|
||||||
if mountReasserter != nil {
|
if mountReasserter != nil {
|
||||||
|
// R-1124: a decommissioned drive that is gone loses its host mount unit first (start-up and every tick), so
|
||||||
|
// no later boot waits for it and the re-assert below no longer lists it.
|
||||||
|
mountReasserter.ForgetRetiredDrives(ctx, intentReader)
|
||||||
mountReasserter.ReassertEnrolledMounts(ctx)
|
mountReasserter.ReassertEnrolledMounts(ctx)
|
||||||
}
|
}
|
||||||
// F9: on startup (the host's bring-up/reconcile trigger), re-assert any enrolled guest data-drive
|
// F9: on startup (the host's bring-up/reconcile trigger), re-assert any enrolled guest data-drive
|
||||||
@@ -1463,6 +1470,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
|||||||
// unit re-disabled at runtime), then re-assert the guest binds against the now-live mounts.
|
// unit re-disabled at runtime), then re-assert the guest binds against the now-live mounts.
|
||||||
localSrv.EnsureInternalPlace(ctx) // `16` §5: a record written since the last tick (the signed job)
|
localSrv.EnsureInternalPlace(ctx) // `16` §5: a record written since the last tick (the signed job)
|
||||||
if mountReasserter != nil {
|
if mountReasserter != nil {
|
||||||
|
mountReasserter.ForgetRetiredDrives(ctx, intentReader) // R-1124: decommissioned + gone → unit removed
|
||||||
mountReasserter.ReassertEnrolledMounts(ctx)
|
mountReasserter.ReassertEnrolledMounts(ctx)
|
||||||
}
|
}
|
||||||
localSrv.ReassertGuestBinds(ctx)
|
localSrv.ReassertGuestBinds(ctx)
|
||||||
@@ -2061,8 +2069,12 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
|
|||||||
},
|
},
|
||||||
Logger: logger,
|
Logger: logger,
|
||||||
}
|
}
|
||||||
|
// R-1012: an orphan thin device (a full host root during a snapshot) is NAMED in the backup failure the controller
|
||||||
|
// mails to the operator, through the read-only root helper. Never removed — the repair stays a human step.
|
||||||
|
thinChecker := &thincheck.Checker{API: px, Runner: &proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath}, Logger: logger}
|
||||||
for _, r := range tierRunners {
|
for _, r := range tierRunners {
|
||||||
r.SetBeforeBackup(func(ctx context.Context, storage string, vmid int) { cleaner.Clean(ctx, storage, vmid) })
|
r.SetBeforeBackup(func(ctx context.Context, storage string, vmid int) { cleaner.Clean(ctx, storage, vmid) })
|
||||||
|
r.SetThinCheck(func(ctx context.Context, vmid int) string { return thincheck.Describe(thinChecker.Check(ctx, vmid)) })
|
||||||
}
|
}
|
||||||
srv.SetPartialSweep(func(ctx context.Context, vmid int) {
|
srv.SetPartialSweep(func(ctx context.Context, vmid int) {
|
||||||
for _, t := range tierTargets {
|
for _, t := range tierTargets {
|
||||||
|
|||||||
@@ -25,6 +25,7 @@
|
|||||||
|
|
||||||
Cmnd_Alias FELHOM_MOUNT = \
|
Cmnd_Alias FELHOM_MOUNT = \
|
||||||
/usr/local/sbin/felhom-priv-apply ^unit mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$, \
|
/usr/local/sbin/felhom-priv-apply ^unit mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$, \
|
||||||
|
/usr/local/sbin/felhom-priv-apply ^unit-forget mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
||||||
/usr/bin/systemctl daemon-reload, \
|
/usr/bin/systemctl daemon-reload, \
|
||||||
/usr/bin/systemctl ^enable --now -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
/usr/bin/systemctl ^enable --now -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
||||||
/usr/bin/systemctl ^disable -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
/usr/bin/systemctl ^disable -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
||||||
@@ -101,6 +102,11 @@ Cmnd_Alias FELHOM_VZPARTIAL = \
|
|||||||
# ruled size rule and formats ONLY that volume, ONLY when it carries no signature. The agent calls it from the signed
|
# ruled size rule and formats ONLY that volume, ONLY when it carries no signature. The agent calls it from the signed
|
||||||
# `add_internal_place` job (an old box), always with 0 GiB planned: its guest's volumes already exist. `plan` changes
|
# `add_internal_place` job (an old box), always with 0 GiB planned: its guest's volumes already exist. `plan` changes
|
||||||
# nothing. The installer runs the helper as root itself (a new box).
|
# nothing. The installer runs the helper as root itself (a new box).
|
||||||
|
# R-1012 (agent v0.159.0): the READ-ONLY orphan-thin check before and after a whole-guest backup — lists the pool's thin
|
||||||
|
# device ids (through a metadata snapshot it reserves and releases) next to LVM's. It sends no other pool message; the
|
||||||
|
# repair stays a human step. Pinned by configs/test_felhom_thin_check.py.
|
||||||
|
Cmnd_Alias FELHOM_THINCHECK = \
|
||||||
|
/usr/local/sbin/felhom-thin-check ^[A-Za-z0-9_+][A-Za-z0-9_.+-]{0\,63} [A-Za-z0-9_+][A-Za-z0-9_.+-]{0\,63}$
|
||||||
Cmnd_Alias FELHOM_INTERNALPLACE = \
|
Cmnd_Alias FELHOM_INTERNALPLACE = \
|
||||||
/usr/local/sbin/felhom-internal-place ^(plan|create) 0$
|
/usr/local/sbin/felhom-internal-place ^(plan|create) 0$
|
||||||
|
|
||||||
@@ -336,4 +342,4 @@ Cmnd_Alias FELHOM_GUESTNET = \
|
|||||||
/usr/sbin/pct ^exec [0-9]+ -- pgrep -x dhclient$, \
|
/usr/sbin/pct ^exec [0-9]+ -- pgrep -x dhclient$, \
|
||||||
/usr/sbin/pct ^exec [0-9]+ -- dhclient -pf /run/dhclient\.eth0\.pid -lf /var/lib/dhcp/dhclient\.eth0\.leases eth0$
|
/usr/sbin/pct ^exec [0-9]+ -- dhclient -pf /run/dhclient\.eth0\.pid -lf /var/lib/dhcp/dhclient\.eth0\.leases eth0$
|
||||||
|
|
||||||
felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_FSTRIM, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_BACKUPTARGET, FELHOM_SELFHEAL, FELHOM_ESCROW, FELHOM_GUESTNET, FELHOM_SCRATCH_TEARDOWN, FELHOM_OSAPPLY, FELHOM_POWEROFF, FELHOM_VZPARTIAL, FELHOM_INTERNALPLACE
|
felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_FSTRIM, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_BACKUPTARGET, FELHOM_SELFHEAL, FELHOM_ESCROW, FELHOM_GUESTNET, FELHOM_SCRATCH_TEARDOWN, FELHOM_OSAPPLY, FELHOM_POWEROFF, FELHOM_VZPARTIAL, FELHOM_INTERNALPLACE, FELHOM_THINCHECK
|
||||||
|
|||||||
@@ -191,6 +191,8 @@ BUNDLE_FILES = [
|
|||||||
("/usr/local/sbin/felhom-backup-target-apply", "felhom-backup-target-apply", 0o755, "bash", "replace"),
|
("/usr/local/sbin/felhom-backup-target-apply", "felhom-backup-target-apply", 0o755, "bash", "replace"),
|
||||||
# R-1006 (agent v0.156.0): the only delete of backup files — a dead half-written vzdump, re-checked as root.
|
# R-1006 (agent v0.156.0): the only delete of backup files — a dead half-written vzdump, re-checked as root.
|
||||||
("/usr/local/sbin/felhom-vzdump-partial-clean", "felhom-vzdump-partial-clean", 0o755, "bash", "replace"),
|
("/usr/local/sbin/felhom-vzdump-partial-clean", "felhom-vzdump-partial-clean", 0o755, "bash", "replace"),
|
||||||
|
# R-1012 (agent v0.159.0): the read-only orphan-thin check around a whole-guest backup (names an orphan, never removes).
|
||||||
|
("/usr/local/sbin/felhom-thin-check", "felhom-thin-check", 0o755, "bash", "replace"),
|
||||||
# felhom.eu architecture/16 §5 (agent v0.158.0): the internal data place — the only lvcreate/mkfs of a system-disk volume.
|
# felhom.eu architecture/16 §5 (agent v0.158.0): the internal data place — the only lvcreate/mkfs of a system-disk volume.
|
||||||
("/usr/local/sbin/felhom-internal-place", "felhom-internal-place", 0o755, "bash", "replace"),
|
("/usr/local/sbin/felhom-internal-place", "felhom-internal-place", 0o755, "bash", "replace"),
|
||||||
("/usr/local/sbin/felhom-os-apply", "felhom-os-apply", 0o755, "python", "replace"),
|
("/usr/local/sbin/felhom-os-apply", "felhom-os-apply", 0o755, "python", "replace"),
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ longer name the destination: each verb has a fixed source and a fixed (or strict
|
|||||||
|
|
||||||
Verbs (each one sudoers line, exact-match pattern):
|
Verbs (each one sudoers line, exact-match pattern):
|
||||||
unit <name> /var/lib/felhom-agent/units/<name> -> /etc/systemd/system/<name> (.mount | .automount)
|
unit <name> /var/lib/felhom-agent/units/<name> -> /etc/systemd/system/<name> (.mount | .automount)
|
||||||
|
unit-forget <name> REMOVES /etc/systemd/system/<name> (R-1124): only a felhom by-UUID drive .mount unit that is
|
||||||
|
disabled, not mounted, and whose drive is absent (its /dev/disk/by-uuid link is gone)
|
||||||
dnsmasq <tmp> <name> /tmp/felhom-resolver-<digits>.conf -> /etc/dnsmasq.d/felhom-<...>.conf
|
dnsmasq <tmp> <name> /tmp/felhom-resolver-<digits>.conf -> /etc/dnsmasq.d/felhom-<...>.conf
|
||||||
wg /var/lib/felhom-agent/wg/wg-felhom.conf -> /etc/wireguard/wg-felhom.conf (0600)
|
wg /var/lib/felhom-agent/wg/wg-felhom.conf -> /etc/wireguard/wg-felhom.conf (0600)
|
||||||
sshd-config /var/lib/felhom-agent/felhom-sshd/sshd_config -> /etc/felhom-sshd/sshd_config
|
sshd-config /var/lib/felhom-agent/felhom-sshd/sshd_config -> /etc/felhom-sshd/sshd_config
|
||||||
@@ -41,7 +43,7 @@ WG_SRC, WG_DEST = STATE + "/wg/wg-felhom.conf", "/etc/wireguard/wg-felhom.conf"
|
|||||||
SSHD_SRC, SSHD_DEST = STATE + "/felhom-sshd/sshd_config", "/etc/felhom-sshd/sshd_config"
|
SSHD_SRC, SSHD_DEST = STATE + "/felhom-sshd/sshd_config", "/etc/felhom-sshd/sshd_config"
|
||||||
KEY_SRC, KEY_DEST = STATE + "/felhom-sshd/authorized_keys.felhom-op", "/etc/felhom-sshd/authorized_keys/felhom-op"
|
KEY_SRC, KEY_DEST = STATE + "/felhom-sshd/authorized_keys.felhom-op", "/etc/felhom-sshd/authorized_keys/felhom-op"
|
||||||
MAX_BYTES = 64 * 1024
|
MAX_BYTES = 64 * 1024
|
||||||
VERBS = ("unit", "dnsmasq", "wg", "sshd-config", "sshd-key", "controller-image")
|
VERBS = ("unit", "unit-forget", "dnsmasq", "wg", "sshd-config", "sshd-key", "controller-image")
|
||||||
# R-861 (a) A1 (`09` §3 decision 165): the SAME pattern as the agent's controllerImageRe (internal/localapi/
|
# R-861 (a) A1 (`09` §3 decision 165): the SAME pattern as the agent's controllerImageRe (internal/localapi/
|
||||||
# controllerswap.go) — a compromised agent cannot hand the guest's bootstrap any other image. Pinned by
|
# controllerswap.go) — a compromised agent cannot hand the guest's bootstrap any other image. Pinned by
|
||||||
# configs/test_felhom_priv_apply.py ControllerImage.
|
# configs/test_felhom_priv_apply.py ControllerImage.
|
||||||
@@ -51,6 +53,11 @@ CONTROLLER_IMAGE_MAX = 256
|
|||||||
VMID_RE = re.compile(r"^[0-9]{1,9}$")
|
VMID_RE = re.compile(r"^[0-9]{1,9}$")
|
||||||
|
|
||||||
UNIT_NAME_RE = re.compile(r"^mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$")
|
UNIT_NAME_RE = re.compile(r"^mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$")
|
||||||
|
DRIVE_UNIT_NAME_RE = re.compile(r"^mnt-[A-Za-z0-9_.\\-]+\.mount$")
|
||||||
|
DRIVE_UNIT_MARKER = "Managed by felhom-agent"
|
||||||
|
NET_UNIT_MARKER = "felhom network storage"
|
||||||
|
WANTS_DIR = UNIT_DIR + "/multi-user.target.wants"
|
||||||
|
BY_UUID_DIR = "/dev/disk/by-uuid"
|
||||||
DNSMASQ_TMP_RE = re.compile(r"^/tmp/felhom-resolver-[0-9]+\.conf$")
|
DNSMASQ_TMP_RE = re.compile(r"^/tmp/felhom-resolver-[0-9]+\.conf$")
|
||||||
DNSMASQ_NAME_RE = re.compile(r"^felhom-[a-z0-9][a-z0-9._-]*\.conf$")
|
DNSMASQ_NAME_RE = re.compile(r"^felhom-[a-z0-9][a-z0-9._-]*\.conf$")
|
||||||
SEG = r"[A-Za-z0-9_-][A-Za-z0-9_.-]*"
|
SEG = r"[A-Za-z0-9_-][A-Za-z0-9_.-]*"
|
||||||
@@ -135,6 +142,36 @@ class Host:
|
|||||||
def read_stdin(self, limit):
|
def read_stdin(self, limit):
|
||||||
return sys.stdin.buffer.read(limit + 1)
|
return sys.stdin.buffer.read(limit + 1)
|
||||||
|
|
||||||
|
# ---- unit-forget (R-1124) ----
|
||||||
|
def read_installed(self, path):
|
||||||
|
"""The installed unit: None when absent; Refused when it is not a regular file (a symlink is never followed)."""
|
||||||
|
try:
|
||||||
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None
|
||||||
|
except OSError as e:
|
||||||
|
raise Refused("F1", f"cannot open {path}: {e.strerror}")
|
||||||
|
with os.fdopen(fd, "rb") as f:
|
||||||
|
if not stat.S_ISREG(os.fstat(f.fileno()).st_mode):
|
||||||
|
raise Refused("F1", f"{path} is not a regular file")
|
||||||
|
return f.read(MAX_BYTES + 1)
|
||||||
|
|
||||||
|
def lexists(self, path):
|
||||||
|
return os.path.lexists(path)
|
||||||
|
|
||||||
|
def mountpoints(self):
|
||||||
|
"""Every mount point in /proc/self/mountinfo (field 5, octal escapes decoded)."""
|
||||||
|
out = set()
|
||||||
|
with open("/proc/self/mountinfo", "r", encoding="utf-8", errors="replace") as f:
|
||||||
|
for line in f:
|
||||||
|
parts = line.split(" ")
|
||||||
|
if len(parts) > 4:
|
||||||
|
out.add(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), parts[4]))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def remove(self, path):
|
||||||
|
os.remove(path)
|
||||||
|
|
||||||
def write_guest_image(self, vmid, data):
|
def write_guest_image(self, vmid, data):
|
||||||
"""As root: `pct exec <vmid> -- tee <the fixed file>` with the checked ref on stdin (no shell)."""
|
"""As root: `pct exec <vmid> -- tee <the fixed file>` with the checked ref on stdin (no shell)."""
|
||||||
r = subprocess.run(["/usr/sbin/pct", "exec", str(vmid), "--", "tee", CONTROLLER_IMAGE_FILE],
|
r = subprocess.run(["/usr/sbin/pct", "exec", str(vmid), "--", "tee", CONTROLLER_IMAGE_FILE],
|
||||||
@@ -424,6 +461,45 @@ def controller_image(rest, host):
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def unit_forget(rest, host):
|
||||||
|
"""R-1124: remove ONE felhom drive mount unit that the agent has already disabled, whose drive is gone. A drive that
|
||||||
|
is still present or still mounted keeps its unit — this verb can only forget, never unmount or take a drive away."""
|
||||||
|
try:
|
||||||
|
if len(rest) != 1 or not DRIVE_UNIT_NAME_RE.match(rest[0]) or "/" in rest[0]:
|
||||||
|
raise Refused("A1", "usage: felhom-priv-apply unit-forget mnt-<escaped path>.mount")
|
||||||
|
name = rest[0]
|
||||||
|
path = os.path.join(UNIT_DIR, name)
|
||||||
|
data = host.read_installed(path)
|
||||||
|
if data is None:
|
||||||
|
host.log(f"felhom-priv-apply: ABSENT unit-forget {path} (nothing to remove)")
|
||||||
|
return 0
|
||||||
|
text = text_of(data, path)
|
||||||
|
if DRIVE_UNIT_MARKER not in text or NET_UNIT_MARKER in text:
|
||||||
|
raise Refused("F2", f"{name} is not a felhom drive unit")
|
||||||
|
check_unit(name, text)
|
||||||
|
s = parse_ini(text, name)
|
||||||
|
what, where = s["Mount"].get("What", ""), s["Mount"]["Where"]
|
||||||
|
mu = re.match(r"^/dev/disk/by-uuid/(.+)$", what)
|
||||||
|
if not mu:
|
||||||
|
raise Refused("F2", f"{name} is not a by-UUID drive unit")
|
||||||
|
if host.lexists(os.path.join(WANTS_DIR, name)):
|
||||||
|
raise Refused("F3", f"{name} is still enabled (disable it first)")
|
||||||
|
if where in host.mountpoints():
|
||||||
|
raise Refused("F4", f"{where} is mounted")
|
||||||
|
if host.lexists(os.path.join(BY_UUID_DIR, mu.group(1))):
|
||||||
|
raise Refused("F5", f"the drive of {name} is present (filesystem UUID {mu.group(1)})")
|
||||||
|
except Refused as e:
|
||||||
|
host.log(f"felhom-priv-apply: REFUSED [{e.rule}] unit-forget {' '.join(rest)[:120]}: {e.reason}")
|
||||||
|
return 2 if e.rule == "A1" else 3
|
||||||
|
try:
|
||||||
|
host.remove(path)
|
||||||
|
except OSError as e:
|
||||||
|
host.log(f"felhom-priv-apply: FAILED unit-forget {path}: {e}")
|
||||||
|
return 4
|
||||||
|
host.log(f"felhom-priv-apply: REMOVED unit-forget {path} (where={where})")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def main(argv, host=None):
|
def main(argv, host=None):
|
||||||
host = host or Host()
|
host = host or Host()
|
||||||
if argv == ["--self-check"]:
|
if argv == ["--self-check"]:
|
||||||
@@ -445,6 +521,8 @@ def main(argv, host=None):
|
|||||||
return 0
|
return 0
|
||||||
if argv and argv[0] == "controller-image":
|
if argv and argv[0] == "controller-image":
|
||||||
return controller_image(argv[1:], host)
|
return controller_image(argv[1:], host)
|
||||||
|
if argv and argv[0] == "unit-forget":
|
||||||
|
return unit_forget(argv[1:], host)
|
||||||
try:
|
try:
|
||||||
verb, src, dest, mode, checker = plan(argv)
|
verb, src, dest, mode, checker = plan(argv)
|
||||||
data = host.read_source(src)
|
data = host.read_source(src)
|
||||||
|
|||||||
Executable
+74
@@ -0,0 +1,74 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# felhom-thin-check — list the thin device ids a thin pool holds next to the ones LVM knows (R-1012). READ-ONLY: it
|
||||||
|
# names an orphan, it never removes one. Installed as /usr/local/sbin/felhom-thin-check (config bundle).
|
||||||
|
#
|
||||||
|
# felhom-thin-check <vg> <pool>
|
||||||
|
#
|
||||||
|
# A full host root while vzdump takes its snapshot can leave a thin snapshot inside the pool that LVM no longer knows
|
||||||
|
# (demo-hp 2026-10-14: `thin_ls` device 4, LVM only 1 and 2); from then on every snapshot backup of that guest fails
|
||||||
|
# („Creation of new snapshot 4 of device 2 failed") and nothing said why. The repair (`dmsetup message … delete <id>`)
|
||||||
|
# stays a HUMAN step, decided by the operator — this helper only reads.
|
||||||
|
#
|
||||||
|
# Reading the live pool's device list needs a metadata snapshot: the kernel's documented way to read metadata while
|
||||||
|
# the pool is in use (`reserve_metadata_snap`, `thin_ls --metadata-snap`, `release_metadata_snap`). It changes no data
|
||||||
|
# and no mapping; it is held for the length of one `thin_ls -o DEV` (well under a second on these pools). Rules:
|
||||||
|
# 1. the arguments are an LVM name pair; <vg>/<pool> is a thin pool (lv_attr starts with `t`);
|
||||||
|
# 2. only one run at a time (a lock file); a metadata snapshot that someone ELSE holds is never touched — the run
|
||||||
|
# refuses (exit 3) and does not release it;
|
||||||
|
# 3. LVM's list is read before AND after the pool's, so a volume created in between is not reported as an orphan
|
||||||
|
# (the caller treats an id as an orphan only when it is in the pool and in neither LVM list).
|
||||||
|
# Output, one record per line: lvm-before <id> <lv> | pool <id> | lvm-after <id> <lv> | ok
|
||||||
|
# Exit: 0 read, 1 refused (arguments / not a thin pool), 3 a metadata snapshot is already held or cannot be reserved,
|
||||||
|
# 4 the pool could not be read. Pinned by configs/test_felhom_thin_check.py.
|
||||||
|
set -u
|
||||||
|
LVS=/usr/sbin/lvs
|
||||||
|
DMSETUP=/usr/sbin/dmsetup
|
||||||
|
THIN_LS=/usr/sbin/thin_ls
|
||||||
|
LOCK=/run/lock/felhom-thin-check.lock
|
||||||
|
NAME_RE='^[A-Za-z0-9_+][A-Za-z0-9_.+-]{0,63}$'
|
||||||
|
|
||||||
|
die() { echo "felhom-thin-check: REFUSED: $1" >&2; exit "${2:-1}"; } # $1 the reason, $2 the exit code
|
||||||
|
|
||||||
|
lvm_ids() { # $1 tag → "<tag> <id> <lv>" for every thin volume of this pool
|
||||||
|
"$LVS" --noheadings --separator , -o lv_name,thin_id --select "pool_lv=$POOL" -- "$VG" 2>/dev/null |
|
||||||
|
awk -F, -v tag="$1" '{ gsub(/ /, ""); if ($2 ~ /^[0-9]+$/) print tag, $2, $1 }'
|
||||||
|
}
|
||||||
|
|
||||||
|
release() { "$DMSETUP" message "$TPOOL" 0 release_metadata_snap >/dev/null 2>&1; }
|
||||||
|
|
||||||
|
main() {
|
||||||
|
[[ $# -eq 2 ]] || die "usage: felhom-thin-check <vg> <pool>"
|
||||||
|
VG=$1 POOL=$2
|
||||||
|
local attr held ids
|
||||||
|
[[ "$VG" =~ $NAME_RE && "$POOL" =~ $NAME_RE ]] || die "not an LVM name pair"
|
||||||
|
|
||||||
|
attr=$("$LVS" --noheadings -o lv_attr -- "$VG/$POOL" 2>/dev/null | tr -d ' ')
|
||||||
|
[[ "$attr" == t* ]] || die "$VG/$POOL is not a thin pool"
|
||||||
|
|
||||||
|
DM="${VG//-/--}-${POOL//-/--}"
|
||||||
|
TPOOL="$DM-tpool"
|
||||||
|
TMETA="/dev/mapper/${DM}_tmeta"
|
||||||
|
|
||||||
|
exec 9>"$LOCK" || die "cannot open the lock file" 4
|
||||||
|
flock -w 30 9 || die "another check holds the lock" 3
|
||||||
|
|
||||||
|
# Field 7 of the thin-pool status is the held metadata root: "-" when no metadata snapshot exists.
|
||||||
|
held=$("$DMSETUP" status "$TPOOL" 2>/dev/null | awk '$3 == "thin-pool" { print $7 }')
|
||||||
|
[[ -n "$held" ]] || die "cannot read the status of $TPOOL" 4
|
||||||
|
[[ "$held" == "-" ]] || die "a metadata snapshot of $TPOOL is already held (by someone else) — not touched" 3
|
||||||
|
|
||||||
|
lvm_ids lvm-before
|
||||||
|
"$DMSETUP" message "$TPOOL" 0 reserve_metadata_snap >/dev/null 2>&1 || die "could not reserve a metadata snapshot of $TPOOL" 3
|
||||||
|
trap release EXIT
|
||||||
|
trap 'release; trap - EXIT; exit 4' INT TERM HUP
|
||||||
|
if ! ids=$("$THIN_LS" --metadata-snap --no-headers -o DEV "$TMETA" 2>/dev/null); then
|
||||||
|
die "thin_ls could not read $TMETA" 4
|
||||||
|
fi
|
||||||
|
release
|
||||||
|
trap - EXIT INT TERM HUP
|
||||||
|
awk '$1 ~ /^[0-9]+$/ { print "pool", $1 }' <<<"$ids"
|
||||||
|
lvm_ids lvm-after
|
||||||
|
echo ok
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@"; fi
|
||||||
@@ -376,5 +376,67 @@ class ControllerImage(unittest.TestCase):
|
|||||||
self.assertIn("controller-image", buf.getvalue())
|
self.assertIn("controller-image", buf.getvalue())
|
||||||
|
|
||||||
|
|
||||||
|
class UnitForget(unittest.TestCase):
|
||||||
|
"""R-1124: the agent forgets a decommissioned drive whose disk is gone — the root half removes its unit file, and
|
||||||
|
ONLY a felhom by-UUID drive unit that is disabled, not mounted, and whose /dev/disk/by-uuid link is absent.
|
||||||
|
RED-PROOF: on the pre-R-1124 wrapper `unit-forget` is not a verb (A1 usage, rc 2) — the removal case fails."""
|
||||||
|
NAME = "mnt-hdd_1.mount"
|
||||||
|
PATH = "/etc/systemd/system/mnt-hdd_1.mount"
|
||||||
|
UUID_LINK = "/dev/disk/by-uuid/91d2dc2d-2d28-4929-9bdd-3e11fa2f41ae"
|
||||||
|
|
||||||
|
def host(self, text=LOCAL_UNIT, kind="file", links=(), mounted=()):
|
||||||
|
h = FakeHost()
|
||||||
|
h.installed = {} if text is None else {self.PATH: (kind, text.encode())}
|
||||||
|
h.links, h.mounted, h.removed = set(links), set(mounted), []
|
||||||
|
h.read_installed = lambda p: (None if p not in h.installed else
|
||||||
|
(_ for _ in ()).throw(pa.Refused("F1", f"{p} is not a regular file"))
|
||||||
|
if h.installed[p][0] != "file" else h.installed[p][1])
|
||||||
|
h.lexists = lambda p: p in h.links
|
||||||
|
h.mountpoints = lambda: set(h.mounted)
|
||||||
|
h.remove = lambda p: h.removed.append(p)
|
||||||
|
return h
|
||||||
|
|
||||||
|
def test_an_absent_disabled_unmounted_drive_unit_is_removed(self):
|
||||||
|
h = self.host()
|
||||||
|
self.assertEqual(run(h, "unit-forget", self.NAME), 0, h.logs)
|
||||||
|
self.assertEqual(h.removed, [self.PATH])
|
||||||
|
|
||||||
|
def test_already_gone_is_fine(self):
|
||||||
|
h = self.host(text=None)
|
||||||
|
self.assertEqual(run(h, "unit-forget", self.NAME), 0, h.logs)
|
||||||
|
self.assertEqual(h.removed, [])
|
||||||
|
|
||||||
|
def refused(self, h, rule, *argv):
|
||||||
|
rc = run(h, *(argv or ("unit-forget", self.NAME)))
|
||||||
|
self.assertIn(rc, (2, 3), h.logs)
|
||||||
|
self.assertEqual(h.removed, [], h.logs)
|
||||||
|
self.assertTrue(any(f"[{rule}]" in l for l in h.logs), h.logs)
|
||||||
|
|
||||||
|
def test_F5_a_present_drive_keeps_its_unit(self):
|
||||||
|
self.refused(self.host(links=[self.UUID_LINK]), "F5")
|
||||||
|
|
||||||
|
def test_F4_a_mounted_drive_keeps_its_unit(self):
|
||||||
|
self.refused(self.host(mounted=["/mnt/hdd_1"]), "F4")
|
||||||
|
|
||||||
|
def test_F3_an_enabled_unit_is_not_removed(self):
|
||||||
|
self.refused(self.host(links=["/etc/systemd/system/multi-user.target.wants/" + self.NAME]), "F3")
|
||||||
|
|
||||||
|
def test_F2_a_network_unit_or_a_foreign_unit_is_not_removed(self):
|
||||||
|
self.refused(self.host(text=LOCAL_UNIT.replace("Managed by felhom-agent", "Written by someone")), "F2")
|
||||||
|
net = LOCAL_UNIT.replace("# Managed by felhom-agent", "# Managed by felhom-agent felhom network storage")
|
||||||
|
self.refused(self.host(text=net), "F2")
|
||||||
|
|
||||||
|
def test_F1_a_symlinked_unit_is_not_followed(self):
|
||||||
|
self.refused(self.host(kind="symlink"), "F1")
|
||||||
|
|
||||||
|
def test_A1_names(self):
|
||||||
|
for argv in (("unit-forget", "mnt-hdd_1.automount"), ("unit-forget", "sshd.service"),
|
||||||
|
("unit-forget", "../mnt-hdd_1.mount"), ("unit-forget",), ("unit-forget", self.NAME, "x")):
|
||||||
|
self.refused(self.host(), "A1", *argv)
|
||||||
|
|
||||||
|
def test_U3_a_unit_whose_name_lies_about_where_is_not_removed(self):
|
||||||
|
self.refused(self.host(text=LOCAL_UNIT.replace("Where=/mnt/hdd_1", "Where=/mnt/other")), "U3")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main(verbosity=2)
|
unittest.main(verbosity=2)
|
||||||
|
|||||||
@@ -0,0 +1,118 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Tests for configs/felhom-thin-check (R-1012): it lists the thin pool's device ids next to LVM's and NEVER removes
|
||||||
|
anything. Fake `lvs`, `dmsetup` and `thin_ls` record every call; nothing real is touched. Each refusal has a positive
|
||||||
|
control (the healthy read), so a test cannot pass because the helper refuses everything.
|
||||||
|
Run: python3 configs/test_felhom_thin_check.py (also run by internal/thincheck's Go test)
|
||||||
|
"""
|
||||||
|
import os, pathlib, re, subprocess, tempfile, unittest
|
||||||
|
|
||||||
|
HELPER = pathlib.Path(__file__).resolve().parent / "felhom-thin-check"
|
||||||
|
|
||||||
|
|
||||||
|
class Box:
|
||||||
|
def __init__(self, pool_ids=(1, 2), lvm=(("vm-9201-disk-0", 1), ("vm-9201-disk-1", 2)), attr="twi-aotz--",
|
||||||
|
held="-", reserve_rc=0, thin_ls_rc=0):
|
||||||
|
self.t = tempfile.TemporaryDirectory()
|
||||||
|
r = pathlib.Path(self.t.name)
|
||||||
|
self.log = r / "calls"
|
||||||
|
self.log.write_text("")
|
||||||
|
lvm_lines = "\\n".join(f" {n},{i}" for n, i in lvm)
|
||||||
|
pool_lines = "\\n".join(str(i) for i in pool_ids)
|
||||||
|
self.bins = {}
|
||||||
|
for name, body in {
|
||||||
|
"lvs": f'''case "$*" in *lv_attr*) echo " {attr}";; *thin_id*) printf "{lvm_lines}\\n";; esac''',
|
||||||
|
"dmsetup": f'''case "$1" in
|
||||||
|
status) echo "0 112975872 thin-pool 585 6860/262144 517080/882624 {held} rw discard_passdown queue_if_no_space - 1024";;
|
||||||
|
message) [ "$4" = reserve_metadata_snap ] && exit {reserve_rc}; exit 0;;
|
||||||
|
esac''',
|
||||||
|
"thin_ls": f'''[ {thin_ls_rc} -ne 0 ] && exit {thin_ls_rc}; printf "{pool_lines}\\n"''',
|
||||||
|
}.items():
|
||||||
|
p = r / name
|
||||||
|
p.write_text(f'#!/bin/bash\necho "{name} $*" >> "{self.log}"\n{body}\n')
|
||||||
|
p.chmod(0o755)
|
||||||
|
self.bins[name] = p
|
||||||
|
self.lock = r / "lock"
|
||||||
|
|
||||||
|
def run(self, *args):
|
||||||
|
script = (f'source "{HELPER}"; LVS="{self.bins["lvs"]}"; DMSETUP="{self.bins["dmsetup"]}"; '
|
||||||
|
f'THIN_LS="{self.bins["thin_ls"]}"; LOCK="{self.lock}"; main "$@"')
|
||||||
|
r = subprocess.run(["bash", "-c", script, "x", *args], capture_output=True, text=True)
|
||||||
|
return r.returncode, r.stdout, r.stderr
|
||||||
|
|
||||||
|
def calls(self):
|
||||||
|
return self.log.read_text().splitlines()
|
||||||
|
|
||||||
|
|
||||||
|
def messages(calls):
|
||||||
|
return [c.split()[-1] for c in calls if c.startswith("dmsetup message")]
|
||||||
|
|
||||||
|
|
||||||
|
class Reads(unittest.TestCase):
|
||||||
|
def test_healthy_pool_lists_both_sides_and_releases(self):
|
||||||
|
b = Box()
|
||||||
|
rc, out, err = b.run("pve", "data")
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertEqual(out.splitlines(), ["lvm-before 1 vm-9201-disk-0", "lvm-before 2 vm-9201-disk-1",
|
||||||
|
"pool 1", "pool 2",
|
||||||
|
"lvm-after 1 vm-9201-disk-0", "lvm-after 2 vm-9201-disk-1", "ok"])
|
||||||
|
self.assertEqual(messages(b.calls()), ["reserve_metadata_snap", "release_metadata_snap"])
|
||||||
|
self.assertIn("thin_ls --metadata-snap --no-headers -o DEV /dev/mapper/pve-data_tmeta", b.calls())
|
||||||
|
|
||||||
|
def test_the_2026_10_14_orphan_is_listed(self):
|
||||||
|
rc, out, _ = Box(pool_ids=(1, 2, 4)).run("pve", "data")
|
||||||
|
self.assertEqual(rc, 0)
|
||||||
|
self.assertIn("pool 4", out.splitlines())
|
||||||
|
|
||||||
|
def test_dashed_names_map_to_the_device_mapper_name(self):
|
||||||
|
b = Box()
|
||||||
|
rc, _, err = b.run("felhom-scratch", "thin-pool")
|
||||||
|
self.assertEqual(rc, 0, err)
|
||||||
|
self.assertIn("dmsetup status felhom--scratch-thin--pool-tpool", b.calls())
|
||||||
|
|
||||||
|
|
||||||
|
class Refuses(unittest.TestCase):
|
||||||
|
def refused(self, box, rc_want, *args):
|
||||||
|
rc, out, err = box.run(*args)
|
||||||
|
self.assertEqual(rc, rc_want, (out, err))
|
||||||
|
self.assertNotIn("ok", out.splitlines())
|
||||||
|
return box.calls()
|
||||||
|
|
||||||
|
def test_arguments(self):
|
||||||
|
for args in (("pve",), ("pve", "data", "x"), ("pve;id", "data"), ("-pve", "data"), ("pve", "../data"),
|
||||||
|
("pve", "da ta")):
|
||||||
|
calls = self.refused(Box(), 1, *args)
|
||||||
|
self.assertEqual(calls, [], args)
|
||||||
|
|
||||||
|
def test_not_a_thin_pool(self):
|
||||||
|
calls = self.refused(Box(attr="Vwi-aotz--"), 1, "pve", "vm-9201-disk-1")
|
||||||
|
self.assertEqual(messages(calls), [])
|
||||||
|
|
||||||
|
def test_a_snapshot_someone_else_holds_is_never_touched(self):
|
||||||
|
b = Box(held="1234")
|
||||||
|
rc, _, err = b.run("pve", "data")
|
||||||
|
self.assertEqual(rc, 3)
|
||||||
|
self.assertEqual(messages(b.calls()), []) # neither reserve nor release
|
||||||
|
# the reason line ends with the reason, not the exit code (found live on demo-hp 2026-10-17)
|
||||||
|
self.assertTrue(err.strip().endswith("not touched"), err)
|
||||||
|
|
||||||
|
def test_reserve_failure_does_not_release(self):
|
||||||
|
calls = self.refused(Box(reserve_rc=1), 3, "pve", "data")
|
||||||
|
self.assertEqual(messages(calls), ["reserve_metadata_snap"])
|
||||||
|
|
||||||
|
def test_thin_ls_failure_still_releases(self):
|
||||||
|
calls = self.refused(Box(thin_ls_rc=1), 4, "pve", "data")
|
||||||
|
self.assertEqual(messages(calls), ["reserve_metadata_snap", "release_metadata_snap"])
|
||||||
|
|
||||||
|
|
||||||
|
class NeverDeletes(unittest.TestCase):
|
||||||
|
def test_the_only_pool_messages_are_reserve_and_release(self):
|
||||||
|
src = HELPER.read_text()
|
||||||
|
sent = re.findall(r'"\$DMSETUP" message "\$TPOOL" 0 (\S+)', src)
|
||||||
|
self.assertEqual(sorted(set(sent)), ["release_metadata_snap", "reserve_metadata_snap"])
|
||||||
|
for word in ("delete", "lvremove", "thin_repair", "create_thin", "create_snap", " rm "):
|
||||||
|
code = "\n".join(l for l in src.splitlines() if not l.lstrip().startswith("#"))
|
||||||
|
self.assertNotIn(word, code, word)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
@@ -1,7 +1,8 @@
|
|||||||
module gitea.dooplex.hu/admin/felhom-agent
|
module gitea.dooplex.hu/admin/felhom-agent
|
||||||
|
|
||||||
go 1.25.0
|
go 1.27.2
|
||||||
|
|
||||||
require golang.org/x/crypto v0.52.0
|
require (
|
||||||
|
golang.org/x/crypto v0.58.0
|
||||||
require golang.org/x/sys v0.45.0 // indirect
|
golang.org/x/sys v0.49.0
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
|
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||||
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
|
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
|
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
|
||||||
golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
|
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package backup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
|
)
|
||||||
|
|
||||||
|
// R-1012: a snapshot backup that fails carries the orphan finding in its error — the text the controller mails to
|
||||||
|
// the operator (`whole_guest_backup_failed`). The check runs ONCE, after the failure — never before the run.
|
||||||
|
// RED-PROOF (P1/redproof-thincheck.txt): with explainFailure's append removed this test fails. RED-PROOF (2026-10-17
|
||||||
|
// night, felhom-agent REPORT.md): with the v0.159.0 pre-run check restored, calls = 2 and this test fails.
|
||||||
|
func TestBackup_SnapshotFailure_NamesTheOrphan(t *testing.T) {
|
||||||
|
const why = "probable cause: thin pool pve/data holds device 4 that LVM does not know"
|
||||||
|
for name, api := range map[string]*fakeBackupAPI{
|
||||||
|
"task fails": {vzdumpUPID: "UPID:x", waitErr: errors.New("task failed: job errors")},
|
||||||
|
"start fails": {vzdumpErr: errors.New("vzdump: 500")},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
r := NewBackupRunner(api, "local", "", "", "", quiet())
|
||||||
|
calls := 0
|
||||||
|
r.SetThinCheck(func(context.Context, int) string { calls++; return why })
|
||||||
|
rec, err := r.Backup(context.Background(), 9201)
|
||||||
|
if err == nil || rec.Success {
|
||||||
|
t.Fatalf("want a failure, got %+v", rec)
|
||||||
|
}
|
||||||
|
if !strings.Contains(rec.Error, why) {
|
||||||
|
t.Fatalf("the failure does not name the orphan: %q", rec.Error)
|
||||||
|
}
|
||||||
|
if calls != 1 {
|
||||||
|
t.Fatalf("thin check ran %d times, want 1 (after the failure only)", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A successful backup never runs the check (it takes a metadata snapshot of the live pool, and nobody needs the answer)
|
||||||
|
// and keeps a clean record; a stop-mode backup takes no snapshot, so the check never runs; an unwired runner behaves
|
||||||
|
// exactly as before. RED-PROOF (2026-10-17 night): with the v0.159.0 pre-run check restored, the success leg fails
|
||||||
|
// with calls=1.
|
||||||
|
func TestBackup_ThinCheck_SilentWhenNotApplicable(t *testing.T) {
|
||||||
|
ok := &fakeBackupAPI{vzdumpUPID: "UPID:x",
|
||||||
|
content: []proxmox.StorageContent{{VolID: "local:backup/vzdump-lxc-9201-x", Content: "backup", VMID: 9201, Size: 100, CTime: 1}}}
|
||||||
|
r := NewBackupRunner(ok, "local", "", "", "", quiet())
|
||||||
|
calls := 0
|
||||||
|
r.SetThinCheck(func(context.Context, int) string { calls++; return "probable cause: x" })
|
||||||
|
rec, err := r.Backup(context.Background(), 9201)
|
||||||
|
if err != nil || !rec.Success || rec.Error != "" || calls != 0 {
|
||||||
|
t.Fatalf("success: err=%v rec=%+v calls=%d (want no check on a successful backup, clean record)", err, rec, calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
stop := NewBackupRunner(&fakeBackupAPI{vzdumpUPID: "UPID:x", waitErr: errors.New("boom")}, "local", proxmox.ModeStop, "", "", quiet())
|
||||||
|
calls = 0
|
||||||
|
stop.SetThinCheck(func(context.Context, int) string { calls++; return "probable cause: x" })
|
||||||
|
rec, _ = stop.Backup(context.Background(), 9201)
|
||||||
|
if calls != 0 || strings.Contains(rec.Error, "probable cause") {
|
||||||
|
t.Fatalf("stop mode: calls=%d error=%q", calls, rec.Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
plain := NewBackupRunner(&fakeBackupAPI{vzdumpUPID: "UPID:x", waitErr: errors.New("boom")}, "local", "", "", "", quiet())
|
||||||
|
if rec, _ := plain.Backup(context.Background(), 9201); rec.Error != "boom" {
|
||||||
|
t.Fatalf("unwired: error = %q, want the task error unchanged", rec.Error)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -61,8 +61,13 @@ type BackupRunner struct {
|
|||||||
// beforeBackup runs just before the space preflight (R-1006): it removes a dead half-written vzdump of this guest on
|
// beforeBackup runs just before the space preflight (R-1006): it removes a dead half-written vzdump of this guest on
|
||||||
// this target, so a leftover from a power cut cannot hold the space this backup needs. nil = nothing.
|
// this target, so a leftover from a power cut cannot hold the space this backup needs. nil = nothing.
|
||||||
beforeBackup func(ctx context.Context, storage string, vmid int)
|
beforeBackup func(ctx context.Context, storage string, vmid int)
|
||||||
logger *slog.Logger
|
// thinCheck (R-1012) names an orphan thin device in the guest's pools in plain words ("" = none / not checked). It
|
||||||
now func() time.Time
|
// runs ONLY when a snapshot-mode vzdump has failed (start or task), so the failure the controller mails to the
|
||||||
|
// operator says why; a successful backup never runs it (it takes a metadata snapshot of the live pool). It never
|
||||||
|
// removes anything. nil = not wired.
|
||||||
|
thinCheck func(ctx context.Context, vmid int) string
|
||||||
|
logger *slog.Logger
|
||||||
|
now func() time.Time
|
||||||
// rejected remembers the volids already announced by warnRejectedArchiveOnce, so an incomplete
|
// rejected remembers the volids already announced by warnRejectedArchiveOnce, so an incomplete
|
||||||
// archive is reported ONCE rather than on every 5-minute due-check. Bounded in practice: one
|
// archive is reported ONCE rather than on every 5-minute due-check. Bounded in practice: one
|
||||||
// entry per aborted upload, and a process restart clears it. Guarded by rejectedMu because the
|
// entry per aborted upload, and a process restart clears it. Guarded by rejectedMu because the
|
||||||
@@ -110,6 +115,22 @@ func (r *BackupRunner) SetBeforeBackup(fn func(ctx context.Context, storage stri
|
|||||||
r.beforeBackup = fn
|
r.beforeBackup = fn
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetThinCheck wires the R-1012 orphan check (see BackupRunner.thinCheck).
|
||||||
|
func (r *BackupRunner) SetThinCheck(fn func(ctx context.Context, vmid int) string) { r.thinCheck = fn }
|
||||||
|
|
||||||
|
// explainFailure appends the orphan finding to a failed snapshot-mode backup's error — the only place the check runs, so
|
||||||
|
// it sees the pool as the failure left it. Bounded, so a stuck helper cannot hold the job.
|
||||||
|
func (r *BackupRunner) explainFailure(ctx context.Context, vmid int, rec *hub.Backup) {
|
||||||
|
if r.thinCheck == nil || r.mode != proxmox.ModeSnapshot {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
if why := r.thinCheck(cctx, vmid); why != "" {
|
||||||
|
rec.Error = rec.Error + " — " + why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// localPruneSpec returns the `--prune-backups` spec to apply to THIS backup, or "" to skip pruning. It
|
// localPruneSpec returns the `--prune-backups` spec to apply to THIS backup, or "" to skip pruning. It
|
||||||
// applies the configured retention ONLY when the target is a non-PBS storage — PBS offsite retention is a
|
// applies the configured retention ONLY when the target is a non-PBS storage — PBS offsite retention is a
|
||||||
// separate lifecycle and must never be pruned by the per-run flag (§9). Fail-safe: if the target's type
|
// separate lifecycle and must never be pruned by the per-run flag (§9). Fail-safe: if the target's type
|
||||||
@@ -201,12 +222,15 @@ func (r *BackupRunner) backup(ctx context.Context, vmid int, onSnapshot func())
|
|||||||
return rec, fmt.Errorf("backup: %s", rec.Error)
|
return rec, fmt.Errorf("backup: %s", rec.Error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// R-1012: no orphan check here. It takes a metadata snapshot of the live pool, and its answer is only needed when the
|
||||||
|
// snapshot step has failed — explainFailure runs it then (pinned by TestBackup_ThinCheck_SilentWhenNotApplicable).
|
||||||
upid, err := r.api.Vzdump(ctx, proxmox.VzdumpOptions{
|
upid, err := r.api.Vzdump(ctx, proxmox.VzdumpOptions{
|
||||||
VMID: vmid, Storage: r.target, Mode: r.mode, Notes: r.notes,
|
VMID: vmid, Storage: r.target, Mode: r.mode, Notes: r.notes,
|
||||||
PruneBackups: r.localPruneSpec(ctx), // local target → keep-last=N; PBS/unknown → "" (no prune)
|
PruneBackups: r.localPruneSpec(ctx), // local target → keep-last=N; PBS/unknown → "" (no prune)
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rec.Error = err.Error()
|
rec.Error = err.Error()
|
||||||
|
r.explainFailure(ctx, vmid, &rec)
|
||||||
rec.DurationSeconds = time.Since(start).Seconds()
|
rec.DurationSeconds = time.Since(start).Seconds()
|
||||||
return rec, fmt.Errorf("backup: vzdump vmid %d: %w", vmid, err)
|
return rec, fmt.Errorf("backup: vzdump vmid %d: %w", vmid, err)
|
||||||
}
|
}
|
||||||
@@ -220,6 +244,7 @@ func (r *BackupRunner) backup(ctx context.Context, vmid int, onSnapshot func())
|
|||||||
}
|
}
|
||||||
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: r.waitTimeout}); err != nil {
|
if _, err := r.api.WaitTask(ctx, upid, proxmox.WaitOptions{Timeout: r.waitTimeout}); err != nil {
|
||||||
rec.Error = err.Error()
|
rec.Error = err.Error()
|
||||||
|
r.explainFailure(ctx, vmid, &rec)
|
||||||
rec.DurationSeconds = time.Since(start).Seconds()
|
rec.DurationSeconds = time.Since(start).Seconds()
|
||||||
return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err)
|
return rec, fmt.Errorf("backup: vzdump task vmid %d: %w", vmid, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -91,9 +91,11 @@ var manifest = []Capability{
|
|||||||
{"disk-mkfs-xfs", "guarded format (xfs)", "/usr/local/sbin/felhom-mkfs-guarded", []string{"/dev/sda", "xfs"}, false, ""},
|
{"disk-mkfs-xfs", "guarded format (xfs)", "/usr/local/sbin/felhom-mkfs-guarded", []string{"/dev/sda", "xfs"}, false, ""},
|
||||||
{"disk-smart", "disk SMART health read", "/usr/sbin/smartctl", []string{"-a", "-j", "/dev/sda"}, false, ""},
|
{"disk-smart", "disk SMART health read", "/usr/sbin/smartctl", []string{"-a", "-j", "/dev/sda"}, false, ""},
|
||||||
{"disk-lvs", "thin-pool usage read", "/usr/sbin/lvs", []string{"--reportformat", "json", "--units", "b", "-o", "lv_name,data_percent,metadata_percent", "--", "pve/data"}, false, ""},
|
{"disk-lvs", "thin-pool usage read", "/usr/sbin/lvs", []string{"--reportformat", "json", "--units", "b", "-o", "lv_name,data_percent,metadata_percent", "--", "pve/data"}, false, ""},
|
||||||
|
{"thin-orphan-check", "orphan thin device check after a failed whole-guest backup (read-only, R-1012)", "/usr/local/sbin/felhom-thin-check", []string{"pve", "data"}, false, ""},
|
||||||
|
|
||||||
// ---- Storage mount units (watchdog re-mount) ----
|
// ---- Storage mount units (watchdog re-mount) ----
|
||||||
{"mount-unit-install", "fs-UUID mount unit install (root content check, R-861)", "/usr/local/sbin/felhom-priv-apply", []string{"unit", "mnt-felhom\\x2dx.mount"}, false, ""},
|
{"mount-unit-install", "fs-UUID mount unit install (root content check, R-861)", "/usr/local/sbin/felhom-priv-apply", []string{"unit", "mnt-felhom\\x2dx.mount"}, false, ""},
|
||||||
|
{"mount-unit-forget", "decommissioned-drive unit removal (root check, R-1124)", "/usr/local/sbin/felhom-priv-apply", []string{"unit-forget", "mnt-felhom\\x2dx.mount"}, false, ""},
|
||||||
{"mount-daemon-reload", "systemd reload after unit write", "/usr/bin/systemctl", []string{"daemon-reload"}, false, ""},
|
{"mount-daemon-reload", "systemd reload after unit write", "/usr/bin/systemctl", []string{"daemon-reload"}, false, ""},
|
||||||
{"mount-unit-enable", "mount unit enable", "/usr/bin/systemctl", []string{"enable", "--now", "--", "mnt-felhom\\x2dx.mount"}, false, ""},
|
{"mount-unit-enable", "mount unit enable", "/usr/bin/systemctl", []string{"enable", "--now", "--", "mnt-felhom\\x2dx.mount"}, false, ""},
|
||||||
{"mount-unit-disable", "mount unit disable", "/usr/bin/systemctl", []string{"disable", "--", "mnt-felhom\\x2dx.mount"}, false, ""},
|
{"mount-unit-disable", "mount unit disable", "/usr/bin/systemctl", []string{"disable", "--", "mnt-felhom\\x2dx.mount"}, false, ""},
|
||||||
|
|||||||
@@ -39,9 +39,21 @@ type decommissionParams struct {
|
|||||||
// none is wanted, because a drive can legitimately be decommissioned while physically ABSENT.
|
// none is wanted, because a drive can legitimately be decommissioned while physically ABSENT.
|
||||||
type DecommissionExecutor struct {
|
type DecommissionExecutor struct {
|
||||||
intent IntentDecommissioner
|
intent IntentDecommissioner
|
||||||
|
binds GuestBindPruner // optional (v0.159.1): drop the drive from every guest's bind record, as the self-serve path does
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GuestBindPruner is the per-guest enrolled-bind record (*localapi.GuestBindStore). The self-serve decommission prunes
|
||||||
|
// the drive from it; the signed one did not, so a forgotten drive (R-1124) left a record the 20-second guest-bind
|
||||||
|
// re-assert warned about forever ("skipping non-enrolled drive"), measured on demo-hp 2026-10-17.
|
||||||
|
type GuestBindPruner interface {
|
||||||
|
Guests() map[int][]string
|
||||||
|
Remove(vmid int, durableID string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetGuestBinds wires the bind record (nil-safe: no record, nothing pruned).
|
||||||
|
func (d *DecommissionExecutor) SetGuestBinds(b GuestBindPruner) { d.binds = b }
|
||||||
|
|
||||||
// NewDecommissionExecutor wires the executor to the agent's intent store.
|
// NewDecommissionExecutor wires the executor to the agent's intent store.
|
||||||
func NewDecommissionExecutor(intent IntentDecommissioner, logger *slog.Logger) *DecommissionExecutor {
|
func NewDecommissionExecutor(intent IntentDecommissioner, logger *slog.Logger) *DecommissionExecutor {
|
||||||
if logger == nil {
|
if logger == nil {
|
||||||
@@ -75,6 +87,20 @@ func (d *DecommissionExecutor) Execute(_ context.Context, op string, params json
|
|||||||
if err := d.intent.SetDecommissioned(p.DurableID); err != nil {
|
if err := d.intent.SetDecommissioned(p.DurableID); err != nil {
|
||||||
return fmt.Errorf("decommission: record intent for %q: %w", p.DurableID, err)
|
return fmt.Errorf("decommission: record intent for %q: %w", p.DurableID, err)
|
||||||
}
|
}
|
||||||
|
if d.binds != nil {
|
||||||
|
for vmid, ids := range d.binds.Guests() {
|
||||||
|
for _, id := range ids {
|
||||||
|
if id != p.DurableID {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := d.binds.Remove(vmid, id); err != nil {
|
||||||
|
d.logger.Warn("decommission: guest-bind record not pruned", "vmid", vmid, "durable_id", id, "err", err)
|
||||||
|
} else {
|
||||||
|
d.logger.Info("decommission: drive removed from the guest's bind record", "vmid", vmid, "durable_id", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
d.logger.Warn("decommission: operator-signed decommission complete (drive will never auto-mount)", "durable_id", p.DurableID)
|
d.logger.Warn("decommission: operator-signed decommission complete (drive will never auto-mount)", "durable_id", p.DurableID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -89,3 +89,37 @@ func TestExecutorChain_DispatchesToOwner(t *testing.T) {
|
|||||||
t.Errorf("want ErrNoExecutor for an unowned op, got %v", err)
|
t.Errorf("want ErrNoExecutor for an unowned op, got %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeBinds struct{ m map[int][]string }
|
||||||
|
|
||||||
|
func (f *fakeBinds) Guests() map[int][]string { return f.m }
|
||||||
|
func (f *fakeBinds) Remove(vmid int, id string) error {
|
||||||
|
var kept []string
|
||||||
|
for _, x := range f.m[vmid] {
|
||||||
|
if x != id {
|
||||||
|
kept = append(kept, x)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.m[vmid] = kept
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// v0.159.1: the signed decommission drops the drive from EVERY guest's bind record, as the self-serve path does — and
|
||||||
|
// only that drive. RED-PROOF (felhom.eu audits/agent-allowlist-rotation-2026-10-17/P1/redproof-bindprune.txt): with
|
||||||
|
// the prune loop removed, the record still holds the drive and this test fails.
|
||||||
|
func TestDecommissionExecutor_PrunesTheGuestBindRecord(t *testing.T) {
|
||||||
|
fd := &fakeDecommissioner{}
|
||||||
|
fb := &fakeBinds{m: map[int][]string{9201: {"uuid:keep", "uuid:gone"}, 9202: {"uuid:gone"}}}
|
||||||
|
ex := NewDecommissionExecutor(fd, nil)
|
||||||
|
ex.SetGuestBinds(fb)
|
||||||
|
params, _ := json.Marshal(decommissionParams{DurableID: "uuid:gone"})
|
||||||
|
if err := ex.Execute(context.Background(), opDecommission, params); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := fb.m[9201]; len(got) != 1 || got[0] != "uuid:keep" {
|
||||||
|
t.Fatalf("9201 record = %v, want [uuid:keep]", got)
|
||||||
|
}
|
||||||
|
if got := fb.m[9202]; len(got) != 0 {
|
||||||
|
t.Fatalf("9202 record = %v, want empty", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ForgetRetiredDrives (R-1124) removes the host mount unit of every DECOMMISSIONED drive that is physically gone, so a
|
||||||
|
// later host boot no longer waits for it and the agent stops listing it (RegistryKnownTargets and
|
||||||
|
// ReassertEnrolledMounts both enumerate the units — no unit, no entry, no „absent by UUID" line every 20 s).
|
||||||
|
//
|
||||||
|
// Only a drive that is BOTH decommissioned (the operator's or the customer's permanent removal) AND absent (its
|
||||||
|
// /dev/disk/by-uuid link does not resolve) AND not mounted is forgotten. A decommissioned drive that is still plugged in
|
||||||
|
// keeps its unit and its raw host mount — that is what makes the one-click re-enroll (H3) work, and decommission stays a
|
||||||
|
// logical retire (`03` §storage). The intent record stays `decommissioned`: a forgotten drive that comes back is a new
|
||||||
|
// candidate, never auto-mounted. The root half (`felhom-priv-apply unit-forget`) re-checks every one of these conditions
|
||||||
|
// and refuses otherwise. Called at agent start, on the 20-second storage tick, and right after a decommission. Best
|
||||||
|
// effort per unit; returns how many units it removed. Pinned by TestForgetRetiredDrives_*.
|
||||||
|
func (h *SudoHostOps) ForgetRetiredDrives(ctx context.Context, intent IntentReader) int {
|
||||||
|
if intent == nil {
|
||||||
|
return 0 // no intent store → we cannot know a drive was retired; forget nothing
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(h.unitDir)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Warn("storage: forget retired drives — cannot read unit dir", "dir", h.unitDir, "err", err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
mounted := map[string]bool{}
|
||||||
|
if ms, merr := h.host.Mounts(); merr == nil {
|
||||||
|
for _, m := range ms {
|
||||||
|
mounted[m.MountPoint] = true
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
h.logger.Warn("storage: forget retired drives — mount table unreadable; forgetting nothing", "err", merr)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
removed := 0
|
||||||
|
for _, e := range entries {
|
||||||
|
name := e.Name()
|
||||||
|
if !strings.HasSuffix(name, ".mount") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
data, rerr := os.ReadFile(filepath.Join(h.unitDir, name))
|
||||||
|
if rerr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
spec, ok := parseFelhomMountUnit(string(data))
|
||||||
|
if !ok {
|
||||||
|
continue // not one of our by-UUID drive units (network units are refused by the parser)
|
||||||
|
}
|
||||||
|
durableID := "uuid:" + spec.UUID
|
||||||
|
if intent.Get(durableID) != IntentDecommissioned {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if mounted[spec.Where] {
|
||||||
|
h.logger.Debug("storage: decommissioned drive still mounted — keeping its unit", "where", spec.Where)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, derr := ResolveStorageDevice(durableID); derr == nil {
|
||||||
|
h.logger.Debug("storage: decommissioned drive is present — keeping its unit (re-enroll stays one click)", "where", spec.Where)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.unitEnabled(name) {
|
||||||
|
if err := h.run(ctx, h.bins.Systemctl, "disable", "--", name); err != nil {
|
||||||
|
h.logger.Warn("storage: forget retired drive — disable failed", "where", spec.Where, "unit", name, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := h.run(ctx, h.bins.PrivApply, "unit-forget", name); err != nil {
|
||||||
|
h.logger.Warn("storage: forget retired drive — unit removal refused or failed", "where", spec.Where, "unit", name, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
removed++
|
||||||
|
h.logger.Info("storage: forgot a decommissioned drive that is gone — its mount unit is removed",
|
||||||
|
"name", spec.Name, "where", spec.Where, "uuid", spec.UUID, "unit", name)
|
||||||
|
}
|
||||||
|
if removed > 0 {
|
||||||
|
if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil {
|
||||||
|
h.logger.Warn("storage: daemon-reload after forgetting retired drives failed", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return removed
|
||||||
|
}
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// forgetRunner records every privileged call and plays the root half of `felhom-priv-apply unit-forget`: the unit file
|
||||||
|
// disappears from the unit dir, so the CONSEQUENCE (the drive is no longer listed) is observable, not just the calls.
|
||||||
|
type forgetRunner struct {
|
||||||
|
unitDir string
|
||||||
|
calls [][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *forgetRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
return r.Run(ctx, name, args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *forgetRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
r.calls = append(r.calls, append([]string{name}, args...))
|
||||||
|
if len(args) == 2 && args[0] == "unit-forget" {
|
||||||
|
_ = os.Remove(filepath.Join(r.unitDir, args[1]))
|
||||||
|
}
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type forgetCase struct {
|
||||||
|
intent DriveIntent
|
||||||
|
present bool // the by-uuid link resolves
|
||||||
|
mounted bool
|
||||||
|
enabled bool // the wants-symlink exists
|
||||||
|
mountErr bool
|
||||||
|
}
|
||||||
|
|
||||||
|
const forgetUUID = "4ebfd36d-87d3-4b63-839e-a5a710fc799a"
|
||||||
|
|
||||||
|
func runForget(t *testing.T, c forgetCase, intent IntentReader) (int, *forgetRunner, *RegistryKnownTargets, string) {
|
||||||
|
t.Helper()
|
||||||
|
unitDir := t.TempDir()
|
||||||
|
unit := "mnt-media_a.mount"
|
||||||
|
writeUnit(t, unitDir, unit, renderMountUnit(MountSpec{Name: forgetUUID, UUID: forgetUUID, Where: "/mnt/media_a", FSType: "ext4"}))
|
||||||
|
if c.enabled {
|
||||||
|
if err := os.MkdirAll(filepath.Join(unitDir, "multi-user.target.wants"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(filepath.Join(unitDir, unit), filepath.Join(unitDir, "multi-user.target.wants", unit)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
devRoot := t.TempDir()
|
||||||
|
if err := os.MkdirAll(filepath.Join(devRoot, "by-uuid"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c.present {
|
||||||
|
dev := filepath.Join(devRoot, "sdz1")
|
||||||
|
if err := os.WriteFile(dev, nil, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(dev, filepath.Join(devRoot, "by-uuid", forgetUUID)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
withDevDiskRoot(t, devRoot)
|
||||||
|
hr := &fakeHostReader{}
|
||||||
|
if c.mounted {
|
||||||
|
hr.mounts = []Mount{{MountPoint: "/mnt/media_a", FSType: "ext4"}}
|
||||||
|
}
|
||||||
|
if c.mountErr {
|
||||||
|
hr.mountsErr = os.ErrPermission
|
||||||
|
}
|
||||||
|
rr := &forgetRunner{unitDir: unitDir}
|
||||||
|
ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, Bins: Binaries{}.withDefaults(), UnitDir: unitDir,
|
||||||
|
StageDir: t.TempDir(), Host: hr, Logger: quietLogger()})
|
||||||
|
if intent == nil {
|
||||||
|
intent = &fakeIntent{m: map[string]DriveIntent{"uuid:" + forgetUUID: c.intent}}
|
||||||
|
}
|
||||||
|
n := ops.ForgetRetiredDrives(context.Background(), intent)
|
||||||
|
return n, rr, NewRegistryKnownTargets(unitDir, intent, quietLogger()), unitDir
|
||||||
|
}
|
||||||
|
|
||||||
|
// The R-1124 case: decommissioned, then unplugged. The unit is disabled and removed through the root checker, systemd
|
||||||
|
// is reloaded, and the CONSEQUENCE — the agent no longer lists the drive — holds. RED-PROOF (P1/redproof-forget.txt):
|
||||||
|
// with ForgetRetiredDrives' body replaced by `return 0` this test fails (the drive is still listed).
|
||||||
|
func TestForgetRetiredDrives_DecommissionedAndGone_UnitRemovedAndNotListed(t *testing.T) {
|
||||||
|
n, rr, known, _ := runForget(t, forgetCase{intent: IntentDecommissioned, enabled: true}, nil)
|
||||||
|
if n != 1 {
|
||||||
|
t.Fatalf("removed = %d, want 1 (calls %v)", n, rr.calls)
|
||||||
|
}
|
||||||
|
pa := Binaries{}.withDefaults()
|
||||||
|
want := [][]string{
|
||||||
|
{pa.Systemctl, "disable", "--", "mnt-media_a.mount"},
|
||||||
|
{pa.PrivApply, "unit-forget", "mnt-media_a.mount"},
|
||||||
|
{pa.Systemctl, "daemon-reload"},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(rr.calls, want) {
|
||||||
|
t.Fatalf("calls = %v\nwant %v", rr.calls, want)
|
||||||
|
}
|
||||||
|
got, err := known.Known(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("the forgotten drive is still listed: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Already disabled (the Tester 1 shape after a manual disable): no disable call, the removal still runs.
|
||||||
|
func TestForgetRetiredDrives_AlreadyDisabled_NoDisableCall(t *testing.T) {
|
||||||
|
n, rr, _, _ := runForget(t, forgetCase{intent: IntentDecommissioned}, nil)
|
||||||
|
pa := Binaries{}.withDefaults()
|
||||||
|
want := [][]string{{pa.PrivApply, "unit-forget", "mnt-media_a.mount"}, {pa.Systemctl, "daemon-reload"}}
|
||||||
|
if n != 1 || !reflect.DeepEqual(rr.calls, want) {
|
||||||
|
t.Fatalf("n=%d calls=%v want %v", n, rr.calls, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Everything that must KEEP its unit: a decommissioned drive still plugged in (re-enroll is one click — H3), one still
|
||||||
|
// mounted, an enrolled or ejected drive that is merely absent (it comes back), a drive with no recorded intent, and an
|
||||||
|
// unreadable mount table. No privileged call at all, and the drive stays listed where it was listed.
|
||||||
|
// RED-PROOF: with the present-drive check removed, the "present" case calls unit-forget and fails.
|
||||||
|
func TestForgetRetiredDrives_KeepsEveryOtherUnit(t *testing.T) {
|
||||||
|
for name, c := range map[string]forgetCase{
|
||||||
|
"decommissioned but present": {intent: IntentDecommissioned, present: true, enabled: true},
|
||||||
|
"decommissioned but mounted": {intent: IntentDecommissioned, mounted: true},
|
||||||
|
"enrolled and absent": {intent: IntentEnrolled},
|
||||||
|
"ejected and absent": {intent: IntentEjected},
|
||||||
|
"no intent": {intent: IntentNew},
|
||||||
|
"mount table unreadable": {intent: IntentDecommissioned, mountErr: true},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
n, rr, _, unitDir := runForget(t, c, nil)
|
||||||
|
if n != 0 || len(rr.calls) != 0 {
|
||||||
|
t.Fatalf("n=%d calls=%v — this unit must be kept", n, rr.calls)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(unitDir, "mnt-media_a.mount")); err != nil {
|
||||||
|
t.Fatalf("unit file gone: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No intent store on the host (open failure): nothing is known to be retired, so nothing is forgotten.
|
||||||
|
func TestForgetRetiredDrives_NilIntent_ForgetsNothing(t *testing.T) {
|
||||||
|
unitDir := t.TempDir()
|
||||||
|
writeUnit(t, unitDir, "mnt-media_a.mount", renderMountUnit(MountSpec{Name: "x", UUID: forgetUUID, Where: "/mnt/media_a"}))
|
||||||
|
rr := &forgetRunner{unitDir: unitDir}
|
||||||
|
ops := NewSudoHostOps(SudoHostOpsConfig{Runner: rr, UnitDir: unitDir, StageDir: t.TempDir(), Host: &fakeHostReader{}, Logger: quietLogger()})
|
||||||
|
if n := ops.ForgetRetiredDrives(context.Background(), nil); n != 0 || len(rr.calls) != 0 {
|
||||||
|
t.Fatalf("n=%d calls=%v", n, rr.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,243 @@
|
|||||||
|
// Package thincheck finds an ORPHAN thin device: one the LVM thin pool holds that LVM itself no longer knows (R-1012).
|
||||||
|
//
|
||||||
|
// A full host root while vzdump takes its storage snapshot can leave such a device behind (demo-hp 2026-10-14: the
|
||||||
|
// pool held devices 1, 2 and 4; LVM knew 1 and 2). From then on every snapshot backup of that guest fails at the
|
||||||
|
// snapshot step („Creation of new snapshot 4 of device 2 failed") and nothing in the product said why. This package
|
||||||
|
// NAMES the orphan, in plain words, in the backup's failure — it never removes one: the repair
|
||||||
|
// (`dmsetup message <pool>-tpool 0 "delete <id>"`) stays a human step, decided by the operator (`03` whole-guest
|
||||||
|
// backup; runbook `whole-guest-backup-orphan-thin.md`).
|
||||||
|
//
|
||||||
|
// The device list comes from the root helper `felhom-thin-check <vg> <pool>` (config bundle), which reads the pool
|
||||||
|
// through a metadata snapshot and LVM's list before and after. An id is an orphan only when the pool holds it and
|
||||||
|
// NEITHER LVM list names it, on two reads a few seconds apart — so a volume being created at that moment is never
|
||||||
|
// reported.
|
||||||
|
package thincheck
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HelperPath is where the config bundle installs the root helper.
|
||||||
|
const HelperPath = "/usr/local/sbin/felhom-thin-check"
|
||||||
|
|
||||||
|
// Pool names one LVM thin pool.
|
||||||
|
type Pool struct{ VG, Pool string }
|
||||||
|
|
||||||
|
func (p Pool) String() string { return p.VG + "/" + p.Pool }
|
||||||
|
|
||||||
|
// Reading is one helper run, parsed.
|
||||||
|
type Reading struct {
|
||||||
|
PoolIDs []int
|
||||||
|
LVMBefore map[int]string
|
||||||
|
LVMAfter map[int]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse reads the helper's output. ok=false unless the closing "ok" line is there (a cut-off run is no reading).
|
||||||
|
func Parse(out string) (Reading, bool) {
|
||||||
|
r := Reading{LVMBefore: map[int]string{}, LVMAfter: map[int]string{}}
|
||||||
|
done := false
|
||||||
|
for _, line := range strings.Split(out, "\n") {
|
||||||
|
f := strings.Fields(line)
|
||||||
|
switch {
|
||||||
|
case len(f) == 1 && f[0] == "ok":
|
||||||
|
done = true
|
||||||
|
case len(f) == 2 && f[0] == "pool":
|
||||||
|
if id, err := strconv.Atoi(f[1]); err == nil {
|
||||||
|
r.PoolIDs = append(r.PoolIDs, id)
|
||||||
|
}
|
||||||
|
case len(f) == 3 && (f[0] == "lvm-before" || f[0] == "lvm-after"):
|
||||||
|
id, err := strconv.Atoi(f[1])
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if f[0] == "lvm-before" {
|
||||||
|
r.LVMBefore[id] = f[2]
|
||||||
|
} else {
|
||||||
|
r.LVMAfter[id] = f[2]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return r, done
|
||||||
|
}
|
||||||
|
|
||||||
|
// Orphans returns the ids the pool holds that neither LVM list names, sorted.
|
||||||
|
func (r Reading) Orphans() []int {
|
||||||
|
var out []int
|
||||||
|
for _, id := range r.PoolIDs {
|
||||||
|
_, b := r.LVMBefore[id]
|
||||||
|
_, a := r.LVMAfter[id]
|
||||||
|
if !b && !a {
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Ints(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runner is the privileged exec surface (proxmox.ExecRunner in sudo mode).
|
||||||
|
type Runner interface {
|
||||||
|
Run(ctx context.Context, name string, args ...string) ([]byte, []byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// StorageLister is the read the checker needs to map a guest's volumes to their thin pools (*proxmox.Client).
|
||||||
|
type StorageLister interface {
|
||||||
|
GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
|
||||||
|
ListStorage(ctx context.Context) ([]proxmox.Storage, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Checker runs the check for one guest's pools.
|
||||||
|
type Checker struct {
|
||||||
|
API StorageLister
|
||||||
|
Runner Runner
|
||||||
|
Helper string // "" → HelperPath
|
||||||
|
Pause time.Duration // between the two reads when the first finds an orphan; 0 → 3 s
|
||||||
|
Logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finding is one pool with orphans.
|
||||||
|
type Finding struct {
|
||||||
|
Pool Pool
|
||||||
|
Orphans []int
|
||||||
|
}
|
||||||
|
|
||||||
|
// PoolsOf maps the guest's volume-backed disks (rootfs, mpN) to the lvmthin pools they live on.
|
||||||
|
func (c *Checker) PoolsOf(ctx context.Context, vmid int) ([]Pool, error) {
|
||||||
|
cfg, err := c.API.GuestConfig(ctx, vmid)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
stores, err := c.API.ListStorage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
thin := map[string]Pool{}
|
||||||
|
for _, s := range stores {
|
||||||
|
if s.Type == "lvmthin" && s.VGName != "" && s.ThinPool != "" {
|
||||||
|
thin[s.Storage] = Pool{VG: s.VGName, Pool: s.ThinPool}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
vols := []string{cfg.RootFS}
|
||||||
|
mps := cfg.MountPoints()
|
||||||
|
keys := make([]string, 0, len(mps))
|
||||||
|
for k := range mps {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, k := range keys {
|
||||||
|
vols = append(vols, mps[k])
|
||||||
|
}
|
||||||
|
seen := map[Pool]bool{}
|
||||||
|
var out []Pool
|
||||||
|
for _, v := range vols {
|
||||||
|
store, _, ok := strings.Cut(strings.Split(v, ",")[0], ":")
|
||||||
|
if !ok || strings.HasPrefix(store, "/") {
|
||||||
|
continue // a bind mount (a path) is no volume
|
||||||
|
}
|
||||||
|
if p, isThin := thin[store]; isThin && !seen[p] {
|
||||||
|
seen[p] = true
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// read runs the helper once.
|
||||||
|
func (c *Checker) read(ctx context.Context, p Pool) (Reading, error) {
|
||||||
|
helper := c.Helper
|
||||||
|
if helper == "" {
|
||||||
|
helper = HelperPath
|
||||||
|
}
|
||||||
|
out, stderr, err := c.Runner.Run(ctx, helper, p.VG, p.Pool)
|
||||||
|
if err != nil {
|
||||||
|
return Reading{}, fmt.Errorf("felhom-thin-check %s: %w: %s", p, err, strings.TrimSpace(string(stderr)))
|
||||||
|
}
|
||||||
|
r, ok := Parse(string(out))
|
||||||
|
if !ok {
|
||||||
|
return Reading{}, fmt.Errorf("felhom-thin-check %s: the output has no closing ok line", p)
|
||||||
|
}
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check returns the pools of vmid that hold an orphan (two reads, intersected). A pool that cannot be read is logged
|
||||||
|
// and left out: the check FAILS OPEN — it explains a failure, it never stops a backup.
|
||||||
|
func (c *Checker) Check(ctx context.Context, vmid int) []Finding {
|
||||||
|
logger := c.Logger
|
||||||
|
if logger == nil {
|
||||||
|
logger = slog.Default()
|
||||||
|
}
|
||||||
|
pools, err := c.PoolsOf(ctx, vmid)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("thincheck: cannot map the guest's disks to thin pools — not checked", "vmid", vmid, "err", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
pause := c.Pause
|
||||||
|
if pause <= 0 {
|
||||||
|
pause = 3 * time.Second
|
||||||
|
}
|
||||||
|
var found []Finding
|
||||||
|
for _, p := range pools {
|
||||||
|
first, err := c.read(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("thincheck: thin pool not read — not checked", "vmid", vmid, "pool", p.String(), "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
o1 := first.Orphans()
|
||||||
|
if len(o1) == 0 {
|
||||||
|
logger.Debug("thincheck: thin pool and LVM agree", "vmid", vmid, "pool", p.String(), "devices", len(first.PoolIDs))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return found
|
||||||
|
case <-time.After(pause):
|
||||||
|
}
|
||||||
|
second, err := c.read(ctx, p)
|
||||||
|
if err != nil {
|
||||||
|
logger.Warn("thincheck: second read failed — not reported", "vmid", vmid, "pool", p.String(), "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
in2 := map[int]bool{}
|
||||||
|
for _, id := range second.Orphans() {
|
||||||
|
in2[id] = true
|
||||||
|
}
|
||||||
|
var both []int
|
||||||
|
for _, id := range o1 {
|
||||||
|
if in2[id] {
|
||||||
|
both = append(both, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(both) > 0 {
|
||||||
|
found = append(found, Finding{Pool: p, Orphans: both})
|
||||||
|
logger.Warn("thincheck: ORPHAN thin device(s) in the pool — LVM does not know them; NOT removed (a human step)",
|
||||||
|
"vmid", vmid, "pool", p.String(), "orphans", both)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
// Describe is the operator sentence for a backup failure. Plain words; names the pool and the ids; says nothing was
|
||||||
|
// removed and where the repair is written down.
|
||||||
|
func Describe(f []Finding) string {
|
||||||
|
if len(f) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var parts []string
|
||||||
|
for _, x := range f {
|
||||||
|
ids := make([]string, len(x.Orphans))
|
||||||
|
for i, id := range x.Orphans {
|
||||||
|
ids[i] = strconv.Itoa(id)
|
||||||
|
}
|
||||||
|
parts = append(parts, fmt.Sprintf("thin pool %s holds device %s that LVM does not know", x.Pool, strings.Join(ids, ", ")))
|
||||||
|
}
|
||||||
|
return "probable cause: " + strings.Join(parts, "; ") +
|
||||||
|
" (an orphan snapshot, usually left when the host disk filled during a backup). Snapshot backups on this pool " +
|
||||||
|
"fail until an operator removes it; Felhom did not remove anything. Repair: runbook whole-guest-backup-orphan-thin.md"
|
||||||
|
}
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
package thincheck
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The helper's output on demo-hp 2026-10-14, before the repair: the pool held 1, 2 and 4; LVM knew 1 and 2
|
||||||
|
// (audits/power-cut-fixes-2026-10-14/E/thin-ls.txt).
|
||||||
|
const orphanOut = `lvm-before 1 vm-9201-disk-0
|
||||||
|
lvm-before 2 vm-9201-disk-1
|
||||||
|
pool 1
|
||||||
|
pool 2
|
||||||
|
pool 4
|
||||||
|
lvm-after 1 vm-9201-disk-0
|
||||||
|
lvm-after 2 vm-9201-disk-1
|
||||||
|
ok
|
||||||
|
`
|
||||||
|
|
||||||
|
const healthyOut = `lvm-before 1 vm-9201-disk-0
|
||||||
|
lvm-before 2 vm-9201-disk-1
|
||||||
|
pool 1
|
||||||
|
pool 2
|
||||||
|
lvm-after 1 vm-9201-disk-0
|
||||||
|
lvm-after 2 vm-9201-disk-1
|
||||||
|
ok
|
||||||
|
`
|
||||||
|
|
||||||
|
func quiet() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }
|
||||||
|
|
||||||
|
func TestParse_TheOrphanOf20261014(t *testing.T) {
|
||||||
|
r, ok := Parse(orphanOut)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("a complete reading was not accepted")
|
||||||
|
}
|
||||||
|
if got := r.Orphans(); !reflect.DeepEqual(got, []int{4}) {
|
||||||
|
t.Fatalf("orphans = %v, want [4]", got)
|
||||||
|
}
|
||||||
|
h, _ := Parse(healthyOut)
|
||||||
|
if got := h.Orphans(); len(got) != 0 {
|
||||||
|
t.Fatalf("a healthy pool reported orphans %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A volume LVM created WHILE the pool was read appears only in the after-list: never an orphan.
|
||||||
|
func TestOrphans_AVolumeCreatedDuringTheReadIsNotAnOrphan(t *testing.T) {
|
||||||
|
r, _ := Parse(strings.Replace(orphanOut, "lvm-after 2 vm-9201-disk-1\n", "lvm-after 2 vm-9201-disk-1\nlvm-after 4 snap_vm-9201-disk-1_vzdump\n", 1))
|
||||||
|
if got := r.Orphans(); len(got) != 0 {
|
||||||
|
t.Fatalf("orphans = %v, want none", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A run cut off before its closing "ok" (the helper died, the output is partial) is no reading at all.
|
||||||
|
func TestParse_NoClosingLineIsNoReading(t *testing.T) {
|
||||||
|
if _, ok := Parse(strings.TrimSuffix(orphanOut, "ok\n")); ok {
|
||||||
|
t.Fatal("a cut-off run was accepted as a reading")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeAPI struct {
|
||||||
|
cfg proxmox.GuestConfig
|
||||||
|
stores []proxmox.Storage
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeAPI) GuestConfig(context.Context, int) (proxmox.GuestConfig, error) { return f.cfg, nil }
|
||||||
|
func (f *fakeAPI) ListStorage(context.Context) ([]proxmox.Storage, error) { return f.stores, nil }
|
||||||
|
|
||||||
|
func guest9201() *fakeAPI {
|
||||||
|
extra := map[string]json.RawMessage{
|
||||||
|
"mp0": json.RawMessage(`"local-lvm:vm-9201-disk-1,mp=/var/lib/felhom,backup=1,size=20G"`),
|
||||||
|
"mp8": json.RawMessage(`"/mnt/felhom-drives,mp=/mnt/felhom-drives"`),
|
||||||
|
"mp9": json.RawMessage(`"local:9201/vm-9201-disk-2.raw,mp=/x"`),
|
||||||
|
}
|
||||||
|
return &fakeAPI{
|
||||||
|
cfg: proxmox.GuestConfig{RootFS: "local-lvm:vm-9201-disk-0,size=8G", Extra: extra},
|
||||||
|
stores: []proxmox.Storage{
|
||||||
|
{Storage: "local", Type: "dir", Path: "/var/lib/vz"},
|
||||||
|
{Storage: "local-lvm", Type: "lvmthin", VGName: "pve", ThinPool: "data"},
|
||||||
|
{Storage: "felhom-pbs", Type: "pbs"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type seqRunner struct {
|
||||||
|
outs []string
|
||||||
|
errs []error
|
||||||
|
calls [][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *seqRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||||
|
r.calls = append(r.calls, append([]string{name}, args...))
|
||||||
|
i := len(r.calls) - 1
|
||||||
|
var err error
|
||||||
|
if i < len(r.errs) {
|
||||||
|
err = r.errs[i]
|
||||||
|
}
|
||||||
|
if i < len(r.outs) {
|
||||||
|
return []byte(r.outs[i]), nil, err
|
||||||
|
}
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPoolsOf_RootfsAndVolumeMountpointsOnly(t *testing.T) {
|
||||||
|
c := &Checker{API: guest9201(), Runner: &seqRunner{}, Logger: quiet()}
|
||||||
|
pools, err := c.PoolsOf(context.Background(), 9201)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(pools, []Pool{{VG: "pve", Pool: "data"}}) {
|
||||||
|
t.Fatalf("pools = %v, want [pve/data] once (bind mounts and dir volumes are not thin)", pools)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The consequence: an orphan seen on BOTH reads becomes a finding whose sentence names the pool and the id, and says
|
||||||
|
// nothing was removed. The only command ever run is the read-only helper with <vg> <pool>.
|
||||||
|
// RED-PROOF (P1/redproof-thincheck.txt): with Orphans() returning nil this test fails.
|
||||||
|
func TestCheck_OrphanOnBothReads_IsNamed(t *testing.T) {
|
||||||
|
rr := &seqRunner{outs: []string{orphanOut, orphanOut}}
|
||||||
|
c := &Checker{API: guest9201(), Runner: rr, Pause: 1, Logger: quiet()}
|
||||||
|
f := c.Check(context.Background(), 9201)
|
||||||
|
if len(f) != 1 || f[0].Pool.String() != "pve/data" || !reflect.DeepEqual(f[0].Orphans, []int{4}) {
|
||||||
|
t.Fatalf("findings = %+v", f)
|
||||||
|
}
|
||||||
|
want := [][]string{{HelperPath, "pve", "data"}, {HelperPath, "pve", "data"}}
|
||||||
|
if !reflect.DeepEqual(rr.calls, want) {
|
||||||
|
t.Fatalf("calls = %v, want only the read-only helper twice", rr.calls)
|
||||||
|
}
|
||||||
|
d := Describe(f)
|
||||||
|
for _, s := range []string{"thin pool pve/data holds device 4 that LVM does not know", "Felhom did not remove anything",
|
||||||
|
"whole-guest-backup-orphan-thin.md"} {
|
||||||
|
if !strings.Contains(d, s) {
|
||||||
|
t.Fatalf("Describe = %q, missing %q", d, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An id that is gone on the second read (a volume mid-creation) is not reported; a healthy pool is read ONCE.
|
||||||
|
func TestCheck_TransientOrHealthy_NothingReported(t *testing.T) {
|
||||||
|
rr := &seqRunner{outs: []string{orphanOut, healthyOut}}
|
||||||
|
if f := (&Checker{API: guest9201(), Runner: rr, Pause: 1, Logger: quiet()}).Check(context.Background(), 9201); len(f) != 0 {
|
||||||
|
t.Fatalf("a transient id was reported: %+v", f)
|
||||||
|
}
|
||||||
|
rr = &seqRunner{outs: []string{healthyOut}}
|
||||||
|
if f := (&Checker{API: guest9201(), Runner: rr, Logger: quiet()}).Check(context.Background(), 9201); len(f) != 0 || len(rr.calls) != 1 {
|
||||||
|
t.Fatalf("healthy: findings %+v, calls %v", f, rr.calls)
|
||||||
|
}
|
||||||
|
if Describe(nil) != "" {
|
||||||
|
t.Fatal("no finding must describe as empty")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The helper refusing (a metadata snapshot held by someone else, exit 3) or failing: no finding, no panic — the check
|
||||||
|
// fails open; it explains a failure, it never stops a backup.
|
||||||
|
func TestCheck_HelperRefuses_FailsOpen(t *testing.T) {
|
||||||
|
rr := &seqRunner{errs: []error{errors.New("exit status 3")}}
|
||||||
|
if f := (&Checker{API: guest9201(), Runner: rr, Logger: quiet()}).Check(context.Background(), 9201); len(f) != 0 {
|
||||||
|
t.Fatalf("findings = %+v", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The root helper's own tests (configs/test_felhom_thin_check.py) run with the Go suite.
|
||||||
|
func TestRootHelperPythonSuite(t *testing.T) {
|
||||||
|
py, err := exec.LookPath("python3")
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("python3 not found")
|
||||||
|
}
|
||||||
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
script := filepath.Join(filepath.Dir(file), "..", "..", "configs", "test_felhom_thin_check.py")
|
||||||
|
out, err := exec.Command(py, script).CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("helper suite failed: %v\n%s", err, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -15,6 +15,9 @@ Gates (all must pass; **non-zero exit on any failure**):
|
|||||||
or because someone is offline on a train. CI runs the FULL set for exactly this
|
or because someone is offline on a train. CI runs the FULL set for exactly this
|
||||||
reason: it is the machine that can afford a network check, and it is the half
|
reason: it is the machine that can afford a network check, and it is the half
|
||||||
that emails when something is wrong.
|
that emails when something is wrong.
|
||||||
|
… gate-decoys the decoy suite (test_gate_decoys.py): every gate here still convicts its decoys
|
||||||
|
(R-1116). ~20 s in the workspace, so the full run only: not the hook, but CI
|
||||||
|
runs the full set on every push. Scratch clones and a local fake Gitea only.
|
||||||
|
|
||||||
WHY THIS FILE EXISTS, WITH ONE GATE (2026-08-02, R-29 leg (b)).
|
WHY THIS FILE EXISTS, WITH ONE GATE (2026-08-02, R-29 leg (b)).
|
||||||
|
|
||||||
@@ -58,6 +61,12 @@ GATES = [
|
|||||||
("release-complete", os.path.join(ROOT, "scripts", "check-release-complete.py"), [], True),
|
("release-complete", os.path.join(ROOT, "scripts", "check-release-complete.py"), [], True),
|
||||||
# R-389 — a REPORT.md observation with no register row behind it. Fast: stdlib file reads.
|
# R-389 — a REPORT.md observation with no register row behind it. Fast: stdlib file reads.
|
||||||
("observations", SHARED_OBSERVATIONS, [ROOT], True),
|
("observations", SHARED_OBSERVATIONS, [ROOT], True),
|
||||||
|
# R-1116 (2026-10-17) — the gates' OWN decoy suite. It was in no runner, the shape that let the catalog's
|
||||||
|
# R-1115 fail unseen for a night. Full run only: ~20 s in the workspace (measured, 30 cases; ~5 s in a
|
||||||
|
# CI-shaped clone, where the shared gates have fewer siblings to search), too slow for the hook — and CI
|
||||||
|
# runs the FULL set, so every push is still judged by it. It plants only in scratch clones and asks a fake
|
||||||
|
# Gitea on 127.0.0.1 (never the network, never the real tree); it does not call this runner (no recursion).
|
||||||
|
("gate-decoys", os.path.join(ROOT, "scripts", "test_gate_decoys.py"), [], False),
|
||||||
]
|
]
|
||||||
|
|
||||||
VERDICT = {0: "OK", 1: "FAILED", 2: "INCONCLUSIVE"}
|
VERDICT = {0: "OK", 1: "FAILED", 2: "INCONCLUSIVE"}
|
||||||
|
|||||||
@@ -69,6 +69,10 @@ fi
|
|||||||
existing="$(curl -fsS -o /dev/null -w '%{http_code}' \
|
existing="$(curl -fsS -o /dev/null -w '%{http_code}' \
|
||||||
"$GITEA_BASE/api/packages/$GITEA_OWNER/generic/felhom-agent/$VERSION/felhom-agent" 2>/dev/null || true)"
|
"$GITEA_BASE/api/packages/$GITEA_OWNER/generic/felhom-agent/$VERSION/felhom-agent" 2>/dev/null || true)"
|
||||||
[[ "$existing" != "200" ]] || die "version $VERSION is ALREADY PUBLISHED — bump the version instead"
|
[[ "$existing" != "200" ]] || die "version $VERSION is ALREADY PUBLISHED — bump the version instead"
|
||||||
|
# R-1098 (2026-10-16): only a clean 404 proves the version is new. Until now a registry that could not be reached
|
||||||
|
# (curl prints 000) or answered anything else passed this check — „I could not ask" is not that proof.
|
||||||
|
# Test: scripts/test_release_push_once.sh.
|
||||||
|
[[ "$existing" == "404" ]] || die "could not establish whether $VERSION is published (registry answered HTTP '${existing:-none}') — refusing"
|
||||||
|
|
||||||
# ── 3. Build ────────────────────────────────────────────────────────────────────────────────────
|
# ── 3. Build ────────────────────────────────────────────────────────────────────────────────────
|
||||||
BIN="$(mktemp -t felhom-agent-XXXXXX)"
|
BIN="$(mktemp -t felhom-agent-XXXXXX)"
|
||||||
|
|||||||
@@ -57,6 +57,20 @@ class AgentGatesTest(unittest.TestCase):
|
|||||||
finally:
|
finally:
|
||||||
shutil.rmtree(tmp, ignore_errors=True)
|
shutil.rmtree(tmp, ignore_errors=True)
|
||||||
|
|
||||||
|
def test_decoy_suite_is_a_full_run_gate(self):
|
||||||
|
"""R-1116: the decoy suite is a gate of the FULL run (which CI runs on every push), and --fast
|
||||||
|
names it as skipped (~20 s, too slow for the hook)."""
|
||||||
|
sys.dont_write_bytecode = True # no __pycache__ left in scripts/ (not gitignored here)
|
||||||
|
sys.path.insert(0, os.path.join(ROOT, "scripts"))
|
||||||
|
import agent_gates as mod
|
||||||
|
rows = [g for g in mod.GATES if g[0] == "gate-decoys"]
|
||||||
|
self.assertEqual(len(rows), 1, "gate-decoys is not registered exactly once")
|
||||||
|
self.assertEqual(os.path.basename(rows[0][1]), "test_gate_decoys.py")
|
||||||
|
self.assertFalse(rows[0][3], "gate-decoys must not be --fast")
|
||||||
|
self.assertEqual([g[0] for g in mod.GATES if not g[3]], ["published", "gate-decoys"])
|
||||||
|
self.assertIn("--fast SKIPPED (deliberate periodic runs, never in a hook): published, gate-decoys",
|
||||||
|
self.out)
|
||||||
|
|
||||||
def test_unknown_argument_is_rejected(self):
|
def test_unknown_argument_is_rejected(self):
|
||||||
p = subprocess.run([sys.executable, ENTRY, "--nope"], cwd=ROOT,
|
p = subprocess.run([sys.executable, ENTRY, "--nope"], cwd=ROOT,
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||||
|
|||||||
@@ -66,6 +66,9 @@ COVERS = {
|
|||||||
"inside an HTML comment (R-426)",
|
"inside an HTML comment (R-426)",
|
||||||
"observations": "R-419 in THIS repo's REPORT.md: an Observations note SAYING it carries no marker - vs the "
|
"observations": "R-419 in THIS repo's REPORT.md: an Observations note SAYING it carries no marker - vs the "
|
||||||
"two genuine markers (R-426)",
|
"two genuine markers (R-426)",
|
||||||
|
"gate-decoys": "this suite, run as a gate of agent_gates.py's full run, which CI runs on every push (R-1116): "
|
||||||
|
"every case above is a decoy of its own gate, and ONE case that stops convicting turns the run "
|
||||||
|
"red - red-proved 2026-10-17 by flipping a reuse-refs FACT case to expect rc 0",
|
||||||
}
|
}
|
||||||
|
|
||||||
fails = []
|
fails = []
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# test_release_push_once.sh — release-agent.sh releases a version only when the registry says it is NEW (R-1098).
|
||||||
|
#
|
||||||
|
# The REAL release-agent.sh runs from a copy inside a scratch git repo (so its tag check reads scratch tags), with
|
||||||
|
# RELEASE_ALLOW_DIRTY=1 (the clean-tree gate has its own history) and a fake `curl` and `go` first on PATH: curl answers
|
||||||
|
# the package probe as FAKE_MODE says, and `go` stops the run right after the check, so nothing is built or published.
|
||||||
|
# published (200) -> refused, "ALREADY PUBLISHED"
|
||||||
|
# absent (404) -> passes the check, reaches the build
|
||||||
|
# unreachable (000) -> refused, "could not establish"
|
||||||
|
# server error (500) -> refused, "could not establish"
|
||||||
|
# the tag already exists -> refused, "already exists"
|
||||||
|
# Run: bash scripts/test_release_push_once.sh (exit 0 all cases right · 1 a case wrong)
|
||||||
|
set -uo pipefail
|
||||||
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
WS="$(mktemp -d)"; trap 'rm -rf "${WS:?}"' EXIT
|
||||||
|
fails=0; ran=0
|
||||||
|
mkdir -p "$WS/bin" "$WS/repo/scripts"
|
||||||
|
cp "$HERE/release-agent.sh" "$WS/repo/scripts/release-agent.sh"
|
||||||
|
(cd "$WS/repo" && git init -q -b main && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m init)
|
||||||
|
cat > "$WS/bin/curl" <<'FAKE'
|
||||||
|
#!/bin/bash
|
||||||
|
case "${FAKE_MODE}" in
|
||||||
|
published) printf 200; exit 0 ;;
|
||||||
|
absent) printf 404; exit 22 ;;
|
||||||
|
down) printf 000; exit 7 ;;
|
||||||
|
error) printf 500; exit 22 ;;
|
||||||
|
esac
|
||||||
|
FAKE
|
||||||
|
printf '#!/bin/sh\necho "FAKE-GO-REACHED $*"; exit 1\n' > "$WS/bin/go"
|
||||||
|
chmod +x "$WS/bin/curl" "$WS/bin/go"
|
||||||
|
run() { # name, mode, expected rc, required fragment, forbidden fragment
|
||||||
|
local out rc
|
||||||
|
out=$(cd "$WS/repo" && FAKE_MODE="$2" RELEASE_ALLOW_DIRTY=1 GITEA_USER=u GITEA_TOKEN=t PATH="$WS/bin:$PATH" \
|
||||||
|
bash scripts/release-agent.sh 9.9.9 2>&1); rc=$?
|
||||||
|
ran=$((ran+1))
|
||||||
|
if [[ "$rc" != "$3" ]] || ! grep -qi -- "$4" <<<"$out" || { [[ -n "$5" ]] && grep -qi -- "$5" <<<"$out"; }; then
|
||||||
|
fails=$((fails+1)); echo " FAIL $1 (exit $rc)"; echo "$out" | tail -4 | sed 's/^/ /'; return
|
||||||
|
fi
|
||||||
|
echo " ok $1"
|
||||||
|
}
|
||||||
|
echo "release-agent.sh — a version is released once, and only when the registry says it is new"
|
||||||
|
run "published (200) -> refused" published 1 "ALREADY PUBLISHED" "FAKE-GO-REACHED"
|
||||||
|
run "absent (404) -> proceeds" absent 1 "FAKE-GO-REACHED" "could not establish"
|
||||||
|
run "unreachable (000) -> refused" down 1 "could not establish" "FAKE-GO-REACHED"
|
||||||
|
run "server error (500) -> refused" error 1 "could not establish" "FAKE-GO-REACHED"
|
||||||
|
(cd "$WS/repo" && git tag v9.9.9)
|
||||||
|
run "the tag exists -> refused" absent 1 "already exists" "FAKE-GO-REACHED"
|
||||||
|
echo
|
||||||
|
if [[ $fails -eq 0 ]]; then echo "all $ran cases behaved"; exit 0; fi
|
||||||
|
echo "$fails of $ran cases FAILED"; exit 1
|
||||||
Reference in New Issue
Block a user