From 94fc3ba9c897be74d143e300a0536f94cb50236f Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 14 Oct 2026 11:55:19 +0200 Subject: [PATCH] =?UTF-8?q?Power-cut=20fixes:=20spike=20C1=E2=80=93C4=20+?= =?UTF-8?q?=20Immich=20repair=20spike=20(F);=20docs=2003/07/13/14;=20R-100?= =?UTF-8?q?3=20+=20R-1000=20closed=20(FAQ=20live),=20R-1006..R-1009=20narr?= =?UTF-8?q?owed,=20R-1010=20filed;=20estimator=20raise=20STOPPED=20(one=20?= =?UTF-8?q?verdict=20flips);=20uninstall=20knows=20the=20new=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- STATUS.md | 10 +- documentation/architecture/03-host-agent.md | 1 + .../architecture/07-backup-architecture.md | 12 ++ documentation/architecture/13-threat-model.md | 1 + .../architecture/14-host-power-off.md | 2 + .../power-cut-fixes-2026-10-14/C2/after.txt | 53 +++++ .../power-cut-fixes-2026-10-14/C2/before.txt | 1 + .../C2/hand-cleanup.txt | 3 + .../power-cut-fixes-2026-10-14/C2/kill.txt | 6 + .../C2/snapshot-delete-err.txt | 1 + .../C3/before-cut.txt | 5 + .../power-cut-fixes-2026-10-14/C3/monitor.txt | 191 +++++++++++++++++ .../power-cut-fixes-2026-10-14/C3/plugcut.log | 5 + .../C3/requests.txt | 11 + .../power-cut-fixes-2026-10-14/C4/after.txt | 7 + .../power-cut-fixes-2026-10-14/C4/fill.txt | 3 + .../C4/hand-fix.txt | 11 + .../C4/vzdump-9201.txt | 25 +++ .../power-cut-fixes-2026-10-14/C4/vzdump.txt | 24 +++ .../F/immich-delete-then-missing.txt | 9 + .../F/immich-repair.txt | 88 ++++++++ .../power-cut-fixes-2026-10-14/README.md | 117 +++++++++++ .../power-cut-fixes-2026-10-14/redproof.txt | 19 ++ .../audits/sizing-check-2026-10-14/CHECK.md | 20 ++ .../sizing-check-2026-10-14/hp-figures.json | 195 ++++++++++++++++++ .../raise-only-source.patch | 68 ++++++ .../tools/hp_figures.py | 32 +++ documentation/backlog/CLOSED-ITEMS.md | 7 + documentation/backlog/OPEN-ITEMS.md | 15 +- scripts/CHANGELOG.md | 7 + scripts/felhom-host-install.sh | 2 + scripts/test_hostinstall.py | 1 + 32 files changed, 939 insertions(+), 13 deletions(-) create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C2/after.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C2/before.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C2/hand-cleanup.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C2/kill.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C2/snapshot-delete-err.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C3/before-cut.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C3/monitor.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C3/plugcut.log create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C3/requests.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C4/after.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C4/fill.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C4/hand-fix.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C4/vzdump-9201.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/C4/vzdump.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/F/immich-delete-then-missing.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/F/immich-repair.txt create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/README.md create mode 100644 documentation/audits/power-cut-fixes-2026-10-14/redproof.txt create mode 100644 documentation/audits/sizing-check-2026-10-14/hp-figures.json create mode 100644 documentation/audits/sizing-check-2026-10-14/raise-only-source.patch create mode 100644 documentation/audits/sizing-check-2026-10-14/tools/hp_figures.py diff --git a/STATUS.md b/STATUS.md index 4a42c7ea..c72c097c 100644 --- a/STATUS.md +++ b/STATUS.md @@ -18,10 +18,10 @@ other two until the floor is raised. Open items: see `documentation/backlog/OPEN - DooPlex: a failed disk with its alarms silenced (R-934); CI jobs still get lost (R-887); something sets the shared checkout to „bare" (R-935); a Longhorn volume at 91 %. - Tester 2 has never made an off-site check (R-862). The scratch box cannot read the drill catalog (R-992). -- After a power cut: a cut during the night backup leaves a half-backup that blocks every later backup (R-1006, P2); Immich keeps photos with empty thumbnails or no original (R-1007). +- After a power cut: the half-backup that blocked every later backup (R-1006) is fixed in agent 0.156.1 — waits on your vouch; Immich keeps photos with empty previews or no original until its own repair job runs (R-1007, your decision). ## What we work on -- The power-off button reaches the other boxes when you vouch and raise the floor (below). +- Agent 0.156.1 (the power-cut clean-up) and the power-off button reach the boxes when you vouch and raise the floor (below); then CC proves the clean-up with one more cut on demo-hp. - The power-cut findings (R-1004 to R-1008) and the estimator figures (R-1009, your choice). ## Waiting on you — decide @@ -36,11 +36,11 @@ other two until the floor is raised. Open items: see `documentation/backlog/OPEN | D6 | R-992 | The scratch box cannot read the private drill catalog: give it a read-only login, or make that copy readable? | **A read-only login** | Readable copy: the drill catalog becomes public | every new-app test needs hand-staging | new-app tests | | D7 | R-990, R-989 | Rule yes / no / later on the forum ideas (`audits/forum-research-2026-10-12/FELHOM.md` §5) and the selfh.st ideas (`audits/selfhst-survey-2026-10-12/FELHOM.md` §6) | — | — | nothing changes; the research ages | — | | D8 | R-206 | Install DooPlex's Docker settings from git (restarts Docker), and prune the build cache weekly: yes or no? | **Yes, in a quiet window** | — | a rebuild of DooPlex still has the settings in git | — | -| D9 | R-1003 (P1) | The FAQ promises a BIOS power-on setting „at every installation"; a self-install never gets it. Reword the FAQ, or make the install check it? | **Reword now** (the household sets it, with the menu names), check it in the install later | Install check: a build, and firmware rarely exposes it | the FAQ keeps a promise the product does not keep; a box can stay off after a cut | — | -| D10 | R-1009 | Move the estimator's figures with the night's check on a second machine? | **A: a small figure update**, no verdict changes (`audits/sizing-check-2026-10-14/CHECK.md`) | A + B: also a sentence for Immich on 8 GB, two strings | the page keeps today's figures; its three answers were all true | — | +| D9 | R-1007 | After an unclean boot, should Immich run its own repair: the „missing files" check (report a lost photo) and „thumbnails: all" (fix empty previews) in the night window? | **Yes** | CPU in proportion to the library (a big one: hours, at night) | broken previews stay; a lost original stays unnoticed | — | +| D10 | R-1009 | The „raise only" estimator update changes ONE verdict: 5 apps without Immich on 8 GB, fits well → tight (0.5 MB over). Publish anyway (A), round down first (B), or keep today's figures (C)? | **A** — the page stays truthful, one line reads stricter | B: one rule sentence, may still flip; C: nothing | the page keeps today's figures; Immich, Jellyfin, Home Assistant, Paperless read low | — | ## Waiting on you — do -- **Vouch** agent 0.155.0 + bundle; **raise the floor** to controller 0.310.0 with `min_agent` 0.155.0. If not: only the N100 has the power-off button, and a fresh install gets 0.154.0. +- **Vouch agent 0.156.1** + its bundle (NOT 0.156.0 — superseded; 0.156.1 includes 0.155.0); **raise the floor** to controller 0.310.0 with `min_agent` 0.155.0. If not: only the N100 has the power-off button, a fresh install gets 0.154.0, and a power cut mid-backup still blocks every later backup. - **Golden bake** at the weekly slot (R-468). If not: the waiver ends 2026-10-20 and pushes stop at the gate. - **N100 BIOS:** read „After Power Loss" (the HP's „Previous State" made it start after 8 of 8 cuts). **HP BIOS:** „Bypass F1 Prompt on Configuration Changes" → Enable. - **Pull the failed disk** Z6A07P2G (R-934; the twin Z6A070XZ is `/mnt/3_hdd`). If not: its alarms return on 11 November. diff --git a/documentation/architecture/03-host-agent.md b/documentation/architecture/03-host-agent.md index ad1c6e19..ebfc41a1 100644 --- a/documentation/architecture/03-host-agent.md +++ b/documentation/architecture/03-host-agent.md @@ -112,6 +112,7 @@ fixed file, delivered by the signed config bundle): | `FELHOM_INTERMEDIARY` | the shared drive parent + live drive binds | boot script + unit are FIXED bundle files (the agent only enables the unit); one-segment drive names (no leading dot, no `..`) | — | | `FELHOM_CONTROLLERSWAP` | the managed controller update | exact vmid; image ref pinned to `gitea.dooplex.hu/admin/felhom-controller:X.Y.Z` for the image check; the inspect template stays free text; **the write goes through `felhom-priv-apply controller-image `** (the ref on stdin, re-checked as root against the same pattern — R-861 (a) A1, `09` §3 decision 165, agent after v0.150.0) | guest-scoped: a compromised agent can restart the guest's bootstrap and choose an OLDER version of OUR controller image (a downgrade); it can no longer hand the guest any other image. Until the A1 bundle reached a box, the agent's in-guest `tee` let it write any ref (sudo cannot see stdin — the 2026-10-06 night correction, `audits/night-burndown-2026-10-06/design-R-861.md`) | | `FELHOM_STALELOCK` / `FELHOM_SCRATCH_TEARDOWN` | stale-lock clear; failed restore-test scratch | exact vmid; the scratch band `99000[0-9]` was already exact | — | +| `FELHOM_VZPARTIAL` | remove a dead half-written vzdump (R-1006, agent v0.156.1) | ONE line: `felhom-vzdump-partial-clean `; the root helper (a config-bundle file) re-checks: a `dir` storage from `storage.cfg`, the name of that vmid (`.tar.dat`, `.vma.dat`, `.tmp`), not a symlink, directly in `/dump`, no vzdump on the host, nothing holds it open, dead (before this boot or quiet 120 s) | a compromised agent can delete a dead partial — never a finished archive, never a live one | | `FELHOM_WG` | the off-site tunnel | conf via `felhom-priv-apply wg` (only the keys `renderConf` writes; no `PostUp`/`PreUp`/`DNS`/`Table`; `/32` only) | — | | `FELHOM_SELFUPDATE` | commit / rollback of the A/B flip | **`apply` removed**: the flip runs only inside `felhom-os-apply agent_update`, after the operator signature, host, window and nonce are checked as root and the staged bytes are hashed ONCE and copied to a root-owned dir (`/var/lib/felhom-os-apply/agent-update/`); the wrapper accepts only that dir | — | | `FELHOM_SSHD` | the out-of-band operator sshd | config via `felhom-priv-apply sshd-config` (the ONE template, only the Port varies, never 22); the felhom-op key via `sshd-key` (one plain key, no `command=`/`from=` options) | felhom-op's key itself is hub-delivered, not signed: a compromised agent can install its own key for **felhom-op** — whose sudo is scoped (`felhom-op.sudoers`: fixed repair verbs and `pct start` / `stop` / `unlock` of ONE numeric vmid, anchored since R-861 (b) B2), not root — accepted (`09` §3 decision 165, B3) | diff --git a/documentation/architecture/07-backup-architecture.md b/documentation/architecture/07-backup-architecture.md index 3be595b9..7bfa2f42 100644 --- a/documentation/architecture/07-backup-architecture.md +++ b/documentation/architecture/07-backup-architecture.md @@ -446,6 +446,18 @@ keep-last=1** (operator ruling 2026-09-24, option A; R-684). PVE prunes AFTER a must hold keep-last + 1 archives during the run — measured: a 9201 archive is 8.18 GB, the root disk 39 GB. The warning that should precede such a failure is R-685. +**[FACT, 2026-10-14] A power cut during the whole-guest backup leaves a half-written archive, and from agent v0.156.1 the +box removes it.** vzdump writes `vzdump-lxc--.tar.dat` (+ a `.tmp/` directory) and renames it only at the end; +a power cut leaves both (demo-hp 2026-10-14: 12.2 GB on the host root, 92 %); Proxmox never lists it; every later backup was +skipped „not enough space” (R-1006). On „No space left” vzdump removes its own partial (both its rsync and its tar path, +measured) — so the space preflight's sizing (R-685) stays as it is. The agent (`internal/vzpartial`) removes a dead partial +of a guest at its start and before each backup of that guest: a `dir` target only, the exact name of that guest, no vzdump of +it in flight (the worker process is gone — Proxmox's own pid + pstart rule, R-1008), modified before this boot or untouched +for 120 s; the root helper `felhom-vzdump-partial-clean` re-checks all of it as root and adds: no vzdump on the host at all, +nothing holds the entry open (a killed worker leaves `tar`/`zstd` writing for minutes). Evidence: +`audits/power-cut-fixes-2026-10-14/`. A full host root also breaks LVM's own metadata writes (C4) — the reason this matters +beyond one skipped night. + **[FACT] What the whole-guest tiers do NOT carry.** `mp8 /mnt/felhom-drives` and `mp9 /etc/felhom-bootstrap` are **host bind mounts** and are out of `vzdump` scope entirely (LIVE `pct config 9201`, both hosts). So the whole-guest tiers carry the guest and **none of the customer's diff --git a/documentation/architecture/13-threat-model.md b/documentation/architecture/13-threat-model.md index 81d9b470..61fc1521 100644 --- a/documentation/architecture/13-threat-model.md +++ b/documentation/architecture/13-threat-model.md @@ -60,6 +60,7 @@ OS packages (exact versions, apt-signed), the installer script (by tag, R-980). | end-user ↔ apps | holds, with gaps | traefik routing and header stripping hold; homepage has the Docker socket and no gate (R-981) | | customer ↔ controller UI | partly | bcrypt and session hold; small gaps (R-983); family sessions survive a reset (R-976) | | controller ↔ agent | partly | every route bound to the caller's guest; a compromised controller can still flush restore points (R-973), void the recovery code (R-978), and — since agent v0.155.0 — switch the whole machine off once per power-on, holding the hub's silence alarms for up to 14 days (R-998; mitigations and residual risk: `14-host-power-off.md` §5) | +| **added:** agent ↔ root on the host | holds by the sudoers | every root verb is one regex-anchored line or a root wrapper that re-checks its input (`03` §3); agent v0.156.1 adds one delete: `felhom-vzdump-partial-clean` removes only a DEAD half-written vzdump of the named guest on a `dir` storage (no vzdump on the host, nothing holding it open) — a compromised agent gains the deletion of a leftover that is not a backup | | agent ↔ hub | holds for destructive ops | signed jobs verified in the documented order; unsigned spec has no size bounds (R-984) | | controller ↔ hub | **did not hold until 2026-10-12** | the global key was public (R-950, rotated) and three routes needed no key (R-951, fixed); per-box keys are bound to their box (R-955) | | box ↔ PBS | holds as written | ciphertext only; the zero-knowledge wording is stronger than a live box allows (R-985) | diff --git a/documentation/architecture/14-host-power-off.md b/documentation/architecture/14-host-power-off.md index 7a1daea3..263e519a 100644 --- a/documentation/architecture/14-host-power-off.md +++ b/documentation/architecture/14-host-power-off.md @@ -35,6 +35,8 @@ came back (BIOS „last state", inferred), its power button did (dashboard in 42 | any other guest or VM running (`/proc`: `lxc-start -n N`, `kvm -id N`) | `other_guests_running` | `RefusedWhileOtherGuestsRun`, `RunningGuestsFromProc` | | heavy-operation gate held (backup, restore test, OS step, sweep, trim) | `backup_running` / `restore_test_running` / `os_update_running` / `host_busy` | `RefusedDuringBackup` | | backup job, controller swap, drive format, NAS verify, escrow ceremony, Proxmox vzdump | `backup_running` / `controller_update_running` / `drive_operation_running` / `escrow_ceremony_running` | `RefusedDuringVzdump`, `FailsClosedWithoutTaskReader` | + +**„A Proxmox vzdump runs” (agent v0.156.1, R-1008):** a vzdump task for the guest in Proxmox's active list whose worker process still lives — Proxmox's own rule (the UPID's pid + pstart against `/proc//stat`). After a power cut the dead task stays listed for ~30–60 s; it no longer counts. Doubt (an unparsable task, an unreadable `/proc`) still refuses. The same answer serves the start-up stale-lock pass, the guest-power watchdog and the controller supervisor (`stalelock_inflight_test.go`). | after acceptance: the heavy gate is TAKEN and kept; before each step the jobs and other guests are read again — a job waits ≤ 15 min, another guest cancels at once, a job still running cancels | (log) | `JobStartedAfterAcceptance`, `OtherGuestStartedAfterAcceptance` | **[DESIGN]** A cancelled power-off leaves the box on (gate released) — better a box that did not switch off than a diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C2/after.txt b/documentation/audits/power-cut-fixes-2026-10-14/C2/after.txt new file mode 100644 index 00000000..8cb4c36c --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C2/after.txt @@ -0,0 +1,53 @@ +== 08:09:51 +-rw-r--r-- 1 root root 1680048128 Oct 14 10:09 vzdump-lxc-9201-2026_10_14-10_08_05.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:08 vzdump-lxc-9201-2026_10_14-10_08_05.tmp +1219394 /bin/bash -c set -o pipefail && lxc-usernsexec -m u:0:100000:65536 -m g:0:100000:6 +1219395 lxc-usernsexec -m u:0:100000:65536 -m g:0:100000:65536 -- tar cpf - --totals --one +1219396 zstd --threads=1 +1219399 tar cpf - --totals --one-file-system -p --sparse --numeric-owner --acls --xattrs - +1229377 bash -c echo "== $(date -u +%T)"; ls -la /var/lib/vz/dump | grep -E "tar.dat|\.tmp +lock: backup +active vzdump: [] +listed: [('local:backup/vzdump-lxc-9201-2026_10_14-09_28_50.tar.zst', 5648463064)] +== 08:10:14 +-rw-r--r-- 1 root root 2113183744 Oct 14 10:10 vzdump-lxc-9201-2026_10_14-10_08_05.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:08 vzdump-lxc-9201-2026_10_14-10_08_05.tmp +1219394 /bin/bash -c set -o pipefail && lxc-usernsexec -m u:0:100000:65536 -m g:0:100000:6 +1219395 lxc-usernsexec -m u:0:100000:65536 -m g:0:100000:65536 -- tar cpf - --totals --one +1219396 zstd --threads=1 +1219399 tar cpf - --totals --one-file-system -p --sparse --numeric-owner --acls --xattrs - +1230532 bash -c echo "== $(date -u +%T)"; ls -la /var/lib/vz/dump | grep -E "tar.dat|\.tmp +lock: backup +active vzdump: [] +listed: [('local:backup/vzdump-lxc-9201-2026_10_14-09_28_50.tar.zst', 5648463064)] +== 08:10:38 +-rw-r--r-- 1 root root 2317705216 Oct 14 10:10 vzdump-lxc-9201-2026_10_14-10_08_05.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:08 vzdump-lxc-9201-2026_10_14-10_08_05.tmp +1219394 /bin/bash -c set -o pipefail && lxc-usernsexec -m u:0:100000:65536 -m g:0:100000:6 +1219395 lxc-usernsexec -m u:0:100000:65536 -m g:0:100000:65536 -- tar cpf - --totals --one +1219396 zstd --threads=1 +1219399 tar cpf - --totals --one-file-system -p --sparse --numeric-owner --acls --xattrs - +1231927 bash -c echo "== $(date -u +%T)"; ls -la /var/lib/vz/dump | grep -E "tar.dat|\.tmp +lock: snapshot-delete +active vzdump: [] +listed: [('local:backup/vzdump-lxc-9201-2026_10_14-09_28_50.tar.zst', 5648463064)] +== writers gone 08:14:24 +-rw-r--r-- 1 root root 5647749761 Oct 14 10:14 vzdump-lxc-9201-2026_10_14-10_08_05.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:08 vzdump-lxc-9201-2026_10_14-10_08_05.tmp +lock: snapshot-delete +status: running +`-> vzdump 2026-10-14 10:08:05 vzdump backup snapshot + `-> current You are here! +Oct 14 10:09:46 demo-hp felhom-agent[1306]: time=2026-10-14T10:09:46.540+02:00 level=ERROR msg="local-api: backup job failed" vmid=9201 target=local job=backup-9201-1791965284583723080 err="backup: vzdump task vmid 9201: +Oct 14 10:10:18 demo-hp felhom-agent[1306]: time=2026-10-14T10:10:18.307+02:00 level=WARN msg="stale-lock: clearing a stale backup lock left by an interrupted backup" vmid=9201 lock=backup onboot=true status=running +Oct 14 10:10:19 demo-hp pct[1230684]: executed 'pct unlock 9201' +Oct 14 10:10:21 demo-hp felhom-agent[1306]: time=2026-10-14T10:10:21.462+02:00 level=ERROR msg="stale-lock: delete dangling vzdump snapshot failed" vmid=9201 err="proxmox: task UPID:demo-hp:0012C7AE:001B7E81:6ACF38EB:vzd +08:20:28 +Oct 14 10:20:18 demo-hp felhom-agent[1306]: time=2026-10-14T10:20:18.828+02:00 level=WARN msg="stale-lock: clearing a stale backup lock left by an interrupted backup" vmid=9201 lock=snapshot-delete onboot=true status=running +Oct 14 10:20:19 demo-hp pct[1259735]: executed 'pct unlock 9201' +Oct 14 10:20:21 demo-hp felhom-agent[1306]: time=2026-10-14T10:20:21.894+02:00 level=INFO msg="stale-lock: removed dangling vzdump snapshot" vmid=9201 +no-lock +`-> current You are here! +08:42:26 +Oct 14 10:42:25 demo-hp felhom-agent[1306]: time=2026-10-14T10:42:25.585+02:00 level=INFO msg="backup: space preflight passed" vmid=9201 target=local last_archive_bytes=5648463064 need_bytes=8134320654 avail_bytes=14542950400 +/dev/mapper/pve-root 39G 24G 14G 63% / diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C2/before.txt b/documentation/audits/power-cut-fixes-2026-10-14/C2/before.txt new file mode 100644 index 00000000..59185567 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C2/before.txt @@ -0,0 +1 @@ +/dev/mapper/pve-root 39G 24G 14G 63% / diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C2/hand-cleanup.txt b/documentation/audits/power-cut-fixes-2026-10-14/C2/hand-cleanup.txt new file mode 100644 index 00000000..82f6cf27 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C2/hand-cleanup.txt @@ -0,0 +1,3 @@ +-rw-r--r-- 1 root root 5647749761 2026-10-14T10:14:23 vzdump-lxc-9201-2026_10_14-10_08_05.tar.dat +drwxr-xr-x 3 root root 4096 2026-10-14T10:08:06 vzdump-lxc-9201-2026_10_14-10_08_05.tmp +/dev/mapper/pve-root 39G 24G 14G 63% / diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C2/kill.txt b/documentation/audits/power-cut-fixes-2026-10-14/C2/kill.txt new file mode 100644 index 00000000..cbb06995 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C2/kill.txt @@ -0,0 +1,6 @@ +08:09:44.581 +-rw-r--r-- 1 root root 1538551808 Oct 14 10:09 vzdump-lxc-9201-2026_10_14-10_08_05.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:08 vzdump-lxc-9201-2026_10_14-10_08_05.tmp +worker pid 1219295: 1219295 task UPID:demo-hp:00129ADF:001B4A0D:6ACF3865:vzdump:9201:felhom-agent@pve!agent: +lock: backup +killed -9 at 08:09:44.683 diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C2/snapshot-delete-err.txt b/documentation/audits/power-cut-fixes-2026-10-14/C2/snapshot-delete-err.txt new file mode 100644 index 00000000..672e5358 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C2/snapshot-delete-err.txt @@ -0,0 +1 @@ +"proxmox: task UPID:demo-hp:0012C7AE:001B7E81:6ACF38EB:vzdelsnapshot:9201:felhom-agent@pve!agent: failed: exitstatus \"lvremove snapshot 'pve/snap_vm-9201-disk-0_vzdump' error: Logical volume pve/snap_vm-9201-disk-0_vzdump contains a filesystem in use.\"" diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C3/before-cut.txt b/documentation/audits/power-cut-fixes-2026-10-14/C3/before-cut.txt new file mode 100644 index 00000000..4a94de2f --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C3/before-cut.txt @@ -0,0 +1,5 @@ +08:44:07 +-rw-r--r-- 1 root root 1532915712 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +lock: backup +btime 1791947400 diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C3/monitor.txt b/documentation/audits/power-cut-fixes-2026-10-14/C3/monitor.txt new file mode 100644 index 00000000..dd329855 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C3/monitor.txt @@ -0,0 +1,191 @@ +ssh up 08:44:57 +== 08:44:57 btime=1791967481 +active vzdump: [('E:001E6EEC:6ACF4071:vzdump:9201:felhom-agent@pve!agent:', 1791967345, 'RUNNING')] +/etc/pve/lxc/9201.conf:lock: backup +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: stopped +== 08:45:30 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:47:04 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:47:37 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:48:10 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +unable to get PID for CT 9201 (not running?) +status: running +== 08:48:43 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:49:16 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:49:49 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:50:22 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:50:55 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:51:28 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:52:01 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:52:33 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:53:06 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:53:39 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:54:12 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:54:45 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:55:18 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:55:50 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:56:23 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:56:56 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:57:29 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:58:02 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:58:35 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:59:07 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 08:59:40 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 09:00:13 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 09:00:46 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 09:01:19 btime=1791967481 +active vzdump: [] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +status: running +== 09:01:51 btime=1791967481 +active vzdump: [('emo-hp:00010419:00018C62:6ACF44F0:vzdump:9202:root@pam:', 1791968496, 'RUNNING')] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +drwxr-xr-x 20 100000 100000 4096 Oct 14 10:46 vzdump-lxc-9202-2026_10_14-11_01_36.tmp +status: running +== 09:02:24 btime=1791967481 +active vzdump: [('emo-hp:00010419:00018C62:6ACF44F0:vzdump:9202:root@pam:', 1791968496, 'RUNNING')] +9201 no lock +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +drwxr-xr-x 20 100000 100000 4096 Oct 14 10:46 vzdump-lxc-9202-2026_10_14-11_01_36.tmp +status: running +done 09:02:56 diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C3/plugcut.log b/documentation/audits/power-cut-fixes-2026-10-14/C3/plugcut.log new file mode 100644 index 00000000..99cb05c5 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C3/plugcut.log @@ -0,0 +1,5 @@ +2026-10-14T08:44:07.931Z hp watchdog armed (pid 1918617, relay on after 25 s at the latest) +2026-10-14T08:44:08.089Z hp CUT: relay off + countdown 10 s sent in one command +2026-10-14T08:44:08.131Z hp read back: relay=off countdown=10 +2026-10-14T08:44:18.165Z hp BACK: relay on by the plug countdown, +10.0 s after the cut +2026-10-14T08:44:33.153Z hp watchdog: relay already on — nothing to do diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C3/requests.txt b/documentation/audits/power-cut-fixes-2026-10-14/C3/requests.txt new file mode 100644 index 00000000..ad95bd73 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C3/requests.txt @@ -0,0 +1,11 @@ +08:47:09 reboot request: ('202', {'data': {'rebooting': True}, 'ok': True}) +Oct 14 10:47:00 demo-hp felhom-agent[1322]: time=2026-10-14T10:47:00.962+02:00 level=INFO msg="reconcile: enrolled drive bound under shared parent (live, no reboot)" vmid=9201 where=/mnt/hdd_1 guest_path=/mnt/felhom-drives/hdd_1 d +Oct 14 10:47:10 demo-hp felhom-agent[1322]: time=2026-10-14T10:47:10.205+02:00 level=WARN msg="local-api: guest reboot requested (activating pending drive binds)" vmid=9201 +Oct 14 10:47:10 demo-hp felhom-agent[1322]: time=2026-10-14T10:47:10.205+02:00 level=WARN msg="guest-reboot: rebooting guest to activate pending mountpoint binds" vmid=9201 +Oct 14 10:47:10 demo-hp sudo[11652]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/sbin/pct reboot 9201 +Oct 14 10:47:11 demo-hp pct[11881]: requesting reboot of CT 9201: UPID:demo-hp:00002E69:00003A99:6ACF418F:vzreboot:9201:root@pam: +Oct 14 10:47:11 demo-hp pct[11654]: starting task UPID:demo-hp:00002E69:00003A99:6ACF418F:vzreboot:9201:root@pam: +2026-10-14T10:44:59+02:00 demo-hp felhom-agent[1322]: time=2026-10-14T10:44:59.654+02:00 level=WARN msg="stale-lock: a vzdump backup is genuinely in-flight — leaving the lock (NOT stale)" vmid=9201 lock=backup +Oct 14 10:45:00 demo-hp pvesh[1525]: removed left over backup lock from '9201'! +Oct 14 10:45:00 demo-hp pve-guests[1768]: removed left over backup lock from '9201'! +08:48:41 power-off request: ('404', {'ok': False, 'error': 'endpoint not found'}) diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C4/after.txt b/documentation/audits/power-cut-fixes-2026-10-14/C4/after.txt new file mode 100644 index 00000000..8a5f50f9 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C4/after.txt @@ -0,0 +1,7 @@ +/dev/mapper/pve-root 39G 25G 13G 66% / + VG #PV #LV #SN Attr VSize VFree + pve 1 6 0 wz--n- <118.00g <14.63g +09:18:12 +`-> current You are here! +Oct 14 11:14:59 demo-hp felhom-agent[1322]: time=2026-10-14T11:14:59.796+02:00 level=WARN msg="stale-lock: clearing a stale backup lock left by an interrupted backup" vmid=9201 lock=snapshot-delete on +Oct 14 11:15:02 demo-hp felhom-agent[1322]: time=2026-10-14T11:15:02.881+02:00 level=ERROR msg="stale-lock: delete dangling vzdump snapshot failed" vmid=9201 err="proxmox: task UPID:demo-hp:00019D45:0 diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C4/fill.txt b/documentation/audits/power-cut-fixes-2026-10-14/C4/fill.txt new file mode 100644 index 00000000..ff1da10e --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C4/fill.txt @@ -0,0 +1,3 @@ +/dev/mapper/pve-root 41424257024 25841410048 13445640192 66% / +fill=12816494592 +/dev/mapper/pve-root 39G 37G 600M 99% / diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C4/hand-fix.txt b/documentation/audits/power-cut-fixes-2026-10-14/C4/hand-fix.txt new file mode 100644 index 00000000..5c83493b --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C4/hand-fix.txt @@ -0,0 +1,11 @@ +CT is locked (snapshot-delete) + snap_vm-9201-disk-1_vzdump pve Vri-a-tz-k 70.00g data vm-9201-disk-1 26.79 +`-> vzdump 2026-10-14 11:02:50 vzdump backup snapshot + `-> current You are here! +lock: snapshot-delete +status: running +lvremove snapshot 'pve/snap_vm-9201-disk-0_vzdump' error: Failed to find logical volume "pve/snap_vm-9201-disk-0_vzdump" + Logical volume "snap_vm-9201-disk-1_vzdump" successfully removed. +no snapshot LV +`-> current You are here! +no-lock diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C4/vzdump-9201.txt b/documentation/audits/power-cut-fixes-2026-10-14/C4/vzdump-9201.txt new file mode 100644 index 00000000..37348ce5 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C4/vzdump-9201.txt @@ -0,0 +1,25 @@ +09:02:49 +INFO: guest is online again after 2 seconds +INFO: creating vzdump archive '/var/lib/vz/dump/vzdump-lxc-9201-2026_10_14-11_02_50.tar.zst' +INFO: zstd: error 70 : Write error : cannot write block : No space left on device +INFO: cleanup temporary 'vzdump' snapshot + /etc/lvm/archive/.lvm_demo-hp_81330_104945305: write error failed: No space left on device + Failed to update thin pool pve/data. + Failed to write metadata to /etc/lvm/backup/.lvm_demo-hp_81330_594373106. + /etc/lvm/backup/.lvm_demo-hp_81330_594373106: fclose failed: No space left on device +snapshot 'vzdump' was not (fully) removed - lvremove snapshot 'pve/snap_vm-9201-disk-0_vzdump' error: Backup of volume group pve metadata failed. +ERROR: Backup of VM 9201 failed - command 'set -o pipefail && lxc-usernsexec -m u:0:100000:65536 -m g:0:100000:65536 -- tar cpf - --totals --one-file-system -p --sparse --numeric-owner --acls --xattrs '--xattrs-include=user.*' '--xattrs-include=security.capability' '--warning=no-file-ignored' '--warning=no-xattr-write' --one-file-system '--warning=no-file-ignored' '--directory=/var/lib/vz/dump/vzdump-lxc-9201-2026_10_14-11_02_50.tmp' ./etc/vzdump/pct.conf ./etc/vzdump/pct.fw '--directory=/mnt/vzsnap0' --no-anchored '--exclude=lost+found' --anchored '--exclude=./tmp/?*' '--exclude=./var/tmp/?*' '--exclude=./var/run/?*.pid' ./ ./var/lib/felhom | zstd '--threads=1' >/var/lib/vz/dump/vzdump-lxc-9201-2026_10_14-11_02_50.tar.dat' failed: exit code 70 +INFO: Failed at 2026-10-14 11:06:02 +INFO: Backup job finished with errors +INFO: notified via target `mail-to-root` +job errors +09:06:02 +-rw-r--r-- 1 root root 12816494592 Oct 14 11:01 C4-fill-test.bin +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +-rw-r--r-- 1 root root 1942 Oct 14 11:06 vzdump-lxc-9201-2026_10_14-11_02_50.log +-rw-r--r-- 1 root root 2234 Oct 14 11:02 vzdump-lxc-9202-2026_10_14-11_01_36.log +lock: snapshot-delete +`-> vzdump 2026-10-14 11:02:50 vzdump backup snapshot + `-> current You are here! +/dev/mapper/pve-root 39G 37G 600M 99% / diff --git a/documentation/audits/power-cut-fixes-2026-10-14/C4/vzdump.txt b/documentation/audits/power-cut-fixes-2026-10-14/C4/vzdump.txt new file mode 100644 index 00000000..981c112e --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/C4/vzdump.txt @@ -0,0 +1,24 @@ +09:01:35 +INFO: ionice priority: 7 +INFO: CT Name: demo-hp-scratch +INFO: including mount point rootfs ('/') in backup +INFO: including mount point mp0 ('/var/lib/felhom') in backup +INFO: excluding bind mount point mp8 ('/mnt/felhom-drives/scratch_hdd') from backup (not a volume) +INFO: excluding bind mount point mp9 ('/etc/felhom-bootstrap') from backup (not a volume) +INFO: starting first sync /proc/2400/root/ to /var/lib/vz/dump/vzdump-lxc-9202-2026_10_14-11_01_36.tmp +ERROR: rsync: [receiver] write failed on "/var/lib/vz/dump/vzdump-lxc-9202-2026_10_14-11_01_36.tmp/var/lib/felhom/docker/overlay2/522329c6fbd577ca05b42eb168d3ea4c18a7aba753d2f1e8f72517e3004ab77e/diff/usr/share/perl5/core_perl/Tie/Handle.pm": No space left on device (28) +ERROR: rsync error: error in file IO (code 11) at receiver.c(657) [receiver=3.5.0] +ERROR: rsync: [sender] write error: Broken pipe (32) +ERROR: Backup of VM 9202 failed - command 'rsync --stats -h -X -A --numeric-ids -aH --delete --no-whole-file --sparse --one-file-system --relative '--exclude=/tmp/?*' '--exclude=/var/tmp/?*' '--exclude=/var/run/?*.pid' '--exclude=/mnt/felhom-drives/scratch_hdd' '--exclude=/etc/felhom-bootstrap' /proc/2400/root//./ /proc/2400/root//./var/lib/felhom /var/lib/vz/dump/vzdump-lxc-9202-2026_10_14-11_01_36.tmp' failed: exit code 11 +INFO: Failed at 2026-10-14 11:02:36 +INFO: Backup job finished with errors +INFO: notified via target `mail-to-root` +job errors +rc=0 +09:02:37 +-rw-r--r-- 1 root root 1161887744 Oct 14 10:44 vzdump-lxc-9201-2026_10_14-10_42_25.tar.dat +drwxr-xr-x 3 root root 4096 Oct 14 10:42 vzdump-lxc-9201-2026_10_14-10_42_25.tmp +-rw-r--r-- 1 root root 2234 Oct 14 11:02 vzdump-lxc-9202-2026_10_14-11_01_36.log +9202 no lock +`-> current You are here! +/dev/mapper/pve-root 39G 37G 600M 99% / diff --git a/documentation/audits/power-cut-fixes-2026-10-14/F/immich-delete-then-missing.txt b/documentation/audits/power-cut-fixes-2026-10-14/F/immich-delete-then-missing.txt new file mode 100644 index 00000000..9fa399d9 --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/F/immich-delete-then-missing.txt @@ -0,0 +1,9 @@ +== truncated 3 assets' files: 6 zero-byte files +== deleted the zero-byte files: 0 left + ac03a80f-2538-49fd-ac9c-3d55934cbaa7 thumb=23 preview=23 + 3211a478-95a0-4d0f-a111-a9ac8e59d85c thumb=23 preview=23 + ecc02159-c38e-4f7e-8442-35bd653747fe thumb=23 preview=23 +PUT missing -> 200 + ac03a80f-2538-49fd-ac9c-3d55934cbaa7 thumb=23 preview=23 + 3211a478-95a0-4d0f-a111-a9ac8e59d85c thumb=23 preview=23 + ecc02159-c38e-4f7e-8442-35bd653747fe thumb=23 preview=23 diff --git a/documentation/audits/power-cut-fixes-2026-10-14/F/immich-repair.txt b/documentation/audits/power-cut-fixes-2026-10-14/F/immich-repair.txt new file mode 100644 index 00000000..0a8887bd --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/F/immich-repair.txt @@ -0,0 +1,88 @@ +upload mount: /usr/src/app/upload <- /mnt/sizing-bench/appdata/immich +== 09:44:41 upload 30 (skipped when already there) +already 30 +== 09:44:57 jobs idle +assets: 30 +== 09:44:57 before the damage + 2146e1a7-d55e-4990-ad2b-3fe609b14679 thumb=6398 preview=203683 original=200:4876806 + 26fd37fe-0595-4e54-a502-fa6ce776b30b thumb=5858 preview=194665 original=200:4861040 + 498d49e7-eacc-49f5-a467-e296e58aa6ab thumb=5964 preview=198427 original=200:4871011 + 9ef70762-2370-44ee-9fd8-9b4c7d783b41 thumb=6698 preview=208760 original=200:4880682 + a6d65165-e4e5-473d-8535-3523e299edbe thumb=5882 preview=194607 original=200:4862271 + 07598631-d50d-423a-9882-1f3a57523652 thumb=5856 preview=193665 original=200:4862757 + 4be9e614-7601-478e-86c8-b43adf9fd1d4 thumb=5630 preview=197443 original=200:4861457 + 6a47e17e-ba9f-408c-bfd6-44b0d2b2bbe6 thumb=6022 preview=204622 original=200:4871732 + 6ef0d4bf-f6bd-4c6a-a1e9-d796d8484187 thumb=5720 preview=199228 original=200:4863146 + a762ba16-dc62-478e-81de-784c9117ffdc thumb=6246 preview=201464 original=200:4872738 + 028083d0-62c8-401f-bdec-56afa40bdc05 thumb=5574 preview=197250 original=200:4861561 +== 09:44:57 damage: 10 assets' thumbnail+preview -> 0 bytes; asset 11's original deleted + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/21/46/2146e1a7-d55e-4990-ad2b-3fe609b14679_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/21/46/2146e1a7-d55e-4990-ad2b-3fe609b14679_preview.jpeg + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/26/fd/26fd37fe-0595-4e54-a502-fa6ce776b30b_preview.jpeg + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/26/fd/26fd37fe-0595-4e54-a502-fa6ce776b30b_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/49/8d/498d49e7-eacc-49f5-a467-e296e58aa6ab_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/49/8d/498d49e7-eacc-49f5-a467-e296e58aa6ab_preview.jpeg + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/9e/f7/9ef70762-2370-44ee-9fd8-9b4c7d783b41_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/9e/f7/9ef70762-2370-44ee-9fd8-9b4c7d783b41_preview.jpeg + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/a6/d6/a6d65165-e4e5-473d-8535-3523e299edbe_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/a6/d6/a6d65165-e4e5-473d-8535-3523e299edbe_preview.jpeg + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/07/59/07598631-d50d-423a-9882-1f3a57523652_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/07/59/07598631-d50d-423a-9882-1f3a57523652_preview.jpeg + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/4b/e9/4be9e614-7601-478e-86c8-b43adf9fd1d4_preview.jpeg + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/4b/e9/4be9e614-7601-478e-86c8-b43adf9fd1d4_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/6a/47/6a47e17e-ba9f-408c-bfd6-44b0d2b2bbe6_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/6a/47/6a47e17e-ba9f-408c-bfd6-44b0d2b2bbe6_preview.jpeg + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/6e/f0/6ef0d4bf-f6bd-4c6a-a1e9-d796d8484187_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/6e/f0/6ef0d4bf-f6bd-4c6a-a1e9-d796d8484187_preview.jpeg + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/a7/62/a762ba16-dc62-478e-81de-784c9117ffdc_thumbnail.webp + truncated thumbs/e21d3f54-60fc-41b3-b274-f3a1e18b5687/a7/62/a762ba16-dc62-478e-81de-784c9117ffdc_preview.jpeg + deleted original of 028083d0-62c8-401f-bdec-56afa40bdc05 (upload/e21d3f54-60fc-41b3-b274-f3a1e18b5687/b1/aa/b1aaa681-0914-4fcb-afca-d8e707dccd05.jpg) +== 09:44:57 after the damage + 2146e1a7-d55e-4990-ad2b-3fe609b14679 thumb=0 preview=0 original=200:4876806 + 26fd37fe-0595-4e54-a502-fa6ce776b30b thumb=0 preview=0 original=200:4861040 + 498d49e7-eacc-49f5-a467-e296e58aa6ab thumb=0 preview=0 original=200:4871011 + 9ef70762-2370-44ee-9fd8-9b4c7d783b41 thumb=0 preview=0 original=200:4880682 + a6d65165-e4e5-473d-8535-3523e299edbe thumb=0 preview=0 original=200:4862271 + 07598631-d50d-423a-9882-1f3a57523652 thumb=0 preview=0 original=200:4862757 + 4be9e614-7601-478e-86c8-b43adf9fd1d4 thumb=0 preview=0 original=200:4861457 + 6a47e17e-ba9f-408c-bfd6-44b0d2b2bbe6 thumb=0 preview=0 original=200:4871732 + 6ef0d4bf-f6bd-4c6a-a1e9-d796d8484187 thumb=0 preview=0 original=200:4863146 + a762ba16-dc62-478e-81de-784c9117ffdc thumb=0 preview=0 original=200:4872738 + 028083d0-62c8-401f-bdec-56afa40bdc05 thumb=5574 preview=197250 original=404:23 +POST /api/jobs integrity-missing-files -> 204 +POST /api/jobs integrity-untracked-files -> 204 +POST /api/jobs integrity-checksum-mismatch -> 204 +== 09:45:44 integrity summary: {"untracked_file":0,"missing_file":1,"checksum_mismatch":0} + report missing_file: {"items":[{"id":"01a139cd-0054-7ebf-89a6-7fb2a7076741","type":"missing_file","path":"/usr/src/app/upload/upload/e21d3f54-60fc-41b3-b274-f3a1e18b5687/b1/aa/b1aaa681-0914-4fcb-afca-d8e707dccd05.jpg","assetId":"028083d0-62c8-401f-bdec-56afa40bdc05","fileAssetId":null,"createdAt":"2026-10-14T09:44:58.707Z"}]} [200] + report untracked_file: {"items":[]} [200] + report checksum_mismatch: {"items":[]} [200] + report missing-files: {"message":"Validation failed","errors":[{"code":"invalid_value","values":["untracked_file","missing_file","checksum_mismatch"],"path":["type"],"message":"Invalid option: expected one of \"untracked_file\"|\"missing_file\"|\"checksum_mismatch\""}]} [400] + report untracked-files: {"message":"Validation failed","errors":[{"code":"invalid_value","values":["untracked_file","missing_file","checksum_mismatch"],"path":["type"],"message":"Invalid option: expected one of \"untracked_file\"|\"missing_file\"|\"checksum_mismatch\""}]} [400] + report checksum-mismatch: {"message":"Validation failed","errors":[{"code":"invalid_value","values":["untracked_file","missing_file","checksum_mismatch"],"path":["type"],"message":"Invalid option: expected one of \"untracked_file\"|\"missing_file\"|\"checksum_mismatch\""}]} [400] +== 09:45:44 thumbnails: missing (force=false) +PUT -> 200 + 2146e1a7-d55e-4990-ad2b-3fe609b14679 thumb=0 preview=0 original=200:4876806 + 26fd37fe-0595-4e54-a502-fa6ce776b30b thumb=0 preview=0 original=200:4861040 + 498d49e7-eacc-49f5-a467-e296e58aa6ab thumb=0 preview=0 original=200:4871011 + 9ef70762-2370-44ee-9fd8-9b4c7d783b41 thumb=0 preview=0 original=200:4880682 + a6d65165-e4e5-473d-8535-3523e299edbe thumb=0 preview=0 original=200:4862271 + 07598631-d50d-423a-9882-1f3a57523652 thumb=0 preview=0 original=200:4862757 + 4be9e614-7601-478e-86c8-b43adf9fd1d4 thumb=0 preview=0 original=200:4861457 + 6a47e17e-ba9f-408c-bfd6-44b0d2b2bbe6 thumb=0 preview=0 original=200:4871732 + 6ef0d4bf-f6bd-4c6a-a1e9-d796d8484187 thumb=0 preview=0 original=200:4863146 + a762ba16-dc62-478e-81de-784c9117ffdc thumb=0 preview=0 original=200:4872738 + 028083d0-62c8-401f-bdec-56afa40bdc05 thumb=5574 preview=197250 original=404:23 +== 09:46:00 thumbnails: all (force=true) +PUT -> 200 + 2146e1a7-d55e-4990-ad2b-3fe609b14679 thumb=6398 preview=203683 original=200:4876806 + 26fd37fe-0595-4e54-a502-fa6ce776b30b thumb=5858 preview=194665 original=200:4861040 + 498d49e7-eacc-49f5-a467-e296e58aa6ab thumb=5964 preview=198427 original=200:4871011 + 9ef70762-2370-44ee-9fd8-9b4c7d783b41 thumb=6698 preview=208760 original=200:4880682 + a6d65165-e4e5-473d-8535-3523e299edbe thumb=5882 preview=194607 original=200:4862271 + 07598631-d50d-423a-9882-1f3a57523652 thumb=5856 preview=193665 original=200:4862757 + 4be9e614-7601-478e-86c8-b43adf9fd1d4 thumb=5630 preview=197443 original=200:4861457 + 6a47e17e-ba9f-408c-bfd6-44b0d2b2bbe6 thumb=6022 preview=204622 original=200:4871732 + 6ef0d4bf-f6bd-4c6a-a1e9-d796d8484187 thumb=5720 preview=199228 original=200:4863146 + a762ba16-dc62-478e-81de-784c9117ffdc thumb=6246 preview=201464 original=200:4872738 + 028083d0-62c8-401f-bdec-56afa40bdc05 thumb=5574 preview=197250 original=404:23 +== 09:46:16 integrity summary after: {"untracked_file":0,"missing_file":1,"checksum_mismatch":0} diff --git a/documentation/audits/power-cut-fixes-2026-10-14/README.md b/documentation/audits/power-cut-fixes-2026-10-14/README.md new file mode 100644 index 00000000..2b17130a --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/README.md @@ -0,0 +1,117 @@ +# Power-cut fixes 2026-10-14 — spike (Part C) and live proof (Part E) + +Times UTC. Box: demo-hp (Tier 0). No key, token or password in this folder. + +## C1 — what the night's cut left (from `audits/night-power-2026-10-13/R3/`) + +| Item | Value | +|---|---| +| left on disk | `/var/lib/vz/dump/vzdump-lxc-9201-2026_10_14-04_42_18.tar.dat` (file, `root:root 0644`, 12.4 GB at the cut, 12.2 GB after the boot — the unwritten tail was lost) and `vzdump-lxc-9201-2026_10_14-04_42_18.tmp/` (directory, `root:root 0755`). **No `.log`** (vzdump writes it at the end) | +| name pattern | `vzdump-lxc--YYYY_MM_DD-HH_MM_SS` + `.tar.dat` / `.tmp`; the timestamp is the host's local time | +| the task | `UPID:demo-hp:000A8C2E:000DFB52:6ACEEC0A:vzdump:9201:felhom-agent@pve!agent:` — started by the agent's API token | +| after the boot | 02:52:56 the agent: „a vzdump backup is genuinely in-flight — leaving the lock" — so the dead task WAS in `tasks?source=active` 1 min after the boot. How long it stays: not recorded → C3 | +| who may delete | the files are root's in a root-owned directory; the agent runs as `felhom-agent` → it **cannot** delete them itself | + +## C2 — predictions (written before the run) + +| Question | Prediction | +|---|---| +| left on disk after `kill -9` of the vzdump worker | the same pair (`.tar.dat` + `.tmp`); the tar/zstd children may keep writing for a moment, then die on the broken pipe | +| `StorageContent` (backup) lists it | **no** — Proxmox lists `vzdump-*.tar.zst`/`.vma.zst` etc.; `.tar.dat` is a temporary name | +| the active-task list keeps the task | **no** — the PVE task daemon sees the worker die and ends the task with an error (unlike a power cut, where nobody ends it) | +| `lock: backup` on the guest | stays (vzdump's cleanup did not run) | +| the next backup | the space preflight sizes from the last COMPLETE archive (5.6 GB) and passes while space is free; the partial still occupies the disk | + +## C2 — a partial without a power cut (`C2/`; 9201, because 9202 has no agent link — its „back up now" cannot run) + +Backup pressed through the product 08:07:30 (`POST /api/guest-backup/trigger`, 200); at 1.5 GB written the vzdump worker +(`task UPID:…:vzdump:9201:…`) was killed `-9` at 08:09:44. + +| Question | Prediction | Measured | +|---|---|---| +| left on disk | `.tar.dat` + `.tmp` | **yes** — `vzdump-lxc-9201-2026_10_14-10_08_05.tar.dat` + `.tmp/`, root 0644 / 0755, no `.log` | +| the writers | die on a broken pipe | **no** — `tar` and `zstd` (children of a `bash -c`, none named „vzdump") kept writing for **4 min 40 s**, to 5.6 GB (a whole archive's size), then exited. The file was never renamed. **A „no vzdump process" check alone would call a file dead while it is being written** | +| `StorageContent` lists it | no | **no** — only the complete `…09_28_50.tar.zst` is listed | +| active-task list keeps the task | no | **no** — gone within seconds (the PVE daemon ended it: „backup job failed" 08:09:46) | +| lock | `backup` stays | the agent's 10-min janitor cleared it at 08:10:18 **while the orphan `tar` still read the snapshot**; its snapshot delete failed („contains a filesystem in use") → `lock: snapshot-delete`; the next janitor pass (08:20:18) cleared that lock and removed the snapshot | +| the next backup | passes the preflight | none came: a failed manual press is not retried (the night loop retries; this one did not within 30 min) | +| clean-up | — | the 5.6 GB partial removed by hand at 08:41 (host root 78 % → 63 %) | + +## C3 — one real power cut mid-write (9201's backup, pressed through the product) + +**Predictions (written before the cut):** + +| Question | Prediction | +|---|---| +| the dead vzdump task in `tasks?source=active` after the boot | **yes**, and it stays there (nobody ends a task whose node died) — for the whole 15 min | +| `BackupRunning` (the agent's startup stale-lock line) | true → „genuinely in-flight — leaving the lock"; Proxmox's start-all removes the lock anyway | +| the guest restart request (`POST /api/server/reboot`, the dashboard's) | refused „backup running" while the dead task is listed | +| the household power-off request | refused — „too soon" first (host up < 10 min), and on demo-hp „other guests" anyway (9202 and Tester 1 run); it cannot reach the backup check here | +| files | `.tar.dat` + `.tmp` stay, nothing removes them | + + +**Measured** (`C3/`): backup pressed 08:42:25; at 1.5 GB the cut: 08:44:08.089 off → 08:44:18.165 on (plug countdown). +Boot `btime` 1791967481; the dead task's start 1791967345 (136 s before the boot). + +| Question | Prediction | Measured | +|---|---|---| +| dead task in the active list | stays all 15 min | **listed RUNNING at 08:44:57; gone at 08:45:30** — Proxmox's own start-up clears it within ~30–60 s of the boot (at 10:45:00 local `pvesh`/`pve-guests`: „removed left over backup lock from '9201'!") | +| `BackupRunning` | true | **true at the agent's start** (08:44:59: „genuinely in-flight — leaving the lock") — the agent starts inside that short window | +| guest restart request (`POST /api/server/reboot`, 08:47:09) | refused „backup running" | **202 — it restarted 9201.** The restart route never asks `BackupRunning`; the `guestpower.go` caller is the guest-power WATCHDOG (it starts stopped guests), not the restart | +| power-off request (08:48:41) | refused | **404** — demo-hp's 9201 runs controller 0.309.0, which has no power-off route; the guard can be exercised only after the floor raise (Part E) | +| files | stay | **stay all 15 min**: `…10_42_25.tar.dat` 1.16 GB (1.5 GB at the cut — the unwritten tail lost) + `.tmp/` | +| lock | Proxmox removes it | removed by Proxmox at boot | + +**What this changes (STOP-rule check):** the night's mechanism is the same — the partial is the damage, Proxmox does not +list it as a backup. R-1008's window is **short** (≈ 1 min after a boot): the startup stale-lock pass, the first guest-power +watchdog sweep and the controller supervisor in that minute see a dead task as running. D1 stays (one helper, cheap, +correct); its live effect is small. D2 is the fix that matters. + +## C4 — a backup that runs out of space (prediction before the run) + +On scratch guest 9202, `vzdump 9202 --storage local --mode snapshot --compress zstd` run as root on the host (9202 has no +agent link; the question is vzdump's own behaviour), with the host root filled by a test file to ~600 MB free. +**Prediction:** vzdump fails with „No space left on device" and **removes its own `.tar.dat`/`.tmp`** (its error path runs, +unlike a power cut); the lock clears; nothing is left. + +**Measured** (`C4/`, host root filled to 600 MB free by `C4-fill-test.bin`, removed after): + +| Run | Result | +|---|---| +| `vzdump 9202` (dir storage → vzdump's rsync path, `suspend` mode) | „No space left on device" in the first sync; **vzdump removed its `.tmp`**; only `vzdump-lxc-9202-…11_01_36.log` (2 KB) left; space back | +| `vzdump 9201` (LVM-thin → the snapshot + `tar … \| zstd > .tar.dat` path, the night's path) | `zstd: error 70 … No space left on device`; **vzdump removed its `.tar.dat`**; only the `.log` left. **But the full disk broke LVM:** „Failed to write metadata … Backup of volume group pve metadata failed"; the `vzdump` snapshot was half removed (disk-0's gone, disk-1's left) → `lock: snapshot-delete` | +| the agent's janitor after it | 11:14:59 local: cleared the lock, then its snapshot delete **failed** („Failed to find logical volume pve/snap_vm-9201-disk-0_vzdump") — it would fail the same way every 10 min, the guest's snapshot list stuck. Fixed by hand 09:19 UTC: `pct unlock 9201; pct delsnapshot 9201 vzdump --force` (removed disk-1's snapshot LV, cleared the entry) | + +**D3 decided by C4:** vzdump cleans its own partial on ENOSPC, so the preflight's sizing (R-1006 part 2) costs only wasted +time, never a leftover; `spaceFits` stays as it is. The real danger C4 shows is different: **a host root that fills up +breaks LVM's own metadata writes** — the night's 12 GB partial took demo-hp to 92 %. Removing a dead partial (D2) is what +keeps the root disk from filling. New finding: the janitor's snapshot delete has no `--force` path for a half-removed +snapshot (row). + + +## F — can Immich repair the power cut's damage by itself? (R-1007, spike only) + +**Prediction (before the run):** Immich's thumbnail job with „missing" does NOT redo a zero-byte thumbnail (the database +says it exists); the „all" (force) run does; nothing in Immich restores a deleted original, but a storage/integrity +report lists it. + +**Measured** (`F/immich-repair.txt`, `F/immich-delete-then-missing.txt`; Immich v3.2.4 on 9202, 30 photos, deployed and +removed through the product): + +| Act | Result | +|---|---| +| 10 assets' thumbnail + preview truncated to 0 bytes; 1 original deleted (the night's damage, R2) | the API serves 0-byte thumbnails/previews; the deleted original: 404 | +| integrity jobs `integrity-missing-files`, `-untracked-files`, `-checksum-mismatch` (`POST /api/jobs`) | **the deleted original is found**: `GET /api/admin/integrity/report?type=missing_file` lists it with its asset id. The zero-byte thumbnails are not reported (the checks cover originals) | +| thumbnails „missing" (`PUT /api/jobs/thumbnailGeneration {force:false}`) | **does not fix** a zero-byte thumbnail — the database says it exists | +| zero-byte files deleted first, then „missing" | **still not fixed** (Immich decides from its database, not the disk) | +| thumbnails „all" (`{force:true}`) | **fixes all 10** — every thumbnail regenerated (30 photos: 16 s; a family library of 50,000: hours of CPU) | +| the deleted original | nothing in Immich restores it — only a backup | + +**Prediction:** right on „missing" and „all"; right that nothing restores an original; the integrity report exists in this +version (`/api/admin/integrity/…`) and finds the missing original. + +**Proposal for R-1007 (an operator decision):** after an unclean boot, run Immich's „integrity-missing-files" job and show +its report (operator page, later the household), and run „thumbnails: all" in the night window. Cost: one catalog-level +hook and CPU in proportion to the library; a missing original then needs a restore from the night backup. If nothing is +done: broken previews stay until someone forces a regeneration by hand; a lost original stays unnoticed. + diff --git a/documentation/audits/power-cut-fixes-2026-10-14/redproof.txt b/documentation/audits/power-cut-fixes-2026-10-14/redproof.txt new file mode 100644 index 00000000..c666898b --- /dev/null +++ b/documentation/audits/power-cut-fixes-2026-10-14/redproof.txt @@ -0,0 +1,19 @@ +RED D1 worker gone is dead ['--- FAIL: TestVzdumpInFlight_WorkerGoneIsDead (0.00s)', '--- FAIL: TestStaleLockController_BackupRunningIgnoresDeadTask (0.00s)', '--- FAIL: TestCaller_StartupStaleLockClearsAfterPowerCut (0.00s)'] +RED D1 reused pid ['--- FAIL: TestVzdumpInFlight_ReusedPidIsDead (0.00s)'] +RED D1 zombie ['--- FAIL: TestVzdumpInFlight_ZombieIsDead (0.00s)'] +RED D1 doubt fails safe ['--- FAIL: TestVzdumpInFlight_DoubtFailsSafe (0.00s)'] +RED D1 hidden proc fails safe ['--- FAIL: TestVzdumpInFlight_HiddenProcFailsSafe (0.00s)'] +RED D1 the one helper ['--- FAIL: TestStaleLockController_BackupRunningIgnoresDeadTask (0.00s)', '--- FAIL: TestBackupRunning_AllCallersUseTheOneHelper (0.04s)'] +RED D2 in-flight check ['--- FAIL: TestClean_Keeps (0.00s)'] +RED D2 dir storage only ['--- FAIL: TestClean_Keeps (0.00s)'] +RED D2 exact name ['--- FAIL: TestClean_OnlyThisGuestsHalfWrittenNames (0.00s)'] +RED D2 dead before boot ['--- FAIL: TestClean_RemovesDeadPartialAndTmp (0.00s)'] +RED D2 symlink skip ['--- FAIL: TestClean_KeepsSymlink (0.00s)'] +RED D2 runs before the preflight ['--- FAIL: TestR1006_LeftoverCleanupRunsBeforeThePreflight (0.00s)'] +RED helper name regex ['--- FAIL: TestRootHelperPythonSuite (1.00s)'] +RED helper dir type (both guards) ['--- FAIL: TestRootHelperPythonSuite (1.01s)'] +RED helper vzdump process ['--- FAIL: TestRootHelperPythonSuite (0.99s)'] +RED helper open file ['--- FAIL: TestRootHelperPythonSuite (1.02s)'] +RED helper quiet rule ['--- FAIL: TestRootHelperPythonSuite (1.03s)'] +RED helper symlink ['--- FAIL: TestRootHelperPythonSuite (1.07s)'] +RED helper kind ['--- FAIL: TestRootHelperPythonSuite (1.10s)'] diff --git a/documentation/audits/sizing-check-2026-10-14/CHECK.md b/documentation/audits/sizing-check-2026-10-14/CHECK.md index 8555ba88..76e09014 100644 --- a/documentation/audits/sizing-check-2026-10-14/CHECK.md +++ b/documentation/audits/sizing-check-2026-10-14/CHECK.md @@ -172,3 +172,23 @@ the weaker one here). No OOM kill. (it works; 14 OOM kills in its ML container, swap full), the family set does not fit (84 of 200 photo uploads failed, transcode 15× slower, Paperless did nothing in 30 min). The headroom of 1.5 is not too strict: the family set's real simultaneous peak was 97 % of the page's sum. Register row R-1009. + +## 2026-10-14 — the operator's „raise only" ruling, built and STOPPED before the push + +The rule was built as a source (`raise-only-source.patch` for `sizing-2026-10-13/tools/build_sizing.py`; the HP figures +from the evidence by `tools/hp_figures.py` → `hp-figures.json`, 27 apps, `used` basis). It raises 23 figures and lowers none +(Immich typical 1,356 → 1,557 and peak 3,061 → 3,290; Jellyfin peak 373 → 448; Home Assistant 318 → 352 / 328 → 434; +Paperless 568 → 645 / 659 → 755; Grimmory typical 346 → 451; and 17 small ones). The presets keep their verdicts (family +5,515 → 6,021 MB: 8 GB no, 12 GB tight, 16 GB well; files + passwords 1,094 unchanged; Immich alone 3,531 → 3,760: 8 GB tight). + +**But one verdict of the `SIZING.md` §5 table changes:** „Household set without Immich" on 8 GB — **fits well → fits, but +tight**. Its peak becomes 2,731 MB; × 1.5 = 4,096.5 MB against the 8 GB guest's 4,096 MB: **0.5 MB over**. Not one figure +does it: without the Jellyfin raise, or the Home Assistant raise, or the Paperless raise, it stays „fits well". By the +task's STOP rule the website was **not** pushed; the data changes were reverted (nothing half-built is committed). + +| Option | What happens | Cost | +|---|---|---| +| A — publish the raise as ruled; accept the one changed verdict | „5 apps without Immich on 8 GB" reads „fits, but tight"; `SIZING.md` §5 and the test case updated | the page is 0.5 MB pessimistic there | +| B — raise only, but round the HP figures to the nearest 10 MB **down** | to be checked: it may still flip | one rule sentence | +| C — keep today's figures | nothing changes; the HP's higher figures stay unused | the page reads low for Immich, Jellyfin, Home Assistant, Paperless | + diff --git a/documentation/audits/sizing-check-2026-10-14/hp-figures.json b/documentation/audits/sizing-check-2026-10-14/hp-figures.json new file mode 100644 index 00000000..4797f210 --- /dev/null +++ b/documentation/audits/sizing-check-2026-10-14/hp-figures.json @@ -0,0 +1,195 @@ +{ + "date": "2026-10-14", + "basis": "used = memory.current - inactive_file", + "apps": { + "immich": { + "typical_mb": 1557, + "peak_mb": 3290, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "jellyfin": { + "typical_mb": 188, + "peak_mb": 448, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "nextcloud": { + "typical_mb": 229, + "peak_mb": 290, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "home-assistant": { + "typical_mb": 352, + "peak_mb": 434, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "vaultwarden": { + "typical_mb": 37, + "peak_mb": 40, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "paperless-ngx": { + "typical_mb": 645, + "peak_mb": 755, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "bookstack": { + "typical_mb": 148, + "peak_mb": 222, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "calibre-web": { + "typical_mb": 161, + "peak_mb": 242, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "kimai": { + "typical_mb": 155, + "peak_mb": 250, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "privatebin": { + "typical_mb": 38, + "peak_mb": 60, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "bentopdf": { + "typical_mb": 10, + "peak_mb": 15, + "box": "demo-hp (HP t740)", + "run": "R1-set", + "cite": "audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json" + }, + "grafana": { + "typical_mb": 269, + "peak_mb": 383, + "box": "demo-hp (HP t740)", + "run": "R4-b1", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b1/summary.json" + }, + "grimmory": { + "typical_mb": 451, + "peak_mb": 624, + "box": "demo-hp (HP t740)", + "run": "R4-b1", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b1/summary.json" + }, + "karakeep": { + "typical_mb": 372, + "peak_mb": 643, + "box": "demo-hp (HP t740)", + "run": "R4-b1", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b1/summary.json" + }, + "mealie": { + "typical_mb": 226, + "peak_mb": 240, + "box": "demo-hp (HP t740)", + "run": "R4-b1", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b1/summary.json" + }, + "n8n": { + "typical_mb": 351, + "peak_mb": 578, + "box": "demo-hp (HP t740)", + "run": "R4-b2", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b2/summary.json" + }, + "sparkyfitness": { + "typical_mb": 316, + "peak_mb": 410, + "box": "demo-hp (HP t740)", + "run": "R4-b2", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b2/summary.json" + }, + "uptime-kuma": { + "typical_mb": 118, + "peak_mb": 119, + "box": "demo-hp (HP t740)", + "run": "R4-b2", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b2/summary.json" + }, + "wishlist": { + "typical_mb": 118, + "peak_mb": 121, + "box": "demo-hp (HP t740)", + "run": "R4-b2", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b2/summary.json" + }, + "crafty-controller": { + "typical_mb": 79, + "peak_mb": 80, + "box": "demo-hp (HP t740)", + "run": "R4-b3", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b3/summary.json" + }, + "emby": { + "typical_mb": 136, + "peak_mb": 142, + "box": "demo-hp (HP t740)", + "run": "R4-b3", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b3/summary.json" + }, + "gitea": { + "typical_mb": 76, + "peak_mb": 89, + "box": "demo-hp (HP t740)", + "run": "R4-b3", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b3/summary.json" + }, + "grocy": { + "typical_mb": 33, + "peak_mb": 34, + "box": "demo-hp (HP t740)", + "run": "R4-b3", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b3/summary.json" + }, + "lubelogger": { + "typical_mb": 56, + "peak_mb": 65, + "box": "demo-hp (HP t740)", + "run": "R4-b4", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b4/summary.json" + }, + "qbittorrent": { + "typical_mb": 18, + "peak_mb": 18, + "box": "demo-hp (HP t740)", + "run": "R4-b4", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b4/summary.json" + }, + "metube": { + "typical_mb": 62, + "peak_mb": 63, + "box": "demo-hp (HP t740)", + "run": "R4-b4", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b4/summary.json" + }, + "transmission": { + "typical_mb": 6, + "peak_mb": 7, + "box": "demo-hp (HP t740)", + "run": "R4-b4", + "cite": "audits/sizing-check-2026-10-14/evidence/R4-b4/summary.json" + } + } +} \ No newline at end of file diff --git a/documentation/audits/sizing-check-2026-10-14/raise-only-source.patch b/documentation/audits/sizing-check-2026-10-14/raise-only-source.patch new file mode 100644 index 00000000..1da023dd --- /dev/null +++ b/documentation/audits/sizing-check-2026-10-14/raise-only-source.patch @@ -0,0 +1,68 @@ +diff --git a/documentation/audits/sizing-2026-10-13/tools/build_sizing.py b/documentation/audits/sizing-2026-10-13/tools/build_sizing.py +index f198bcb4..4e3caf6a 100644 +--- a/documentation/audits/sizing-2026-10-13/tools/build_sizing.py ++++ b/documentation/audits/sizing-2026-10-13/tools/build_sizing.py +@@ -10,6 +10,10 @@ Sources, per app, in the spec's order of preference (sizing-2026-10-13 TASK §7 + (evidence/onboarding-peaks.json, collected 2026-10-13; per run, containers summed) + 3 measured — this run's bench on the N100 (evidence//summary.json) + 4 unmeasured — the catalog's own mem_request / mem_limit ++ + hp-check — RAISE ONLY (operator ruling 2026-10-14, R-1009): the second machine's figures ++ (../sizing-check-2026-10-14/hp-figures.json, HP t740, `used` basis). For each app there, ++ each figure (typical, peak) becomes the higher of the figure above and the HP figure; nothing ++ goes down. The record names every raised figure in `raised_by_hp`. + Rules: telemetry under 7 days counts only when nothing better exists (`short-telemetry`). When two + sources' peaks differ by more than 50 %, both are reported and the HIGHER is used for mem_peak_mb. + mem_typical_mb is the chosen source's typical figure. CPU always comes from the N100 bench when it +@@ -136,8 +140,28 @@ def floor(): + ) + + ++HP = os.path.join(os.path.dirname(AUDIT), "sizing-check-2026-10-14", "hp-figures.json") ++ ++ ++def hp_check(): ++ return json.load(open(HP))["apps"] if os.path.exists(HP) else {} ++ ++ ++def raise_only(typ, peak, h): ++ """The second machine can only RAISE a figure. Returns (typical, peak, raised) — raised = [(field, old, new)].""" ++ raised = [] ++ if h: ++ if h.get("typical_mb") is not None and (typ is None or h["typical_mb"] > typ): ++ raised.append(("typical", typ, h["typical_mb"])); typ = h["typical_mb"] ++ if h.get("peak_mb") is not None and (peak is None or h["peak_mb"] > peak): ++ raised.append(("peak", peak, h["peak_mb"])); peak = h["peak_mb"] ++ if typ is not None and peak is not None and peak < typ: ++ raised.append(("peak", peak, typ)); peak = typ ++ return typ, peak, raised ++ ++ + def main(): +- cat, tel, onb, ben = catalog(), telemetry(), onboarding(), bench() ++ cat, tel, onb, ben, hpc = catalog(), telemetry(), onboarding(), bench(), hp_check() + apps, table = [], [] + for slug, c in cat.items(): + t, o, b = tel.get(slug), onb.get(slug), ben.get(slug) +@@ -162,6 +186,7 @@ def main(): + peak = typ # a peak below the typical figure is two sources' noise, never a real peak + else: + src, typ, peak, when, box, disagree = "unmeasured", mb(c["mem_request"]), mb(c["mem_limit"]), None, None, False ++ typ, peak, raised = raise_only(typ, peak, hpc.get(slug)) + cpu_rest = b["cpu_rest_avg_pct"] if b else (round(t["cpu_avg_dockerpct"] / 4, 2) if t else None) + cpu_pk = b["cpu_peak_1m_pct"] if b else (round(t["cpu_max_15min"] / 4, 1) if t else None) + rec = dict( +@@ -170,11 +195,13 @@ def main(): + cpu_class=cpu_class(cpu_rest, cpu_pk), cpu_note=CPU_NOTES.get(slug, ""), + source=src, measured_on=box, date=when, + ) ++ if raised: ++ rec["raised_by_hp"] = [dict(field=f, was=o, now=n) for f, o, n in raised] + if c["lifecycle"] != "active": + rec["lifecycle"] = c["lifecycle"] + apps.append(rec) + table.append(dict(rec, catalog_request_mb=mb(c["mem_request"]), catalog_limit_mb=mb(c["mem_limit"]), +- sources_disagree=disagree, cpu_rest_pct=cpu_rest, cpu_peak_1m_pct=cpu_pk, ++ sources_disagree=disagree, cpu_rest_pct=cpu_rest, cpu_peak_1m_pct=cpu_pk, hp=hpc.get(slug), + telemetry=t, onboarding=o, bench=b)) + power = json.load(open(os.path.join(EV, "power.json"))) if os.path.exists(os.path.join(EV, "power.json")) else None + out = dict( diff --git a/documentation/audits/sizing-check-2026-10-14/tools/hp_figures.py b/documentation/audits/sizing-check-2026-10-14/tools/hp_figures.py new file mode 100644 index 00000000..baf20be3 --- /dev/null +++ b/documentation/audits/sizing-check-2026-10-14/tools/hp_figures.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""hp_figures.py — the HP t740 check's per-app figures (CHECK.md), as one file: ../hp-figures.json. + +Basis: `used` = memory.current - inactive_file, summed over an app's containers (the sizing run's definition). + R1 (evidence/R1-set/set-summary.json, 9201, full memory): typical = the median in rest B (01:08:40-01:29:30 UTC), + peak = the highest `used` over the whole run (the three heavy apps' import/transcode/upload included). + R4 (evidence/R4-b1..b4/summary.json, 9202, full memory, light batches): typical = mem_rest_avg_mb, + peak = mem_peak_used_mb (a start-up peak only — no action). +Only the apps CHECK.md compares with the page. R3 (the 4 GB guest) is NOT a source: its figures are memory pressure. +Run: python3 -I tools/hp_figures.py +""" +import json, os +HERE = os.path.dirname(os.path.abspath(__file__)); AUD = os.path.dirname(HERE); EV = os.path.join(AUD, "evidence") +R1_APPS = ["immich", "jellyfin", "nextcloud", "home-assistant", "vaultwarden", "paperless-ngx", + "bookstack", "calibre-web", "kimai", "privatebin", "bentopdf"] +out = {} +s = json.load(open(os.path.join(EV, "R1-set", "set-summary.json"))) +for a in R1_APPS: + o = s["apps"][a] + out[a] = dict(typical_mb=o["restB_median_mb"], peak_mb=o["peak_used_mb"], box="demo-hp (HP t740)", + run="R1-set", cite="audits/sizing-check-2026-10-14/evidence/R1-set/set-summary.json") +for b in (1, 2, 3, 4): + s = json.load(open(os.path.join(EV, f"R4-b{b}", "summary.json"))) + evs = json.load(open(os.path.join(EV, f"R4-b{b}", "events.json"))) + deployed = {e["app"] for e in evs if e["kind"] == "deploy_done"} + for a, o in s["apps"].items(): + if a in deployed and o.get("rest_samples"): + out[a] = dict(typical_mb=o["mem_rest_avg_mb"], peak_mb=o["mem_peak_used_mb"], box="demo-hp (HP t740)", + run=f"R4-b{b}", cite=f"audits/sizing-check-2026-10-14/evidence/R4-b{b}/summary.json") +json.dump(dict(date="2026-10-14", basis="used = memory.current - inactive_file", apps=out), + open(os.path.join(AUD, "hp-figures.json"), "w"), indent=1) +print(len(out), "apps") diff --git a/documentation/backlog/CLOSED-ITEMS.md b/documentation/backlog/CLOSED-ITEMS.md index a37ab56f..90318ab7 100644 --- a/documentation/backlog/CLOSED-ITEMS.md +++ b/documentation/backlog/CLOSED-ITEMS.md @@ -26,6 +26,13 @@ --- +## 2026-10-14 — the FAQ says who sets the BIOS, and how to turn the machine off + +| Row | What | Closed | Evidence | +|---|---|---|---| +| **R-1003** | **The FAQ promised that the BIOS is set at installation to switch the box on after a power cut — „a standard part of every installation" — but a self-install from the ISO has no such step** (P1) | CLOSED 2026-10-14 — the operator's ruling and text: „If I install the machine, I set its BIOS … If you install Felhom yourself, I help you set it"; tested with real power cuts (8 of 8, 2026-10-14). No menu steps on the site (ruling). Live on https://felhom.eu/gyik and https://felhom.eu/en/faq, JSON-LD and visible copy, read back 2026-10-14 | felhom.eu `d61aa206`; `audits/night-power-2026-10-13/` | +| **R-1000** | **The website did not say how to turn a Felhom machine off** (P4) | CLOSED 2026-10-14 — new FAQ answer „Hogyan kapcsolhatom ki a gépet?" / „How do I turn the machine off?": the dashboard path with the controller's own labels (Beállítások → Rendszer → „A gép kikapcsolása"), the 14-day quiet, the power button, and why not to pull the plug (R-1007). Live, both languages, read back 2026-10-14 | felhom.eu `d61aa206` | + ## 2026-10-13 (night) — the household can turn the machine off Built and proven live on the N100 (`documentation/audits/poweroff-2026-10-13/`, architecture `14-host-power-off.md`). Fleet delivery waits on the operator: golden bake, vouch agent 0.155.0 + bundle, floor 0.310.0 with min_agent 0.155.0. diff --git a/documentation/backlog/OPEN-ITEMS.md b/documentation/backlog/OPEN-ITEMS.md index 1714aea8..24769815 100644 --- a/documentation/backlog/OPEN-ITEMS.md +++ b/documentation/backlog/OPEN-ITEMS.md @@ -112,13 +112,11 @@ match what the reader sees is how an instrument stops being believed (R-421). It **nothing was proven on the day this line was drawn** and a stopping line that moves a status is a stopping line that lies. -## Install & onboarding — 11 rows (P1 1, P3 7, P4 3) +## Install & onboarding — 9 rows (P3 7, P4 2) | ID | Category | Sev | What | State | Blocked on | Next action | Owner | |---|---|---|---|---|---|---|---| -| **R-1003** | Install & onboarding | P1 | **The FAQ promises that the BIOS is set at installation to switch the box on after a power cut — „a standard part of every installation" — but a self-install from the ISO has no such step, and the setting decides whether a box comes back.** Measured 2026-10-13/14 (`audits/night-power-2026-10-13/`): demo-hp stayed OFF after a real cut until the operator set „After Power Loss = Previous State" by hand (20:05); with it, demo-hp came back by itself from 6 cuts that night. The N100's setting is unknown (no monitor). In July both boxes stayed off 4.5 h (`AUDIT-power-outage-recovery-2026-07-22.md` F8). Same text in `website/gyik.html` and `website/en/faq.html`. | **OPEN** — filed 2026-10-14. P1 by the register's own definition (a promise we make is false today); the operator ranks it | operator: what to promise | Operator: choose — (a) the FAQ says the household (or the install guide) sets it, with the menu names; (b) the installer's first-boot check reads it where the firmware exposes it; the website is not changed until then | operator | -| **R-1009** | Install & onboarding | P3 | **Operator: decide whether the machine estimator's figures move with the 2026-10-14 check on a second machine** (`audits/sizing-check-2026-10-14/CHECK.md`, „Proposal"). The three answers on an 8 GB box held (files + passwords fit well; Immich alone tight — 14 OOM kills in its ML container; the family set does not fit — real harm); the family set's real simultaneous peak was 97 % of the page's sum. Proposed (A): Immich peak 3,061 → 3,300, Jellyfin 373 → 450, Home Assistant 328 → 435, Nextcloud 525 → 300, Vaultwarden 99 → 45, Grimmory typical 346 → 450; no verdict changes (site rule: family 5,515 → 5,659). (B) a sentence for Immich on 8 GB. | **WAITING-ON-OPERATOR** — filed 2026-10-14 | operator | Operator: A, A + B, or neither; then CC edits the data and rebuilds the page's data file | operator | -| **R-1000** | Install & onboarding | P4 | **The website does not say how to turn a Felhom machine off.** Since controller v0.310.0 (R-998) the dashboard has „A gép kikapcsolása" (Beállítások → Rendszer); the FAQ still has only the power-cut line. Text to add (HU/EN): turn it off there; switch it on again with the power button; the hub stays quiet for 14 days. The plug trick is not mentioned to households. | **READY** — filed 2026-10-13 | — | One website task: one FAQ entry per language, `site_gates.py` | CC | +| **R-1009** | Install & onboarding | P3 | **Operator: decide whether the machine estimator's figures move with the 2026-10-14 check on a second machine** (`audits/sizing-check-2026-10-14/CHECK.md`, „Proposal"). The three answers on an 8 GB box held (files + passwords fit well; Immich alone tight — 14 OOM kills in its ML container; the family set does not fit — real harm); the family set's real simultaneous peak was 97 % of the page's sum. Proposed (A): Immich peak 3,061 → 3,300, Jellyfin 373 → 450, Home Assistant 328 → 435, Nextcloud 525 → 300, Vaultwarden 99 → 45, Grimmory typical 346 → 450; no verdict changes (site rule: family 5,515 → 5,659). (B) a sentence for Immich on 8 GB. | **WAITING-ON-OPERATOR** — 2026-10-14: the „raise only" ruling was built (`raise-only-source.patch`, `hp-figures.json`: 23 figures up, none down) and **STOPPED before the push**: one `SIZING.md` §5 verdict changes — „Household set without Immich" on 8 GB, fits well → tight (peak 2,731 MB × 1.5 = 4,096.5 vs 4,096 MB: 0.5 MB over; any one of the Jellyfin, Home Assistant, Paperless raises removed keeps it). Presets unchanged. `CHECK.md` last section | operator: A, B or C | Operator: (A) publish the raise and accept that one verdict; (B) round the HP figures down to 10 MB first (to be checked); (C) keep today's figures | operator | | **R-980** | Install & onboarding | P3 | **A person cannot check that the installer is ours: the ISO's first boot runs the install script from a movable tag with no hash or signature, and the ISO checksum sits on the same website as the download.** Fix: a signed manifest (or signature) checked by the ISO before it runs the script, and the public key printed in the guide. Security review 2026-10-12 (`audits/security-review-2026-10-12/`); detail held off-git for the operator (`/mnt/5_hdd/felhom.eu/evidence/security-review-2026-10-12-PRIVATE/`). | OPEN — design-proposal | — | Design note, then build | CC | | **R-250** | Install & onboarding | P3 | **A customer create can fail fail-closed because the host-key scan ladder is shorter than the DNS/AAAA settle time.** Found 2026-08-07 creating the fifth walk's venue. `POST /configs/new` with off-site enabled provisions the Storage Box sub-account and then scans its SSH host key to pin it — **fail-closed by design** (`offsite.go:111-121`, *"don't serve a descriptor the controller can't verify"*), with `defaultScanBackoff` = 2+4+8+16+30 ≈ **60 s**, sized by its own comment to *"the observed DNS propagation lag"*. **Measured, both halves:** the first create exhausted the ladder — five `no such host`, then `dial tcp [2a01:4f8:bacc:2:200::d30]:23: connect: network is unreachable` (the name had just begun resolving, **AAAA-first, into a pod with no IPv6 route**) — and the hub logged `[ERROR] offsite provision for walk5`. An **identical second POST succeeded** ~70 s later on its own final rung (`shared already provisioned … (subaccount 285351)`). **Total settle ≈ 100 s against a 60 s budget.** The endpoint was never the problem: verified afterwards from both the node and the hub pod, by name, `SSH-2.0-OpenSSH_9.6p1`. **Two distinct things are wrong and should not be merged:** (1) the budget is sized against DNS *existence*, but what actually bit is the **AAAA-before-A** window, a different and longer phenomenon — lengthen the ladder and/or prefer the A record for the scan dial; (2) **the operator is told nothing actionable** — the create fails with a generic error, the remedy is "press it again", and nothing says so. The retry is genuinely safe (idempotent on the label; confirmed afterwards — `one_time_secrets` 1, one sub-account, no double-provision) **but that safety is invisible to the person deciding whether pressing again will double-charge them.** **Severity LOW-MEDIUM:** self-clearing, no data at risk — but it is the first thing a new customer's provisioning does and it fails looking like an outage. | **VERIFY** — **2026-10-12: hub 0.147.0 live** (S29, decision 224): the host-key scan dials IPv4 first, waits ~2 min (was ~1), and when it still runs out says the store was created and it is safe to press Save again (`TestScannerDialsIPv4First`, `TestScanOutOfRetriesSaysSafeToPressAgain`, red-proved). Known limit: the hub Ingress reads for 60 s, so a very slow first try can show a gateway timeout while provisioning finishes in the background (pressing again is safe). **READY** — operator ruling 2026-10-12 (S29, `09` §3 decision 224): yes — the scan prefers IPv4, waits longer, and the error says „safe to press again". **READY** — owner Viktor | — | The next real off-site setup (a new customer): read the hub log for the scan; success or the new message → close | CC | | **R-516** | Install & onboarding | P3 | **[P3-LOW] English a customer meets on a fresh box and its apps' first screens — enumerated by the big night.** MEASURED 2026-09-14 (BIGNIGHT, VM 333, ISO 1.27.1, controller 0.242.0). Felhom-owned: (1) the dashboard menu item **„Debug"**; (2) the dashboard CPU tile **„Load: 0.29 / 0.39 / 0.37"**; (3) the launcher tile **„Filebrowser"** opens a login in English with no Felhom text (R-513); (4) the storage page mixes formal „Adjon hozzá / Csatlakoztasson" with the product's „te". App first screens a household meets before any Felhom text helps: (5) **Uptime Kuma 2.4 opens on „Which database would you like to use?"** (SQLite / Embedded MariaDB, „Next") — the app card's „Első lépések" does not mention it; (6) PrivateBin, Gokapi, AdventureLog and FileBrowser UIs are English (the apps' own). Already rows: the Proxmox installer screens (R-495, answered by the guide), `wiki.DOMAIN` (R-498). **Fix shape:** rename „Debug"/„Load" (controller); add the Uptime Kuma database step to its card, or pre-seed `db-config.json` for SQLite in the template (catalog). **Added by F4 (20:04:54Z):** (7) the storage page prints the disconnect time as a raw ISO UTC string „Leválasztva: 2026-09-14T19:58:02Z"; (8) the „Meghajtó leválasztva" banner appears twice on every page; (9) „4 telepített alkalmazás nem fut — nézze meg a rendszermonitort" uses the formal form. **Added by F7 (20:50–21:00Z, system disk at 95 %):** (10) a banner on every page in English, „**SSD disk usage high: 90%**"; (11) the dashboard tile reads „Rendszer (/) 61.8 GB / 68.7 GB (**90%**)" while `df` reports **95 %** (reserved blocks ignored), and „(/)" labels the data volume `/mnt/sys_drive`; the deploy page says nothing about free disk. **Added by the i18n spike (2026-09-17, controller v0.247.0):** (12) **six formal („ön") forms in the converted dashboard copy** — „Olvassa be telefonnal" (launcher QR hint), „Biztosan kikapcsolja a megosztást?" (launcher), „Ha újratelepíti … importálnia kell" (the layout's remove-app modal), „Kérjük, vegye fel a kapcsolatot" (backups empty state). They were NOT fixed — a localisation release may not change Hungarian bytes — and are now COUNTED by `controller/scripts/i18n_missing_gate.py` (`HU_FORMAL_CEILING` = 6, a ratchet: a new one convicts, fixing one lowers it). The inventory (`audits/I18N-INVENTORY-2026-09-17.md`) is the list this row closes against in localisation slice 6 (R-561). **Extended 2026-09-17 by slice 1 (controller v0.248.0–v0.250.0):** the converted copy now counts **16** formal forms (`HU_FORMAL_CEILING` 6 → 10 → 12 → 16, each raise stated, Hungarian unchanged by rule) — and the count is an UNDER-count: the gate's stem list sees 4 on the release C pages (storage, drive wizards, debug), while a wider list („írja be”, „adja meg”, „adjon hozzá”, „válassza ki”, „biztosan eltávolítja”, „engedélyezze”, „hozzon létre”, „kattintson”) finds at least **22** keys there alone (login „Adja meg a jelszavát”, the NAS guide, the storage confirms). Widen the stems when this row is worked; the ceiling rises with them. **NARROWED 2026-09-20 by localisation slice 6's walk, item by item** (`audits/i18n-slice6-2026-09-20/R-516-item-by-item.md`): item 2 FIXED (slice 1), items 3 and 6 APP-OWNED (and for an English household item 6 is an advantage), item 11 is a wrong NUMBER rather than a language and needs its own row, item 1 is correct English for an English reader and still open for a Hungarian one. **Items 4, 7, 8, 9 and 10 are NOT DECIDABLE FROM AN ENGLISH WALK** - they are about Hungarian copy quality, a disconnected drive and a full disk, none of which this walk had. **What this row is now waiting for is a HUNGARIAN walk on a box with a second drive**, not another English one; saying it closed would be the kind of closure that makes a register stop meaning anything. **-- 2026-10-08:** the recovery screen's own answers are Go literals in Hungarian only — 14 `renderRecovery` messages in `controller/internal/web/recovery_handlers.go` (seen: the safe default „A művelet nem fejeződött be…" rendered on an English page in a red-proof run). Today's new R-304 sentence is a bundle key in both languages; the 14 old ones are not. | **NARROWED 2026-09-20 - rank P3-LOW; owner: CC; needs a Hungarian walk** **2026-10-05 (burn-down night): items 1 and 4 FIXED on controller `main`** (`d7efa7b`): the menu says „Hibakeresés” (English „Debug”); the storage pages use the te-form (formal-form ceiling 17 → 14; 116 parity fixtures changed by exactly those bytes). Item 2 was already fixed. **Left:** items 5–6 (apps' own screens) and the rest. **2026-10-06: items 1 and 4 DELIVERED** (controller v0.298.0). **2026-10-06 (burn-down night, later): items 7–10 and 12 FIXED on controller `main`** (`e4774e6`, `2e5ef7e`): the disconnect time in local time; one banner per unplugged drive; „nézd meg”; the disk/memory/CPU/temperature banners in the household's language (`health.*` keys; the wire text to the hub unchanged and pinned); the last eleven counted formal forms are te-form (ceiling 14 → 0; 119 parity fixtures changed by exactly those bytes). **Left:** items 5–6 (apps' own screens); about 20 formal forms the gate does not count yet („írja be”, „adja meg”, „hozzon létre”, …) on the debug, storage, network-storage and security pages. Item 11 moved to its own row **R-889**. **2026-10-06 02:40: the bundle has NO formal form left** (controller `4c3c203`, unreleased): 49 sentences in the te-form; the gate's stem list widened (it counted 61 on the previous bundle, 0 now) with decoys and one listed third-person exception. **Left (narrowed):** Felhom-owned formal forms in Go and template literals the gate cannot read yet — the network-storage attach errors, escrow and share handlers, the single-copy notice, a settings refusal, the SMART mail, a fill-watch „Kérjük”, and the setup wizard pages; each moves into the bundle in the te-form. Items 5–6 are the apps' own screens (not ours). **2026-10-06: items 7–10, 12 and the bundle's 49 formal forms DELIVERED** in controller v0.299.0. **2026-10-06 night: the Go-literal leftover moved** (controller main `3d6ba28`, unreleased): 22 Go-literal messages — escrow handler (6), share password page (2), export upload (1), network-storage uid/remove/attach failures (13) — are bundle keys (`api.escrow.*`, `api.share.*`, `api.export.*`, `api.netstorage.*`) in the te-form with English (14 were formal); a detached attach failure renders in the reader's language; the NAS refusal says „Válassz". `TestR516_GoLiteralFormalFormsAreGone` (source scan + positive control), `TestR516_NetAddFailureFollowsTheReader`; red-proofs `audits/night-burndown-2026-10-06/ctrl/R-516-red*.txt`. **Left:** the setup wizard and recovery-info.txt (R-554); the SMART and fill-watch texts (the hub event's message — wire text, needs the household-copy producer shape); items 5–6 (apps' own screens). | — | — | CC | @@ -139,7 +137,7 @@ stopping line that lies. | **R-707** | Apps & catalog | P4 | **[P2] 37 apps still start with a login a stranger can take (`09` §3 decision 45).** Audit of all 53 apps: `app-catalog-felhom.eu/FIRST-ADMIN.md` (class, fix route, status, measured or read). Open: **3 hard-coded defaults** — calibre-web (`admin / admin123`, measured working on demo-hp and 9202; its own `cps.py -s` route needs a generated password WITH a special character — our generator is letters+digits, a controller change), mealie (`changeme@example.com / MyPassword`), wger (`admin / adminadmin`); **34 open first-run screens** (the first visitor creates the admin: actualbudget, adventurelog, audiobookshelf, calcom, docmost, emby, ghost, gitea, gramps-web, home-assistant, homebox, immich, jellyfin, komga, n8n, navidrome, opengist, outline, papra, plant-it, radarr, rallly, recipe-importer, romm, seerr, sonarr, sparkyfitness, tandoor, termix, uptime-kuma, vikunja, wanderer, wishlist, zipline). **Stale notes:** romm's `default_creds` `admin / admin` answers 401 on demo-hp (like a wrong password) — the page now warns with a login that does not exist; zipline's looks stale too. **Measured on demo-hp 2026-09-28 (read-only):** bookstack's default still logs in on the INSTALLED app (the fix is for new installs; the page now warns). Each fix: route (a) env or (b) the app's own CLI/API via `after_install:`, proven on 9202 with the default failing and the generated password working; route (c) a page sentence. Several sessions (operator, 2026-09-28). **2026-09-29 (controller v0.280.0, catalog `d0e7e2e`):** every class-3 app fixed — mealie, wger, calibre-web by `after_install` (calibre-web with the new `password:24:special`), proven on 9202 fresh installs (`audits/login-gate-2026-09-29/D/`); the setup gate (decision 46, spike PASSED) built and live on immich, n8n, audiobookshelf (probes measured) and uptime-kuma (button) (`…/C/`); romm's and zipline's stale notes removed. **Left: 30 class-4 apps** — gate each (probe measured on 9202 where one exists — 11 upstream candidates listed in `…/B/B-VERDICT.md` §3; the button otherwise). **2026-09-29 afternoon (controller v0.281.0, catalog `6faf432`):** 28 more class-4 apps gated — 32 of 34 — each proven on 9202 (`audits/gate-rollout-2026-09-29/`B): stranger → gate page / 401, household reached the first-setup screen, the gate opened (9 by a measured probe, the rest by the press), the app answered after. seerr, outline, rallly: gated, their opening needs a media server / e-mail (not proven). **Left:** wanderer (R-714); plant-it is not installable. | **NARROWED** (2026-10-03 triage: the row's verdict was finished, but it names open work no other row carries — seerr, outline and rallly are gated, but the gate OPENING is not proven (needs a media server / e-mail)) — **CLOSED — 2026-09-29 (the rest → R-714)** | — | — | CC | | **R-905** | Apps & catalog | P4 | **wger's collected style files (283 MB) go into every wger backup, though the app rebuilds them at every start.** Since R-762's fix (`DJANGO_DEBUG=False` + `collectstatic`, catalog `cf1ed43`) the static files sit in a named volume that the backup legs copy like data (measured 283 MB, `audits/design-build-2026-10-06/`); `collectstatic` regenerates them, so they are rebuildable bytes in every Tier-1 unit, Tier-2 copy and off-site snapshot. wger is `lifecycle: hidden` and no box runs it (hub read 2026-10-08). Options to weigh: a non-backed-up volume class for regenerable data, or an anonymous volume. Found by the R-762 helpers 2026-10-08. | **OPEN — needs a design before wger is shown** | — | Design the „regenerable volume" exclusion (catalog + controller backup legs) | CC | | **R-995** | Apps & catalog | P4 | **Removing an app through the product can leave unnamed Docker volumes and an empty import folder behind.** MEASURED 2026-10-13 on demo-felhom after 61 removals (`documentation/audits/sizing-2026-10-13/METHOD.md` „Teardown”, `evidence/leftover-volumes-after-bench.txt`): 8 unnamed volumes (4–72 KB; immich 1, romm 2, privatebin 3, onlyoffice 2 — volumes the images declare themselves) and `userdata/import/{paperless,calibre,grimmory}` (empty, created by the deploy). The removal answered 200 and its own leftover check was clean, because it looks only at labelled volumes. No data risk; disk and tidiness only. Removed by name on the bench. | **READY** — filed 2026-10-13 | — | Make the removal account for the containers' unnamed volumes (and an empty import folder it created), with a test that deploys an image that declares a VOLUME | CC | -| **R-1007** | Apps & catalog | P3 | **After a power cut Immich shows photos as done that are broken, and nothing repairs them: of 101 photos uploaded in the last ~30 s before the cut, 81 kept their original but got an EMPTY thumbnail and preview (162 zero-byte files), and 1 — answered „201 Created" 0.3 s before the cut — has no original at all.** Immich's job queues are empty after the start, so it never redoes them (`audits/night-power-2026-10-13/` R2, `immich-check.txt`). Same ext4 effect on a plain 40-byte file. Other apps may do the same with files written just before a cut. | **READY** — filed 2026-10-14 | — | CC: on a scratch box, find whether Immich's own „missing thumbnails" job and its integrity report find and fix these; if so, decide (operator) whether Felhom runs them after an unclean boot | CC | +| **R-1007** | Apps & catalog | P3 | **After a power cut Immich shows photos as done that are broken, and nothing repairs them: of 101 photos uploaded in the last ~30 s before the cut, 81 kept their original but got an EMPTY thumbnail and preview (162 zero-byte files), and 1 — answered „201 Created" 0.3 s before the cut — has no original at all.** Immich's job queues are empty after the start, so it never redoes them (`audits/night-power-2026-10-13/` R2, `immich-check.txt`). Same ext4 effect on a plain 40-byte file. Other apps may do the same with files written just before a cut. | **WAITING-ON-OPERATOR** — 2026-10-14 spike (`audits/power-cut-fixes-2026-10-14/README.md` F, Immich v3.2.4): „thumbnails: missing" does not fix a zero-byte thumbnail (nor after deleting it — Immich decides from its database); „thumbnails: all" (force) fixes them; `integrity-missing-files` (`/api/admin/integrity/report?type=missing_file`) finds a lost original; nothing restores one but a backup | operator: the proposal | Operator: yes/no — after an unclean boot, run Immich's integrity-missing-files and show its report, and run „thumbnails: all" in the night window (CPU in proportion to the library); then CC builds it as a catalog-level hook | operator | ## App updates — 4 rows (P3 4) @@ -155,7 +153,7 @@ stopping line that lies. | ID | Category | Sev | What | State | Blocked on | Next action | Owner | |---|---|---|---|---|---|---|---| -| **R-1006** | Backup & restore | P2 | **A power cut during the night's whole-guest backup leaves the half-written archive on the host, and from then on every local whole-guest backup of that box is skipped „not enough space" — nothing removes it.** Measured 2026-10-14 on demo-hp (`audits/night-power-2026-10-13/` R3): the cut at 02:52 left `vzdump-lxc-9201-2026_10_14-04_42_18.tar.dat` (12.2 GB) + `.tmp`; host root at 92 %; agent retry 03:08 „skipped: not enough space: local has 2.9 GiB free; the last archive … was 4.6 GiB". The operator got `whole_guest_backup_failed` (true). Second part: the space preflight sizes the new archive from the LAST one (7.2 GB planned) while the guest had grown (12.4 GB written when cut) — a backup that outgrows the plan can fill the host root. Third part, the same rule the other way: after one big archive (14.4 GB at 05:17, while bench data sat on the guest disk) the next preflight would need ~21 GB of 5.5 GB free — the box skips every later backup though the guest is small again; cleared by hand at 07:28 UTC (`night-power-2026-10-13/README.md`). | **READY** — filed 2026-10-14 | — | CC: on agent start and before each backup, remove a `vzdump-*.tar.dat`/`.tmp` left by a run that is not in flight (with a test that plants one); size the preflight from the guest's used space, not the last archive; prove on demo-hp with a cut | CC | +| **R-1006** | Backup & restore | P2 | **A power cut during the night's whole-guest backup leaves the half-written archive on the host, and from then on every local whole-guest backup of that box is skipped „not enough space" — nothing removes it.** Measured 2026-10-14 on demo-hp (`audits/night-power-2026-10-13/` R3): the cut at 02:52 left `vzdump-lxc-9201-2026_10_14-04_42_18.tar.dat` (12.2 GB) + `.tmp`; host root at 92 %; agent retry 03:08 „skipped: not enough space: local has 2.9 GiB free; the last archive … was 4.6 GiB". The operator got `whole_guest_backup_failed` (true). Second part: the space preflight sizes the new archive from the LAST one (7.2 GB planned) while the guest had grown (12.4 GB written when cut) — a backup that outgrows the plan can fill the host root. Third part, the same rule the other way: after one big archive (14.4 GB at 05:17, while bench data sat on the guest disk) the next preflight would need ~21 GB of 5.5 GB free — the box skips every later backup though the guest is small again; cleared by hand at 07:28 UTC (`night-power-2026-10-13/README.md`). | **WAITING-ON-OPERATOR** — 2026-10-14: built and released in agent v0.156.1 (0.156.0 superseded); `audits/power-cut-fixes-2026-10-14/` (spike C1–C4, 19 red-proofs); part 2 (the preflight sizing) settled by C4 — vzdump cleans its own ENOSPC partial, `spaceFits` stays | operator: vouch agent 0.156.1 + bundle | CC: after the vouch — deliver (agent → step bundle → bundle) to demo-hp; one real cut mid-backup; prove the partial is gone, the next backup succeeds; then Tester 1 and the N100 | CC | | **R-978** | Backup & restore | P3 | **Starting the escrow ceremony again can make the customer's written-down recovery code useless: the new package replaces the old one before the new code is confirmed.** It can happen to an ordinary customer who starts and walks away. Fix: keep the previous package until the new code is confirmed. Security review 2026-10-12 (`audits/security-review-2026-10-12/`); detail held off-git for the operator (`/mnt/5_hdd/felhom.eu/evidence/security-review-2026-10-12-PRIVATE/`). | OPEN | — | Fix in the agent with a test | CC | | **R-973** | Backup & restore | P3 | **A compromised controller can make the whole-guest backup run again and again, and each run prunes older restore points: the local tier directly, and the off-site tier through the store's own nightly keep-last-2.** So `07` §8 row 10 (ransomware) is weaker than it reads. The second reader confirmed the local half; the off-site half is inferred. Fix: a cadence or daily cap on the local-API backup route, and off-site snapshots that a nightly prune cannot remove. Security review 2026-10-12 (`audits/security-review-2026-10-12/`); detail held off-git for the operator (`/mnt/5_hdd/felhom.eu/evidence/security-review-2026-10-12-PRIVATE/`). | OPEN | — | Add the cap with a test; decide the off-site retention shape | CC | | **R-926** | Backup & restore | P3 | **After a household removes an `after_install` app keeping its backups and then restores it, nobody has checked whether the app page still shows a password that works.** A `type: password` deploy field is NOT in `stacks.PortableSecretEnvVars` (that list is `type: secret` only, `deploy.go:1199`), so it does not travel in the backup unit — the same shape as R-765, where Radicale's restored login was replaced by a value no page showed and every phone lost its calendar. MEASURED on 9202 2026-10-10 for grocy: remove keeping the backups, restore, and **the household's own password still signs in and the seeded data reads back** — so the DATABASE half is right. What was NOT settled is the PAGE half: `app.yaml` holds the value encrypted at rest, so reading it proves nothing (a 76-character string that of course did not sign in), and `POST /apps/grocy/initial-credentials/reveal` answered 404 for this app. Seven apps are in the same class: bookstack, calibre-web, claper, dawarich, grocy, mealie, wger. | **OPEN — the question is unanswered, not the answer bad** | — | Ask the controller what the app page renders for a generated `type: password` after a restore (one endpoint read, one app), and if it differs from the one that works, decide: carry the value in the unit, or re-run `after_install` after a restore | CC | @@ -214,7 +212,7 @@ stopping line that lies. | **R-779** | Security & access | P4 | **[P3-LOW] Part A's "two outside addresses seen as two" is proven through the simulated tunnel only; on the REAL tunnel the second outside address (ep0, one request allowed) was refused by Cloudflare's edge with 403 and never reached the box.** Measured 2026-10-01 19:51 UTC (`audits/visitors-2026-10-01/A/L2-demo-hp-real-tunnel.txt`): no log line on demo-hp; demo-hp's box has no geo restriction in its settings, so a Cloudflare ZONE rule (country or bot, not read) refused a German datacenter address. DooPlex's own address on the real tunnel was seen as itself. **Needs:** one sign-in from a second Hungarian address (the operator's phone off wifi) while DooPlex is locked out — 2 minutes; and say which Cloudflare rule refused ep0. | **WAITING-ON-OPERATOR — rank P3-LOW; owner: operator (a phone), CC reads the logs** **Re-ranked 2026-10-03: P3→P4: a proof gap on the real tunnel; operator-only follow-up.** | — | — | CC + operator | | **R-904** | Security & access | P4 | **Cloudflare can read every household's app traffic; replacing it with our own relay is a later item.** Facts (reviewer discussion 2026-10-08; `01`): app traffic and the dashboard reach the box through the Cloudflare Tunnel (`01` §5 trust table, rows end-user ↔ apps and customer ↔ controller UI; §7), and the tunnel's public end is Cloudflare's edge, where TLS ends — so Cloudflare can technically read that traffic (the FAQ says so since 2026-10-08, R-900; „TLS ends at the edge" is not written in `01` — add it there). What Cloudflare gives today, free: inbound reach with no router setup, the CGNAT answer (`01` §4, §7); certificates (`01` §7, the free tier covers one level below a zone); the geo-WAF the hub enforces (`01` §5 last row, §7); flood protection (not in `01`). The alternative named: our own EU relay over WireGuard with TLS passthrough by SNI, certificates on the box, the geo-block on the relay. Its costs: one more machine the operator keeps up, and a single point of reach for every box; weaker flood protection; about a week of work after a spike. Operator ruling 2026-10-08 09:07 (`09` §3 decision 184): a later item. | **DEFERRED** — **2026-10-10: the asked sentence is in `01` §7** („TLS ends at Cloudflare's edge …”); the spike stays deferred. **DEFERRED — after the first customers (operator ruling 2026-10-08 09:07)** | — | A spike after the first customers (the relay's reach, cost and flood behaviour, measured) | operator | -## Box system & updates — 10 rows (P2 1, P3 6, P4 3) +## Box system & updates — 11 rows (P2 1, P3 7, P4 3) | ID | Category | Sev | What | State | Blocked on | Next action | Owner | |---|---|---|---|---|---|---|---| @@ -227,7 +225,8 @@ stopping line that lies. | **R-78** | Box system & updates | P3 | **`local_api` authority ruling — auto-reconcile vs detect-only** **MIGRATED FROM `ROADMAP.md` 2026-08-22 (R-369) — originally filed 2026-07-25, size M, roadmap state `idea (deferred OUT of R-77 on purpose)`.** Moved verbatim; nothing added or reinterpreted. The roadmap keeps its copy as history, marked moved. **An OWED OPERATOR DECISION, not a defect — moved because an owed ruling hidden among feature ideas is the shape this session exists to remove.** | **OPEN — migrated from ROADMAP 2026-08-22, rank unchanged** | — | R-77 ships detection because the fix is genuinely undecided, and **both directions can lose customer-visible function**. **Direction 1 (today):** `controller.yaml` wins and drift is silent → the 2026-07-25 island migration blinded the whole fleet's control plane for 17.5 h (drive gate, guest-reboot recovery, quiesce/backup all degrade). R-77 makes that loud but does not stop it recurring. **Direction 2 (`bootstrap.json` wins, auto-reconcile on boot):** a guest whose `controller.yaml` is CORRECT and whose `bootstrap.json` is stale — a half-completed re-provision, a hand-repaired guest, a setup-wizard box — gets a **working channel clobbered on the next restart**, fleet-wide and silently, during a routine deploy. That is not obviously better than the bug. Needs a spike: which writer is authoritative per field (endpoint vs fingerprint vs token — `mergeLocalAPI` replaces the whole block, so they cannot be reconciled independently today), whether the agent side should stamp a generation/mtime so 'newer wins' is even expressible, and whether reconcile should require an operator ack. Until then the drift alert plus a manual edit is the supported path. | CC | | **R-468** | Box system & updates | P3 | **[P3-LOW] THE GOLDEN WAIVER — goldens on a cadence, not per release (operator ruling 2026-09-13).** 25 goldens in 26 days in August, almost one per release, because `golden_currency_gate.py` trips on every release by design and the only honest ways past it were a bake or a declared `--no-verify` (thirteen by 2026-09-01, R-404/R-417). **The ruling: bake WEEKLY, and always before any drill or fresh install.** Every release still raises the FLOOR, so both demo boxes keep getting each release in ~20 s; only the golden — which protects a fresh install and nothing else — moves to a cadence. **The mechanism (built 2026-09-13):** `documentation/tests/golden-waiver. **⚠ CORRECTED THE SAME DAY (R-472): between bakes the floor does NOT carry a release — the hub holds any floor above the vouched golden (publish-train rule 1), so releases between bakes reach the demo boxes only by hand-deploy.**yml`, four lines (`issued`, `expires`, `reason`, `register_row: R-468`), read by the gate. While valid, a golden BEHIND the record makes the gate print a loud ADVISORY and exit 0; when it expires the gate is red again until someone bakes or renews. **The 14-day cap is enforced by the gate, not the runbook** — a longer, undated, unparseable, reason-less or row-less waiver is INCONCLUSIVE (exit 2), never 0 and never silently ignored. **It never covers a golden that is UNRECORDED (R-385)** — that is not a cadence choice. **A dated waiver cannot be forgotten; it just expires** — the difference from R-242's original rule, which recurred the day after it was written. Tests: `scripts/test_golden_currency_gate.py` cases 5–15 (E/F/G/H, a 15-day, absent, unparseable, bad-row and empty-reason waiver each 2; the R-421 decoy — a file saying only `expires` — 2). **This is a PRE-CUSTOMER arrangement: the first external install retires it** (delete the file in that commit). Cadence written into `RUNBOOK-manual-build.md` §4.2 and the `felhom.eu` end-of-session checklist. **Does NOT touch R-242's open half (nothing gates the VOUCH).** | **WATCHING — rank P3-LOW; owner: CC (renew ≤ 14 days or bake); retire at the first external install** | — | — | CC | | **R-1004** | Box system & updates | P4 | **After an unclean stop every guest ext4 volume waits ~43 s in multi-mount protection before it mounts, so each restart after a power cut takes ~1.5 min longer.** Kernel: „EXT4-fs warning (device dm-6): ext4_multi_mount_protect: MMP interval 42 higher than expected, please wait" on dm-6/dm-7 and loop0/loop1, demo-hp and Tester 1, every round of `audits/night-power-2026-10-13/` (R1 `repairs.txt`). | **READY** — filed 2026-10-14 | — | CC: find where MMP is switched on for the guest volumes (Proxmox `pct` create, or our own mkfs), whether it protects anything on a one-host disk, and measure the restart without it on a scratch guest | CC | -| **R-1008** | Box system & updates | P4 | **After a power cut the agent's stale-lock check calls the dead backup „genuinely in-flight" and leaves `lock: backup` on the guest.** Harmless today only because Proxmox's own start-all removes it in the same second („removed left over backup lock from '9201'!", demo-hp 2026-10-14 02:52:56, `audits/night-power-2026-10-13/R3/lock-lines.txt`). Nothing can be in flight on a fresh boot. | **READY** — filed 2026-10-14 | — | CC: treat a vzdump task from before the current boot as not in flight (boot id / btime), with a test | CC | +| **R-1008** | Box system & updates | P4 | **After a power cut the agent's stale-lock check calls the dead backup „genuinely in-flight" and leaves `lock: backup` on the guest.** Harmless today only because Proxmox's own start-all removes it in the same second („removed left over backup lock from '9201'!", demo-hp 2026-10-14 02:52:56, `audits/night-power-2026-10-13/R3/lock-lines.txt`). Nothing can be in flight on a fresh boot. | **WAITING-ON-OPERATOR** — 2026-10-14: built in agent v0.156.1 — in flight = the worker process lives (Proxmox's pid + pstart rule), not a wall-clock start (0.156.0's btime rule was fail-open under a clock step — security review); measured: the dead task stays listed only ~30–60 s after a boot (C3), so the live harm is that first minute | operator: vouch agent 0.156.1 | CC: the live proof with R-1006 (the agent's start-up line after the cut) | CC | +| **R-1010** | Box system & updates | P3 | **The agent's stale-lock janitor cannot remove a half-removed `vzdump` snapshot: Proxmox's delete fails on the first missing volume („Failed to find logical volume pve/snap_vm-9201-disk-0_vzdump") and the janitor retries the same call every 10 min forever, leaving the other volume's snapshot and the snapshot entry.** Seen 2026-10-14 after a backup ran out of space on a full host root (LVM could not write its metadata; `audits/power-cut-fixes-2026-10-14/README.md` C4). Fixed by hand with `pct delsnapshot 9201 vzdump --force`. | **READY** — filed 2026-10-14 | — | CC: on that error, retry once with `--force` (a sudoers line or the API's `force=1`), with a test that replays the error text; next agent release | CC | ## Monitoring & notifications — 8 rows (P2 1, P3 5, P4 2) diff --git a/scripts/CHANGELOG.md b/scripts/CHANGELOG.md index 21fd0717..0e51cd00 100644 --- a/scripts/CHANGELOG.md +++ b/scripts/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2026-10-14 — host install: the uninstall removes the new root helper felhom-vzdump-partial-clean (R-1006) + +- `felhom-host-install.sh` step 4d removes `/usr/local/sbin/felhom-vzdump-partial-clean` (agent v0.156.1's bundle installs + it; a fresh install gets it with the vouched bundle). `test_hostinstall.py`'s frozen bundle list carries it + (`test_bundle_list_is_current` caught the gap). `SCRIPT_VERSION` stays 1.32.0: the change reaches boxes with the next + installer release. + ## 2026-10-14 (night) — plugcut: a real power cut on the HP, with three ways back on - New `scripts/bench/plugcut.py` + `test_plugcut.py` (12 tests): refuses any plug but `hp` (N100 only with diff --git a/scripts/felhom-host-install.sh b/scripts/felhom-host-install.sh index 71a7a4d8..77def038 100644 --- a/scripts/felhom-host-install.sh +++ b/scripts/felhom-host-install.sh @@ -1351,6 +1351,8 @@ run_uninstall() { if [[ -f /usr/local/sbin/felhom-mkfs-guarded ]]; then run rm -f /usr/local/sbin/felhom-mkfs-guarded; else log_skip " felhom-mkfs-guarded already absent"; fi if [[ -f /usr/local/sbin/felhom-pbs-apply ]]; then run rm -f /usr/local/sbin/felhom-pbs-apply; else log_skip " felhom-pbs-apply already absent"; fi if [[ -f /usr/local/sbin/felhom-backup-target-apply ]]; then run rm -f /usr/local/sbin/felhom-backup-target-apply; else log_skip " felhom-backup-target-apply already absent"; fi + # agent v0.156.1 (R-1006): the dead half-written vzdump remover the config bundle installs. + if [[ -f /usr/local/sbin/felhom-vzdump-partial-clean ]]; then run rm -f /usr/local/sbin/felhom-vzdump-partial-clean; else log_skip " felhom-vzdump-partial-clean already absent"; fi if [[ -f /usr/local/sbin/felhom-os-apply ]]; then run rm -f /usr/local/sbin/felhom-os-apply; else log_skip " felhom-os-apply already absent"; fi # R-881: the content checker the config bundle installs since agent v0.146.1 (R-861). if [[ -f /usr/local/sbin/felhom-priv-apply ]]; then run rm -f /usr/local/sbin/felhom-priv-apply; else log_skip " felhom-priv-apply already absent"; fi diff --git a/scripts/test_hostinstall.py b/scripts/test_hostinstall.py index 729b867f..739e05d7 100644 --- a/scripts/test_hostinstall.py +++ b/scripts/test_hostinstall.py @@ -34,6 +34,7 @@ BUNDLE_DESTS = [ "/usr/local/sbin/felhom-selfupdate-guarded", "/usr/local/sbin/felhom-pbs-apply", "/usr/local/sbin/felhom-backup-target-apply", + "/usr/local/sbin/felhom-vzdump-partial-clean", # agent v0.156.1 (R-1006) "/usr/local/sbin/felhom-os-apply", "/usr/local/sbin/felhom-crash-guard", "/usr/local/sbin/felhom-priv-apply",