### identity
demo-felhom
2026-11-10T07:33:04Z
### in flight
(end in flight)
### agent state dir (depth 2, bytes)
724164608	/var/lib/felhom-agent
723935232	/var/lib/felhom-agent/selfupdate
24576	/var/lib/felhom-agent/os
20480	/var/lib/felhom-agent/wg
16384	/var/lib/felhom-agent/guests
16384	/var/lib/felhom-agent/felhom-sshd
12288	/var/lib/felhom-agent/pbsdr
12288	/var/lib/felhom-agent/guests/9201
8192	/var/lib/felhom-agent/escrow-stage
### selfupdate listing
total 706972
drwxr-x--- 2 felhom-agent felhom-agent     4096 2026-11-07 23:05 .
drwxr-x--- 9 felhom-agent felhom-agent     4096 2026-11-10 06:07 ..
-rwxr-xr-x 1 felhom-agent felhom-agent 14190233 2026-09-16 11:04 felhom-agent-0.131.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14195690 2026-09-17 10:34 felhom-agent-0.132.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14262881 2026-09-24 21:34 felhom-agent-0.133.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14271901 2026-09-24 22:51 felhom-agent-0.134.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14273488 2026-09-27 12:21 felhom-agent-0.135.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14274024 2026-09-27 13:51 felhom-agent-0.136.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14273888 2026-09-27 14:07 felhom-agent-0.137.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14278858 2026-09-30 10:37 felhom-agent-0.138.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14284239 2026-10-04 09:07 felhom-agent-0.139.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14355058 2026-10-04 11:37 felhom-agent-0.140.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14376457 2026-10-04 13:22 felhom-agent-0.141.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14376105 2026-10-04 13:52 felhom-agent-0.141.1
-rwxr-xr-x 1 felhom-agent felhom-agent 14427309 2026-10-04 16:22 felhom-agent-0.142.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14427261 2026-10-04 18:22 felhom-agent-0.142.1
-rwxr-xr-x 1 felhom-agent felhom-agent 14461055 2026-10-04 19:38 felhom-agent-0.143.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14493513 2026-10-05 07:23 felhom-agent-0.144.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14494141 2026-10-05 07:53 felhom-agent-0.144.1
-rwxr-xr-x 1 felhom-agent felhom-agent 14494291 2026-10-05 09:38 felhom-agent-0.145.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14504200 2026-10-05 12:38 felhom-agent-0.146.1
-rwxr-xr-x 1 felhom-agent felhom-agent 14504898 2026-10-05 19:09 felhom-agent-0.147.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14515292 2026-10-06 00:39 felhom-agent-0.148.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14569321 2026-10-06 11:51 felhom-agent-0.149.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14590727 2026-10-07 09:06 felhom-agent-0.150.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14631775 2026-10-07 13:45 felhom-agent-0.151.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14715119 2026-10-07 17:26 felhom-agent-0.152.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14719868 2026-10-07 18:57 felhom-agent-0.153.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14731041 2026-10-09 07:16 felhom-agent-0.154.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14783135 2026-10-13 20:36 felhom-agent-0.155.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14820843 2026-10-14 14:34 felhom-agent-0.156.1
-rwxr-xr-x 1 felhom-agent felhom-agent 14874944 2026-10-15 21:19 felhom-agent-0.157.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14914581 2026-10-16 04:04 felhom-agent-0.158.0
-rwxr-xr-x 1 felhom-agent felhom-agent 14919180 2026-10-16 04:20 felhom-agent-0.158.1
-rwxr-xr-x 1 felhom-agent felhom-agent 16007680 2026-10-17 17:39 felhom-agent-0.159.1
-rwxr-xr-x 1 felhom-agent felhom-agent 16007752 2026-10-18 14:25 felhom-agent-0.159.2
-rwxr-xr-x 1 felhom-agent felhom-agent 16071264 2026-10-19 03:55 felhom-agent-0.160.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16082682 2026-10-20 08:55 felhom-agent-0.161.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16094507 2026-10-22 09:54 felhom-agent-0.162.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16183935 2026-10-23 08:54 felhom-agent-0.163.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16272301 2026-10-24 09:39 felhom-agent-0.164.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16286780 2026-10-25 17:09 felhom-agent-0.165.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16291150 2026-10-26 09:10 felhom-agent-0.166.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16296021 2026-10-27 10:25 felhom-agent-0.167.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16320123 2026-10-29 10:03 felhom-agent-0.168.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16350287 2026-10-29 12:48 felhom-agent-0.169.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16379051 2026-10-29 17:04 felhom-agent-0.170.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16389260 2026-10-30 10:19 felhom-agent-0.171.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16393888 2026-11-03 08:49 felhom-agent-0.172.0
-rwxr-xr-x 1 felhom-agent felhom-agent 16393888 2026-11-07 23:04 felhom-agent-0.172.1
### selfupdate subdirs listing
### agent state top-level listing (names, sizes, dates only)
total 160
drwxr-x---  9 felhom-agent felhom-agent  4096 2026-11-10 06:07 .
drwxr-xr-x 48 root         root          4096 2026-10-08 04:39 ..
-rw-------  1 felhom-agent felhom-agent   631 2026-11-10 05:40 backup-success-state.json
-rw-------  1 felhom-agent felhom-agent   290 2026-11-09 13:14 controller-swap-9201.json
-rw-------  1 felhom-agent felhom-agent    61 2026-08-03 09:15 drive-intents.json
drwx------  2 felhom-agent felhom-agent  4096 2026-10-03 17:03 escrow-stage
drwx------  2 felhom-agent felhom-agent  4096 2026-08-03 09:15 felhom-sshd
-rw-------  1 felhom-agent felhom-agent    67 2026-10-14 15:29 guest-binds.json
-rw-------  1 felhom-agent felhom-agent   243 2026-11-04 10:54 guest-disk-trim.json
drwx------  3 felhom-agent felhom-agent  4096 2026-08-03 09:17 guests
-rw-------  1 felhom-agent felhom-agent 65125 2026-11-10 06:07 journal.log
-rw-------  1 felhom-agent felhom-agent   305 2026-11-03 10:22 line-test.json
-rw-r--r--  1 felhom-agent felhom-agent   591 2026-08-03 09:15 local-api.crt
-rw-------  1 felhom-agent felhom-agent   227 2026-08-03 09:15 local-api.key
-rw-------  1 felhom-agent felhom-agent   164 2026-08-09 09:51 local-tokens.log
-rw-------  1 felhom-agent felhom-agent  5440 2026-11-07 23:20 nonces.log
drwx------  2 felhom-agent felhom-agent  4096 2026-11-10 08:20 os
drwx------  2 felhom-agent felhom-agent  4096 2026-08-03 09:15 pbsdr
-rw-------  1 felhom-agent felhom-agent    77 2026-10-13 20:52 poweroff.json
-rw-------  1 felhom-agent felhom-agent   383 2026-11-10 06:07 restore-test-state.json
drwxr-x---  2 felhom-agent felhom-agent  4096 2026-11-07 23:05 selfupdate
-rw-------  1 felhom-agent felhom-agent    40 2026-11-03 08:49 store-grant-seen.json
drwx------  2 felhom-agent felhom-agent  4096 2026-08-03 09:15 wg
### os-apply state
376832	/var/lib/felhom-os-apply
364544	/var/lib/felhom-os-apply/bundle-prev
159744	/var/lib/felhom-os-apply/bundle-prev/20261107T222001Z-before-0.172.1
159744	/var/lib/felhom-os-apply/bundle-prev/20261026T081007Z-before-0.166.0
40960	/var/lib/felhom-os-apply/bundle-prev/20261029T160408Z-before-0.170.0
4096	/var/lib/felhom-os-apply/agent-update
/var/lib/felhom-os-apply:
total 20
drwx------  4 root root 4096 2026-11-07 23:20 .
drwxr-xr-x 48 root root 4096 2026-10-08 04:39 ..
drwxr-xr-x  2 root root 4096 2026-11-07 23:04 agent-update
drwxr-xr-x  5 root root 4096 2026-11-07 23:20 bundle-prev
-rw-r--r--  1 root root   96 2026-11-07 23:20 nonces.json

/var/lib/felhom-os-apply/bundle-prev:
total 20
drwxr-xr-x 5 root root 4096 2026-11-07 23:20 .
drwx------ 4 root root 4096 2026-11-07 23:20 ..
drwxr-xr-x 3 root root 4096 2026-10-26 09:10 20261026T081007Z-before-0.166.0
drwxr-xr-x 3 root root 4096 2026-10-29 17:04 20261029T160408Z-before-0.170.0
drwxr-xr-x 3 root root 4096 2026-11-07 23:20 20261107T222001Z-before-0.172.1
### usr local bin + sbin
/usr/local/bin:
total 183644
drwxr-xr-x  2 root root     4096 2026-11-07 23:04 .
drwxr-xr-x 11 root root     4096 2026-05-21 13:31 ..
-rwxr-xr-x  1 root root 16393888 2026-11-07 23:04 felhom-agent
-rwxr-xr-x  1 root root 13984713 2026-08-03 15:06 felhom-agent.bak-0.120.0
-rwxr-xr-x  1 root root 14081336 2026-08-03 15:27 felhom-agent.bak-0.121.0
-rwxr-xr-x  1 root root 14085464 2026-08-03 16:41 felhom-agent.bak-0.121.1
-rwxr-xr-x  1 root root 14076649 2026-08-03 18:54 felhom-agent.bak-0.122.0
-rwxr-xr-x  1 root root 14081569 2026-08-04 09:38 felhom-agent.bak-0.123.0
-rwxr-xr-x  1 root root 14093378 2026-08-04 13:42 felhom-agent.bak-0.124.1
-rwxr-xr-x  1 root root 14115378 2026-08-06 16:58 felhom-agent.bak-0.125.0
-rwxr-xr-x  1 root root 14124876 2026-08-08 16:30 felhom-agent.bak-0.127.0
-rwxr-xr-x  1 root root 14141686 2026-08-12 18:45 felhom-agent.bak-0.128.0
-rwxr-xr-x  1 root root 14161760 2026-08-20 12:18 felhom-agent.bak-0.129.0
-rwxr-xr-x  1 root root 14284239 2026-10-04 11:02 felhom-agent.bak-0.139.0
-rwxr-xr-x  1 root root 16393888 2026-11-03 08:49 felhom-agent.prev

/usr/local/sbin:
total 336
drwxr-xr-x  2 root root   4096 2026-11-07 23:20 .
drwxr-xr-x 11 root root   4096 2026-05-21 13:31 ..
-rwxr-xr-x  1 root root   6654 2026-08-03 09:15 felhom-backup-target-apply
-rwxr-xr-x  1 root root  12986 2026-07-18 18:17 felhom-bootstrap.sh
-rwxr-xr-x  1 root root   8687 2026-10-04 16:12 felhom-crash-guard
-rwxr-xr-x  1 root root   6816 2026-10-16 04:35 felhom-internal-place
-rwxr-xr-x  1 root root   3393 2026-08-03 09:15 felhom-mgmt-watchdog
-rwxr-xr-x  1 root root   5439 2026-10-16 04:35 felhom-mkfs-guarded
-rwxr-xr-x  1 root root 142720 2026-11-07 23:20 felhom-os-apply
-rwxr-xr-x  1 root root  50933 2026-10-04 18:22 felhom-os-apply.pre-bundle
-rwxr-xr-x  1 root root   9087 2026-08-03 09:15 felhom-pbs-apply
-rwxr-xr-x  1 root root   6316 2026-07-18 18:30 felhom-pbs-apply.bak-20260718-preR39
-rwxr-xr-x  1 root root   6822 2026-07-18 20:21 felhom-pbs-apply.bak-20260721-101635
-rwxr-xr-x  1 root root  26062 2026-10-17 17:55 felhom-priv-apply
-rwxr-xr-x  1 root root   7806 2026-10-05 13:09 felhom-selfupdate-guarded
-rwxr-xr-x  1 root root    943 2026-08-03 09:15 felhom-shared-parent.sh
-rwxr-xr-x  1 root root   3844 2026-10-17 17:55 felhom-thin-check
-rwxr-xr-x  1 root root   5537 2026-10-14 15:04 felhom-vzdump-partial-clean
### dumps on drives
-- /mnt/hdd_1/dump
total 4256676
drwxr-xr-x 2 root root       4096 2026-11-10 05:35 .
drwxr-xr-x 5 root root       4096 2026-07-28 21:25 ..
-rw-r--r-- 1 root root       1506 2026-11-08 05:29 vzdump-lxc-9201-2026_11_08-05_28_21.log
-rw-r--r-- 1 root root 1530164897 2026-11-08 05:29 vzdump-lxc-9201-2026_11_08-05_28_21.tar.zst
-rw-r--r-- 1 root root         16 2026-11-08 05:29 vzdump-lxc-9201-2026_11_08-05_28_21.tar.zst.notes
-rw-r--r-- 1 root root       1506 2026-11-09 05:31 vzdump-lxc-9201-2026_11_09-05_31_03.log
-rw-r--r-- 1 root root 1528966739 2026-11-09 05:31 vzdump-lxc-9201-2026_11_09-05_31_03.tar.zst
-rw-r--r-- 1 root root         16 2026-11-09 05:31 vzdump-lxc-9201-2026_11_09-05_31_03.tar.zst.notes
-rw-r--r-- 1 root root       1506 2026-11-10 05:35 vzdump-lxc-9201-2026_11_10-05_34_26.log
-rw-r--r-- 1 root root 1299653068 2026-11-10 05:35 vzdump-lxc-9201-2026_11_10-05_34_26.tar.zst
-rw-r--r-- 1 root root         16 2026-11-10 05:35 vzdump-lxc-9201-2026_11_10-05_34_26.tar.zst.notes
-- /var/lib/vz/dump
total 19679272
drwxr-xr-x 2 root root       4096 2026-10-05 05:19 .
drwxr-xr-x 7 root root       4096 2026-07-18 18:32 ..
-rw-r--r-- 1 root root  612397450 2026-07-18 18:31 vzdump-lxc-9100-2026_07_18-18_30_52.tar.zst
-rw-r--r-- 1 root root        779 2026-07-20 17:51 vzdump-lxc-9100-2026_07_20-17_50_57.log
-rw-r--r-- 1 root root  648549366 2026-07-20 17:51 vzdump-lxc-9100-2026_07_20-17_50_57.tar.zst
-rw-r--r-- 1 root root        779 2026-07-23 15:54 vzdump-lxc-9100-2026_07_23-15_54_21.log
-rw-r--r-- 1 root root        698 2026-08-03 06:58 vzdump-lxc-9100-2026_08_03-06_58_00.log
-rw-r--r-- 1 root root  649547835 2026-08-03 06:58 vzdump-lxc-9100-2026_08_03-06_58_00.tar.zst
-rw-r--r-- 1 root root  649547835 2026-08-03 09:16 vzdump-lxc-9100-2026_08_03-09_15_34.tar.zst
-rw-r--r-- 1 root root       1579 2026-07-28 01:32 vzdump-lxc-9201-2026_07_28-01_27_56.log
-rw-r--r-- 1 root root 5820882781 2026-07-28 01:32 vzdump-lxc-9201-2026_07_28-01_27_56.tar.zst
-rw-r--r-- 1 root root         16 2026-07-28 01:32 vzdump-lxc-9201-2026_07_28-01_27_56.tar.zst.notes
-rw-r--r-- 1 root root       1579 2026-07-28 06:49 vzdump-lxc-9201-2026_07_28-06_44_22.log
-rw-r--r-- 1 root root 5840583923 2026-07-28 06:49 vzdump-lxc-9201-2026_07_28-06_44_22.tar.zst
-rw-r--r-- 1 root root         16 2026-07-28 06:49 vzdump-lxc-9201-2026_07_28-06_44_22.tar.zst.notes
-rw-r--r-- 1 root root       1579 2026-07-28 17:58 vzdump-lxc-9201-2026_07_28-17_53_26.log
-rw-r--r-- 1 root root 5929975285 2026-07-28 17:58 vzdump-lxc-9201-2026_07_28-17_53_26.tar.zst
-rw-r--r-- 1 root root         16 2026-07-28 17:58 vzdump-lxc-9201-2026_07_28-17_53_26.tar.zst.notes
### template cache
/var/lib/vz/template/cache:
total 126920
drwxr-xr-x 2 root root      4096 2026-07-20 17:47 .
drwxr-xr-x 4 root root      4096 2026-07-18 18:15 ..
-rw-r--r-- 1 root root 129954319 2026-07-14 21:37 debian-13-standard_13.6-1_amd64.tar.zst

/var/lib/vz/template/iso:
total 8
drwxr-xr-x 2 root root 4096 2026-07-23 18:37 .
drwxr-xr-x 4 root root 4096 2026-07-18 18:15 ..
### journald disk usage + limits
Archived and active journals take up 3.9G in the file system.
Nov 08 05:35:21 demo-felhom systemd-journald[308]: Runtime Journal (/run/log/journal/2b66cc60125645b48684cec2f00e2137) is 8M, max 157.7M, 149.7M free.
Nov 08 05:35:21 demo-felhom systemd-journald[308]: System Journal (/var/log/journal/2b66cc60125645b48684cec2f00e2137) is 3.9G, max 4G, 48.3M free.
### journal oldest + newest entry
2026-11-10T08:33:04+01:00 demo-felhom sudo[3488171]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=999)
2026-10-04T21:43:33+02:00 demo-felhom systemd[3282788]: Queued start job for default target default.target.
2026-10-04T21:43:33+02:00 demo-felhom systemd[3282788]: Created slice app.slice - User Application Slice.
### journal files (count, oldest, newest)
files=74 bytes=5410652160 oldest=2026-10-04 newest=2026-11-10
### journal bytes exact
4282028032	/var/log/journal
### journal last hour: lines by identifier (top 12)
  12753 "SYSLOG_IDENTIFIER":"sudo"
    442 "SYSLOG_IDENTIFIER":"felhom-agent"
    363 "SYSLOG_IDENTIFIER":"systemd"
     66 "SYSLOG_IDENTIFIER":"postfix/smtp"
     21 "SYSLOG_IDENTIFIER":"sshd-session"
     17 "SYSLOG_IDENTIFIER":"systemd-logind"
     12 "SYSLOG_IDENTIFIER":"pveproxy"
     11 "SYSLOG_IDENTIFIER":"postfix/qmgr"
      8 "SYSLOG_IDENTIFIER":"pvedaemon"
      4 "SYSLOG_IDENTIFIER":"tailscaled"
      4 "SYSLOG_IDENTIFIER":"(systemd)"
      3 "SYSLOG_IDENTIFIER":"(sd-pam)"
### journal last hour: total lines
13708
### journald conf files
[Journal]
### thin-check script
-- /usr/local/sbin/felhom-thin-check
#!/bin/bash
# felhom-thin-check — list the thin device ids a thin pool holds next to the ones LVM knows (R-1012). READ-ONLY: it
# names an orphan, it never removes one. Installed as /usr/local/sbin/felhom-thin-check (config bundle).
#
#   felhom-thin-check <vg> <pool>
#
# A full host root while vzdump takes its snapshot can leave a thin snapshot inside the pool that LVM no longer knows
# (demo-hp 2026-10-14: `thin_ls` device 4, LVM only 1 and 2); from then on every snapshot backup of that guest fails
# („Creation of new snapshot 4 of device 2 failed") and nothing said why. The repair (`dmsetup message … delete <id>`)
# stays a HUMAN step, decided by the operator — this helper only reads.
#
# Reading the live pool's device list needs a metadata snapshot: the kernel's documented way to read metadata while
# the pool is in use (`reserve_metadata_snap`, `thin_ls --metadata-snap`, `release_metadata_snap`). It changes no data
# and no mapping; it is held for the length of one `thin_ls -o DEV` (well under a second on these pools). Rules:
#   1. the arguments are an LVM name pair; <vg>/<pool> is a thin pool (lv_attr starts with `t`);
#   2. only one run at a time (a lock file); a metadata snapshot that someone ELSE holds is never touched — the run
#      refuses (exit 3) and does not release it;
#   3. LVM's list is read before AND after the pool's, so a volume created in between is not reported as an orphan
#      (the caller treats an id as an orphan only when it is in the pool and in neither LVM list).
# Output, one record per line:  lvm-before <id> <lv> | pool <id> | lvm-after <id> <lv> | ok
# Exit: 0 read, 1 refused (arguments / not a thin pool), 3 a metadata snapshot is already held or cannot be reserved,
# 4 the pool could not be read. Pinned by configs/test_felhom_thin_check.py.
set -u
LVS=/usr/sbin/lvs
DMSETUP=/usr/sbin/dmsetup
THIN_LS=/usr/sbin/thin_ls
LOCK=/run/lock/felhom-thin-check.lock
NAME_RE='^[A-Za-z0-9_+][A-Za-z0-9_.+-]{0,63}$'

die() { echo "felhom-thin-check: REFUSED: $1" >&2; exit "${2:-1}"; }  # $1 the reason, $2 the exit code

lvm_ids() { # $1 tag → "<tag> <id> <lv>" for every thin volume of this pool
	"$LVS" --noheadings --separator , -o lv_name,thin_id --select "pool_lv=$POOL" -- "$VG" 2>/dev/null |
		awk -F, -v tag="$1" '{ gsub(/ /, ""); if ($2 ~ /^[0-9]+$/) print tag, $2, $1 }'
}

release() { "$DMSETUP" message "$TPOOL" 0 release_metadata_snap >/dev/null 2>&1; }

main() {
	[[ $# -eq 2 ]] || die "usage: felhom-thin-check <vg> <pool>"
	VG=$1 POOL=$2
	local attr held ids
	[[ "$VG" =~ $NAME_RE && "$POOL" =~ $NAME_RE ]] || die "not an LVM name pair"

	attr=$("$LVS" --noheadings -o lv_attr -- "$VG/$POOL" 2>/dev/null | tr -d ' ')
	[[ "$attr" == t* ]] || die "$VG/$POOL is not a thin pool"

	DM="${VG//-/--}-${POOL//-/--}"
	TPOOL="$DM-tpool"
	TMETA="/dev/mapper/${DM}_tmeta"

	exec 9>"$LOCK" || die "cannot open the lock file" 4
	flock -w 30 9 || die "another check holds the lock" 3

	# Field 7 of the thin-pool status is the held metadata root: "-" when no metadata snapshot exists.
	held=$("$DMSETUP" status "$TPOOL" 2>/dev/null | awk '$3 == "thin-pool" { print $7 }')
	[[ -n "$held" ]] || die "cannot read the status of $TPOOL" 4
	[[ "$held" == "-" ]] || die "a metadata snapshot of $TPOOL is already held (by someone else) — not touched" 3

	lvm_ids lvm-before
	"$DMSETUP" message "$TPOOL" 0 reserve_metadata_snap >/dev/null 2>&1 || die "could not reserve a metadata snapshot of $TPOOL" 3
	trap release EXIT
	trap 'release; trap - EXIT; exit 4' INT TERM HUP
	if ! ids=$("$THIN_LS" --metadata-snap --no-headers -o DEV "$TMETA" 2>/dev/null); then
		die "thin_ls could not read $TMETA" 4
	fi
	release
	trap - EXIT INT TERM HUP
	awk '$1 ~ /^[0-9]+$/ { print "pool", $1 }' <<<"$ids"
	lvm_ids lvm-after
	echo ok
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@"; fi
-- /usr/local/sbin/felhom-vzdump-partial-clean
#!/bin/bash
# felhom-vzdump-partial-clean — remove ONE dead, half-written vzdump backup (R-1006). The only delete the agent's
# sudoers allows for backup files. Installed as /usr/local/sbin/felhom-vzdump-partial-clean (config bundle).
#
#   felhom-vzdump-partial-clean <storage> <vmid> <name>
#
# A power cut during the night backup leaves `vzdump-lxc-<vmid>-<date>.tar.dat` (+ its `.tmp/` directory) on the
# storage; nothing removes it, every later backup is then skipped „not enough space", and a full host root breaks
# LVM's own metadata writes (power-cut-fixes-2026-10-14 C4). This helper removes such a leftover — and NOTHING else.
# It trusts nothing the agent says; every rule is checked here, as root, and any doubt REFUSES (exit 1):
#   1. the arguments: a storage id, a vmid, and a name that is exactly
#      vzdump-(lxc|qemu)-<THAT vmid>-YYYY_MM_DD-HH_MM_SS.(tar.dat|vma.dat|tmp) — never a finished archive
#      (.tar.zst, .vma.zst, …), never a .log/.notes, never another vmid;
#   2. the storage: its section in /etc/pve/storage.cfg is `dir: <storage>` with a `path`; the target is
#      <path>/dump/<name>, not a symlink, and its real directory IS <path>/dump;
#   3. not in use: no vzdump runs on the host at all (a task `…:vzdump:…` or a `vzdump` command — a job for several
#      guests carries no vmid in its task name), and no
#      process holds the target (or, for the .tmp directory, anything under it) open or as its working directory
#      (C2: after a killed worker, tar and zstd kept writing into the .tar.dat for 4 min 40 s, none named „vzdump");
#   4. dead: last modified before this boot (`btime` in /proc/stat), or untouched for at least QUIET_S seconds.
# Prints one line on success: „removed <name> <bytes> B (<why>)". Pinned by configs/test_felhom_vzdump_partial_clean.py.
set -u
STORAGE_CFG=/etc/pve/storage.cfg
PROC=/proc
QUIET_S=120

die() { echo "felhom-vzdump-partial-clean: REFUSED: $*" >&2; exit 1; }

storage_path() { # $1 storage id → prints the dir storage's path, or nothing
	awk -v want="$1" '
		/^[a-z]+:[ \t]/ { split($0, h, /:[ \t]+/); type = h[1]; id = h[2]; sub(/[ \t]+$/, "", id); in_s = (id == want); next }
		in_s && /^[ \t]+path[ \t]/ { if (type == "dir") { sub(/^[ \t]+path[ \t]+/, ""); sub(/[ \t]+$/, ""); print; exit } }
	' "$STORAGE_CFG"
}

storage_type() {
	awk -v want="$1" '/^[a-z]+:[ \t]/ { split($0, h, /:[ \t]+/); id = h[2]; sub(/[ \t]+$/, "", id); if (id == want) { print h[1]; exit } }' "$STORAGE_CFG"
}

boot_time() { awk '$1 == "btime" { print $2 }' "$PROC/stat"; }

newest_mtime() { # a file's mtime, or the newest mtime of anything in a directory (the dir itself included)
	if [[ -d "$1" ]]; then find "$1" -xdev -printf '%T@\n' 2>/dev/null | sort -n | tail -1 | cut -d. -f1
	else stat -c %Y "$1"; fi
}

busy_reason() { # $1 vmid, $2 target → prints why it is in use, or nothing
	local vmid=$1 target=$2 p link cmd
	for p in "$PROC"/[0-9]*; do
		[[ -r "$p/cmdline" ]] || continue
		cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
		# ANY vzdump on the host refuses: a job for several guests runs as `task UPID:…:vzdump::…` (no vmid in it), and
		# its tar may be between this guest's files. Waiting costs one backup run; deleting a live one costs the backup.
		if [[ "$cmd" == *":vzdump:"* ]] || [[ "$cmd" =~ (^|/)vzdump([[:space:]]|$) ]]; then
			echo "a vzdump runs on this host (pid ${p##*/})"; return
		fi
		for link in "$p/cwd" "$p"/fd/*; do
			local to
			to=$(readlink "$link" 2>/dev/null) || continue
			if [[ "$to" == "$target" || "$to" == "$target/"* ]]; then
				echo "pid ${p##*/} has it open"; return
			fi
		done
	done
}

main() {
	local storage=${1:-} vmid=${2:-} name=${3:-}
	[[ $# -eq 3 ]] || die "usage: felhom-vzdump-partial-clean <storage> <vmid> <name>"
	[[ "$storage" =~ ^[A-Za-z][A-Za-z0-9_-]{0,63}$ ]] || die "bad storage id"
	[[ "$vmid" =~ ^[1-9][0-9]{2,8}$ ]] || die "bad vmid"
	[[ "$name" =~ ^vzdump-(lxc|qemu)-${vmid}-[0-9]{4}_[0-9]{2}_[0-9]{2}-[0-9]{2}_[0-9]{2}_[0-9]{2}\.(tar\.dat|vma\.dat|tmp)$ ]] \
		|| die "name is not a half-written vzdump of guest $vmid: $name"
	[[ "$(storage_type "$storage")" == "dir" ]] || die "storage $storage is not a dir storage"
	local path; path=$(storage_path "$storage")
	[[ "$path" == /* ]] || die "storage $storage has no path"
	local dump="$path/dump" target="$path/dump/$name"
	[[ -e "$target" ]] || die "no such entry: $target"
	[[ ! -L "$target" ]] || die "a symlink: $target"
	[[ "$(realpath -e "$dump")" == "$(realpath -e "$(dirname "$target")")" ]] || die "not directly in $dump"
	if [[ "$name" == *.tmp ]]; then [[ -d "$target" ]] || die ".tmp is not a directory"
	else [[ -f "$target" ]] || die "not a regular file"; fi
	local busy; busy=$(busy_reason "$vmid" "$target")
	[[ -z "$busy" ]] || die "in use: $busy"
	local bt mt now why
	bt=$(boot_time); mt=$(newest_mtime "$target"); now=$(date +%s)
	[[ "$bt" =~ ^[0-9]+$ && "$mt" =~ ^[0-9]+$ ]] || die "cannot read the boot time or the modification time"
	if (( mt < bt )); then why="written before this boot"
	elif (( now - mt >= QUIET_S )); then why="untouched for $(( now - mt )) s, no vzdump for $vmid, nothing holds it"
	else die "modified $(( now - mt )) s ago — may still be written"; fi
	local size; size=$(du -sb "$target" 2>/dev/null | cut -f1)
	if [[ -d "$target" ]]; then rm -rf --one-file-system -- "$target"; else rm -f -- "$target"; fi
	[[ ! -e "$target" ]] || die "remove failed: $target"
	echo "removed $name ${size:-?} B ($why)"
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
	main "$@"
fi
### units mentioning thin-check / partial-clean
/etc/sudoers.d/felhom-agent
### lvm thin settings
thin_pool_autoextend_threshold=100
thin_pool_autoextend_percent=20
monitoring=1
active
### agent config: selected non-secret keys
-- /etc/felhom-agent/agent.json
"local_backup_target": "felhom-backup"
retention": 3
"keep_last": 0
### vzdump.conf + storage prune settings
1:dir: local
2:	path /var/lib/vz
3:	content iso,backup,import,vztmpl
5:lvmthin: local-lvm
8:	content images,rootdir
10:pbs: felhom-pbs
13:	content backup
19:dir: felhom-backup
20:	path /mnt/hdd_1
21:	content backup
22:	is_mountpoint 1
### kernels
proxmox-kernel-7.0.14-20-pve			un 
proxmox-kernel-7.0.14-20-pve-signed	7.0.14-20	1006525	ii 
proxmox-kernel-7.0.14-22-pve			un 
proxmox-kernel-7.0.14-22-pve-signed	7.0.14-22	1006522	ii 
proxmox-kernel-7.0.14-23-pve			un 
proxmox-kernel-7.0.14-23-pve-signed	7.0.14-23	1006525	ii 
proxmox-kernel-7.0.14-24-pve			un 
proxmox-kernel-7.0.14-24-pve-signed	7.0.14-24	1006528	ii 
proxmox-kernel-7.0.14-25-pve			un 
proxmox-kernel-7.0.14-25-pve-signed	7.0.14-25	1007819	ii 
proxmox-kernel-7.0.14-26-pve			un 
proxmox-kernel-7.0.14-26-pve-signed	7.0.14-26	1007815	ii 
proxmox-kernel-7.0.14-28-pve			un 
proxmox-kernel-7.0.14-28-pve-signed	7.0.14-28	1007817	ii 
proxmox-kernel-7.0.2-6-pve			un 
proxmox-kernel-7.0.2-6-pve-signed	7.0.2-6	1005219	ii 
proxmox-kernel-helper	9.1.0+fde2	109	ii 
7.0.14-28-pve
### apt history: kernel installs/removes
Start-Date: 2026-11-08  05:32:58
Start-Date: 2026-10-07  12:09:02
Start-Date: 2026-10-09  04:44:23
Start-Date: 2026-10-10  04:44:06
Start-Date: 2026-10-16  05:06:43
Start-Date: 2026-10-21  05:21:10
Start-Date: 2026-10-29  04:45:31
### apt autoremove config
APT::NeverAutoRemove "";
APT::NeverAutoRemove:: "^firmware-linux.*";
APT::NeverAutoRemove:: "^linux-firmware$";
APT::NeverAutoRemove:: "^linux-image-[a-z0-9]*$";
APT::NeverAutoRemove:: "^linux-image-[a-z0-9]*-[a-z0-9]*$";
APT::NeverAutoRemove:: "^pve-kernel-7\.0$";
APT::NeverAutoRemove:: "^proxmox-kernel-7\.0$";
APT::NeverAutoRemove:: "^proxmox-kernel-7\.0-signed$";
APT::NeverAutoRemove:: "^pve-kernel-7\.0\.14-26-pve$";
APT::NeverAutoRemove:: "^proxmox-kernel-7\.0\.14-26-pve$";
APT::NeverAutoRemove:: "^proxmox-kernel-7\.0\.14-26-pve-signed$";
APT::NeverAutoRemove:: "^pve-kernel-7\.0\.14-28-pve$";
APT::NeverAutoRemove:: "^proxmox-kernel-7\.0\.14-28-pve$";
APT::NeverAutoRemove:: "^proxmox-kernel-7\.0\.14-28-pve-signed$";
### boot
Filesystem              1B-blocks        Used   Available Use% Mounted on
/dev/mapper/pve-root 100861726720 38093606912 57597382656  40% /
/dev/sda2              1071624192     9469952  1062154240   1% /boot/efi
total 901784
drwxr-xr-x  5 root root     4096 2026-11-08 05:33 .
drwxr-xr-x 19 root root     4096 2026-07-18 18:16 ..
-rw-r--r--  1 root root   308363 2026-09-24 12:43 config-7.0.14-20-pve
-rw-r--r--  1 root root   308363 2026-10-06 07:41 config-7.0.14-22-pve
-rw-r--r--  1 root root   308363 2026-10-07 22:25 config-7.0.14-23-pve
-rw-r--r--  1 root root   308363 2026-10-09 13:32 config-7.0.14-24-pve
-rw-r--r--  1 root root   308500 2026-10-17 22:42 config-7.0.14-25-pve
-rw-r--r--  1 root root   308500 2026-10-22 20:45 config-7.0.14-26-pve
-rw-r--r--  1 root root   308500 2026-11-02 12:54 config-7.0.14-28-pve
-rw-r--r--  1 root root   308342 2026-05-20 10:55 config-7.0.2-6-pve
drwxr-xr-x  3 root root     4096 1970-01-01 01:00 efi
drwxr-xr-x  6 root root     4096 2026-11-08 05:36 grub
-rw-r--r--  1 root root 87751579 2026-10-07 14:16 initrd.img-7.0.14-20-pve
-rw-r--r--  1 root root 87761381 2026-10-09 04:44 initrd.img-7.0.14-22-pve
-rw-r--r--  1 root root 87787201 2026-10-10 04:44 initrd.img-7.0.14-23-pve
-rw-r--r--  1 root root 87763821 2026-10-16 05:06 initrd.img-7.0.14-24-pve
-rw-r--r--  1 root root 87852290 2026-10-21 05:21 initrd.img-7.0.14-25-pve
-rw-r--r--  1 root root 87853652 2026-10-29 04:45 initrd.img-7.0.14-26-pve
-rw-r--r--  1 root root 87847391 2026-11-08 05:33 initrd.img-7.0.14-28-pve
-rw-r--r--  1 root root 91998300 2026-10-04 13:35 initrd.img-7.0.2-6-pve
-rw-r--r--  1 root root   151020 2024-11-17 17:43 memtest86+ia32.bin
-rw-r--r--  1 root root   152064 2024-11-17 17:43 memtest86+ia32.efi
-rw-r--r--  1 root root   155992 2024-11-17 17:43 memtest86+x64.bin
-rw-r--r--  1 root root   157184 2024-11-17 17:43 memtest86+x64.efi
drwxr-xr-x  2 root root     4096 2026-11-08 05:33 pve
-rw-r--r--  1 root root 10483512 2026-09-24 12:43 System.map-7.0.14-20-pve
-rw-r--r--  1 root root 10483429 2026-10-06 07:41 System.map-7.0.14-22-pve
-rw-r--r--  1 root root 10483429 2026-10-07 22:25 System.map-7.0.14-23-pve
-rw-r--r--  1 root root 10483455 2026-10-09 13:32 System.map-7.0.14-24-pve
-rw-r--r--  1 root root 10341401 2026-10-17 22:42 System.map-7.0.14-25-pve
-rw-r--r--  1 root root 10341401 2026-10-22 20:45 System.map-7.0.14-26-pve
-rw-r--r--  1 root root 10341916 2026-11-02 12:54 System.map-7.0.14-28-pve
-rw-r--r--  1 root root 10452992 2026-05-20 10:55 System.map-7.0.2-6-pve
-rw-r--r--  1 root root 16284776 2026-09-24 12:43 vmlinuz-7.0.14-20-pve
-rw-r--r--  1 root root 16288872 2026-10-06 07:41 vmlinuz-7.0.14-22-pve
-rw-r--r--  1 root root 16288872 2026-10-07 22:25 vmlinuz-7.0.14-23-pve
-rw-r--r--  1 root root 16288872 2026-10-09 13:32 vmlinuz-7.0.14-24-pve
-rw-r--r--  1 root root 16284776 2026-10-17 22:42 vmlinuz-7.0.14-25-pve
-rw-r--r--  1 root root 16280680 2026-10-22 20:45 vmlinuz-7.0.14-26-pve
### pve tasks / rrd
3162112	/var/log/pve
21041152	/var/lib/rrdcached
706
### root dir extras
16269312	/root
1007616	/root/n100-rehearsal
20480	/root/.ssh
12288	/root/.cache
8192	/root/.config
### df now
Filesystem              1B-blocks        Used       Avail Use% Mounted on
/dev/mapper/pve-root 100861726720 38093606912 57597382656  40% /
  data           374538764288B 3.03   0.58  
  root           103079215104B              
  swap             8589934592B              
  vm-9201-disk-2  34359738368B 6.48         
  vm-9201-disk-3 268435456000B 3.40         
### end
2026-11-10T07:33:05Z
