Compare commits
568
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
91bd96c755 | ||
|
|
6f933d6ee3 | ||
|
|
5ddd124cfc | ||
|
|
3055bc54c1 | ||
|
|
007199342f | ||
|
|
54d3a1057b | ||
|
|
fbbb168d1f | ||
|
|
3911cc6f52 | ||
|
|
ad80ee749a | ||
|
|
0683cc2dab | ||
|
|
b1bc63dd32 | ||
|
|
e66a48f2b7 | ||
|
|
b890b57540 | ||
|
|
aa69114d18 | ||
|
|
abcc1e6833 | ||
|
|
d912307bb7 | ||
|
|
feaaa0e8ff | ||
|
|
6b6ba0c17f | ||
|
|
4697f8ac9e | ||
|
|
4a1a84941c | ||
|
|
c8d3c17b13 | ||
|
|
f5c3d77a61 | ||
|
|
5e05627c4e | ||
|
|
1e087bb158 | ||
|
|
29bcf4403e | ||
|
|
08ca67861f | ||
|
|
f7e1dcbb27 | ||
|
|
2d4b1930df | ||
|
|
b4721eee11 | ||
|
|
1109092def | ||
|
|
b45c47be6c | ||
|
|
b9b831fb90 | ||
|
|
39fb9cbf6c | ||
|
|
ae69ad4512 | ||
|
|
e803ad2b91 | ||
|
|
7651461be2 | ||
|
|
a0c88e5183 | ||
|
|
2d5840ba11 | ||
|
|
b310773dbf | ||
|
|
0da792e670 | ||
|
|
286a7e3ecf | ||
|
|
4e1621c650 | ||
|
|
51521230c9 | ||
|
|
84ef5818d5 | ||
|
|
56aabc729c | ||
|
|
0cbd500154 | ||
|
|
1edfd8e331 | ||
|
|
0bc0bc5da0 | ||
|
|
45458196d1 | ||
|
|
124164db73 | ||
|
|
c11e3e2a87 | ||
|
|
1b6e31675b | ||
|
|
1693da4d1a | ||
|
|
79ba884859 | ||
|
|
f17e1979bd | ||
|
|
0963bae3c3 | ||
|
|
8561dc8a5c | ||
|
|
f7b7308c4d | ||
|
|
27bbdaaa22 | ||
|
|
e25454ccc3 | ||
|
|
65c1994d3b | ||
|
|
6545cccff7 | ||
|
|
eb38d78945 | ||
|
|
4a1c07b288 | ||
|
|
5c3b218822 | ||
|
|
e15d1d30ea | ||
|
|
344c4f9030 | ||
|
|
c52d591851 | ||
|
|
37b99d73b3 | ||
|
|
dd8b2ace02 | ||
|
|
383522450a | ||
|
|
1a8bed11e7 | ||
|
|
98f3cdd76a | ||
|
|
888a98571b | ||
|
|
912f7c127e | ||
|
|
e569ec6c5f | ||
|
|
52ed52c131 | ||
|
|
c5daa9480d | ||
|
|
d2053e7d47 | ||
|
|
7d4e564758 | ||
|
|
9b749ee7ad | ||
|
|
7ae3081a76 | ||
|
|
7ecd63cd61 | ||
|
|
a2f3aeeeff | ||
|
|
e7d4ecd607 | ||
|
|
10ca63aa32 | ||
|
|
68a2130681 | ||
|
|
bdf25f019c | ||
|
|
c6d9435316 | ||
|
|
f38b03cdb8 | ||
|
|
3403c87c80 | ||
|
|
86adbcaf8a | ||
|
|
24ea960229 | ||
|
|
cff6fb5a45 | ||
|
|
db25b469ba | ||
|
|
b228b44f0e | ||
|
|
91b9405c81 | ||
|
|
4c69c25b48 | ||
|
|
c9013bb47d | ||
|
|
2d1e5d0774 | ||
|
|
f09c53efc1 | ||
|
|
92d647a6f6 | ||
|
|
d03ab7f1f5 | ||
|
|
b17d1c597d | ||
|
|
0ae01dfb52 | ||
|
|
dd7cdc09e7 | ||
|
|
7b0a8b234b | ||
|
|
ce1a4b4758 | ||
|
|
ac90169a5d | ||
|
|
154d6dcaa9 | ||
|
|
2f7072050f | ||
|
|
85e799f360 | ||
|
|
3a72a4811b | ||
|
|
a165d53c88 | ||
|
|
adaf86ad57 | ||
|
|
74b5eae5b0 | ||
|
|
7e82f325b8 | ||
|
|
acccb66bd3 | ||
|
|
de812bc027 | ||
|
|
3e8ebeb96c | ||
|
|
cefdc731a4 | ||
|
|
e56dcb8a4c | ||
|
|
f277e619e2 | ||
|
|
386f51edc6 | ||
|
|
130e3ed882 | ||
|
|
be398f92e8 | ||
|
|
ee71abd1d4 | ||
|
|
37e98f452b | ||
|
|
b2b82ae828 | ||
|
|
b78a0ff3ac | ||
|
|
96047453cb | ||
|
|
4bf5db6875 | ||
|
|
87977ff40a | ||
|
|
861d32a4b4 | ||
|
|
769c4c3cf2 | ||
|
|
64f704d0f7 | ||
|
|
208fac8027 | ||
|
|
f1b9b41214 | ||
|
|
d83316326e | ||
|
|
e06ed97fa8 | ||
|
|
e4b5cf9693 | ||
|
|
faa3cad92e | ||
|
|
fdd87178d2 | ||
|
|
0342c7bb57 | ||
|
|
6ab1e7c56c | ||
|
|
61345790ed | ||
|
|
7c4b8e599f | ||
|
|
56ef1d6655 | ||
|
|
78c890e4bb | ||
|
|
d48f1bbb23 | ||
|
|
8401a30917 | ||
|
|
d1b6004458 | ||
|
|
ca78c17b29 | ||
|
|
c8d12f1f2a | ||
|
|
dc9164c5af | ||
|
|
c9fa2e717b | ||
|
|
0af1187e03 | ||
|
|
495003051b | ||
|
|
42af3ab9bc | ||
|
|
f24dce5b95 | ||
|
|
b1746c25af | ||
|
|
2e2e8f56b8 | ||
|
|
a6bc3f1197 | ||
|
|
3bf77c3423 | ||
|
|
cfba0d022a | ||
|
|
c3d08b4821 | ||
|
|
a55eedcf2c | ||
|
|
9cac3462bb | ||
|
|
1bb8608e88 | ||
|
|
b84e0dd1bd | ||
|
|
23a8ef3de4 | ||
|
|
596238cc2e | ||
|
|
475bdce7e4 | ||
|
|
d766666ff8 | ||
|
|
a4c09a7c11 | ||
|
|
904dc20466 | ||
|
|
e1b8269be0 | ||
|
|
5c68c869b6 | ||
|
|
728d12b1a0 | ||
|
|
dd81866b16 | ||
|
|
3ef095fb71 | ||
|
|
7c986915ca | ||
|
|
16dbc83221 | ||
|
|
9555a7f93b | ||
|
|
9ff937d8fb | ||
|
|
d4be12ca95 | ||
|
|
7403c2a838 | ||
|
|
309e368731 | ||
|
|
0722b2cdb0 | ||
|
|
4fe2f81a32 | ||
|
|
9bdb4dae8f | ||
|
|
d9864a94bf | ||
|
|
77cd70f7c0 | ||
|
|
1030abd7d6 | ||
|
|
18d03bd437 | ||
|
|
e98b857684 | ||
|
|
dcdeb3d16d | ||
|
|
610804b98d | ||
|
|
4586f0f7f6 | ||
|
|
205e22babe | ||
|
|
058b945064 | ||
|
|
40d857b527 | ||
|
|
7ae6990bac | ||
|
|
7569f34aeb | ||
|
|
ede49b610d | ||
|
|
f17ed11599 | ||
|
|
1db56bf837 | ||
|
|
53d047a6c1 | ||
|
|
28ba8593b8 | ||
|
|
6981450110 | ||
|
|
703db166e7 | ||
|
|
aa74294a7d | ||
|
|
5b2666e3a2 | ||
|
|
062a7027ab | ||
|
|
a2e914f683 | ||
|
|
0404f60e6a | ||
|
|
3f5f61b716 | ||
|
|
6d7904786c | ||
|
|
856a127cd6 | ||
|
|
257c4d85c0 | ||
|
|
72161f6cf0 | ||
|
|
03b58cec0a | ||
|
|
fe14bc62c0 | ||
|
|
0b28eae7bb | ||
|
|
7581f8140a | ||
|
|
3d0a1d615d | ||
|
|
77e2cc4583 | ||
|
|
cd1b087db7 | ||
|
|
53d0c6bfc4 | ||
|
|
4d82591052 | ||
|
|
4618169036 | ||
|
|
1b14cfd0b4 | ||
|
|
0db77666c6 | ||
|
|
dd2d1feb6e | ||
|
|
9dfd89cb94 | ||
|
|
4bb84fc3ca | ||
|
|
cd6e26785a | ||
|
|
587dbb43fe | ||
|
|
eb99144509 | ||
|
|
2c4efed5de | ||
|
|
75245a467c | ||
|
|
054e85a2bf | ||
|
|
4663df7ff3 | ||
|
|
14642e3c7b | ||
|
|
6b5dade4dc | ||
|
|
1c8a67eece | ||
|
|
1913e12031 | ||
|
|
966d8f41ff | ||
|
|
6be168d1a0 | ||
|
|
d4eb259da2 | ||
|
|
21b0164fad | ||
|
|
2f4ccab166 | ||
|
|
a58239f6de | ||
|
|
b58d7bcf39 | ||
|
|
58b598b697 | ||
|
|
958e54f6a6 | ||
|
|
38176ada9d | ||
|
|
d5c769173b | ||
|
|
50751b8901 | ||
|
|
ff7f68e089 | ||
|
|
88b3cf03dd | ||
|
|
f27f7a2659 | ||
|
|
8db92947cd | ||
|
|
367a503a0f | ||
|
|
a18b18e5de | ||
|
|
af1c21abc4 | ||
|
|
c9a5cc664a | ||
|
|
023655370b | ||
|
|
9842c52853 | ||
|
|
1c2664b0c1 | ||
|
|
5bca7bfc9a | ||
|
|
043c7622bc | ||
|
|
765d8b3168 | ||
|
|
edde8a01ca | ||
|
|
a7ef497cc4 | ||
|
|
5acf1033a2 | ||
|
|
e4f22f4c4f | ||
|
|
13ca2d96b2 | ||
|
|
005083b558 | ||
|
|
0fabc15896 | ||
|
|
a7421b09c7 | ||
|
|
3d955e4edd | ||
|
|
a667c269c7 | ||
|
|
68bcebe493 | ||
|
|
739b3c3b58 | ||
|
|
dfd5d731ee | ||
|
|
36ed6594d4 | ||
|
|
271aa3d9ed | ||
|
|
ed97232598 | ||
|
|
643899c191 | ||
|
|
21fee69154 | ||
|
|
c230258542 | ||
|
|
eba040d0be | ||
|
|
a452dc3314 | ||
|
|
935904fa4e | ||
|
|
332235bfd1 | ||
|
|
3f58121b98 | ||
|
|
71e1777369 | ||
|
|
98adb72204 | ||
|
|
0e8fd814ba | ||
|
|
c0966d753d | ||
|
|
08b55a1015 | ||
|
|
042a401e71 | ||
|
|
edac02ec9f | ||
|
|
d4a7a5bad3 | ||
|
|
dcf3a5530b | ||
|
|
ab8f682f0b | ||
|
|
b2ca63ee9f | ||
|
|
8c55ac7fda | ||
|
|
d81fddc7b1 | ||
|
|
8fc44d8d9e | ||
|
|
25c30ba625 | ||
|
|
f22f70ce58 | ||
|
|
9596d5a8d0 | ||
|
|
f222a7b366 | ||
|
|
8208a80cc7 | ||
|
|
ac112c956e | ||
|
|
9127f547f9 | ||
|
|
9effc85a47 | ||
|
|
a659e5dc09 | ||
|
|
c040c180e9 | ||
|
|
1c3a3ef9ad | ||
|
|
adf7882f7d | ||
|
|
4e6171442f | ||
|
|
3c174bc6f2 | ||
|
|
c20814e6c2 | ||
|
|
bcb8dad2aa | ||
|
|
dbd6d4c57a | ||
|
|
b6300250aa | ||
|
|
47eb0bf967 | ||
|
|
0b55a11667 | ||
|
|
a527b29ac3 | ||
|
|
bc4eda926b | ||
|
|
bec4bac076 | ||
|
|
a8b773f892 | ||
|
|
474b858c0b | ||
|
|
0df72ea643 | ||
|
|
cb692f8788 | ||
|
|
461eaf42c1 | ||
|
|
fa9c7fe198 | ||
|
|
1e60e88eb2 | ||
|
|
added9d226 | ||
|
|
300f06722b | ||
|
|
3d1323cc09 | ||
|
|
e5e8f3920a | ||
|
|
a6e8bcb475 | ||
|
|
301c84d9b5 | ||
|
|
bd9e777f41 | ||
|
|
98ea1ed49c | ||
|
|
4449118783 | ||
|
|
0c22b9bbf3 | ||
|
|
7f07393623 | ||
|
|
b1697874ec | ||
|
|
5a72a4b59c | ||
|
|
b3446213df | ||
|
|
3bf0110697 | ||
|
|
c12b512316 | ||
|
|
4c40846769 | ||
|
|
ced60ddf76 | ||
|
|
ca0b169a4e | ||
|
|
e04b75e1f8 | ||
|
|
7545af8a2c | ||
|
|
26664d6d51 | ||
|
|
223c235f01 | ||
|
|
72d6132907 | ||
|
|
2bf9354c0e | ||
|
|
b99d02b7a9 | ||
|
|
d880289b06 | ||
|
|
a34aac64d0 | ||
|
|
9d9e4a5ea7 | ||
|
|
c983a25609 | ||
|
|
effff53f99 | ||
|
|
4b7c5bf128 | ||
|
|
8675bff3da | ||
|
|
fd4e177216 | ||
|
|
1c75a45a42 | ||
|
|
084fc691f5 | ||
|
|
915642aaaa | ||
|
|
8033a522cd | ||
|
|
b7cbded429 | ||
|
|
72d7f05bf1 | ||
|
|
bd4bced771 | ||
|
|
567cf9f401 | ||
|
|
dc70e15d28 | ||
|
|
fee1fcfccd | ||
|
|
027948bf3f | ||
|
|
734f45c422 | ||
|
|
c618fc69f7 | ||
|
|
ef18454e65 | ||
|
|
312fd5ee29 | ||
|
|
03edb40fc2 | ||
|
|
e2b6c63ea2 | ||
|
|
fb248961c6 | ||
|
|
0daae92c4f | ||
|
|
4ba1b144d6 | ||
|
|
ceca355493 | ||
|
|
c9f963d9f4 | ||
|
|
f31a76f788 | ||
|
|
84f3f7ddb1 | ||
|
|
3f37c5fc23 | ||
|
|
3e760a559a | ||
|
|
2a024b6039 | ||
|
|
0db09bf34f | ||
|
|
3f382bf762 | ||
|
|
cc93dae792 | ||
|
|
b9356d60ab | ||
|
|
55ade9e254 | ||
|
|
39ec76dc5a | ||
|
|
5ff5f8e0ab | ||
|
|
b52680ab47 | ||
|
|
3825664aed | ||
|
|
6448b80361 | ||
|
|
67e2144f11 | ||
|
|
e593fb277a | ||
|
|
4f8d2fcb57 | ||
|
|
91f6a26490 | ||
|
|
066e3bf153 | ||
|
|
52098302ab | ||
|
|
05f22a9ab4 | ||
|
|
7639ab5c4b | ||
|
|
33e3443545 | ||
|
|
0573ec95c1 | ||
|
|
1502ca644a | ||
|
|
06e0bc9c25 | ||
|
|
79eb0a8486 | ||
|
|
63aa63d0d6 | ||
|
|
7aeb7caefe | ||
|
|
d077c8e4dd | ||
|
|
1c078b943d | ||
|
|
6e38e2f921 | ||
|
|
81954c3d2a | ||
|
|
bf8e3be3f4 | ||
|
|
61f9b4dcc3 | ||
|
|
3844df7c23 | ||
|
|
bb548e3c5a | ||
|
|
9b0d6c2c82 | ||
|
|
de6b72651b | ||
|
|
8a4ccab3e6 | ||
|
|
61c89a7efa | ||
|
|
25024d9dda | ||
|
|
19582046ba | ||
|
|
55888e5686 | ||
|
|
c3020ee3a8 | ||
|
|
a8d14fc384 | ||
|
|
29aeaa6bb4 | ||
|
|
aaa276a7b9 | ||
|
|
2dcee2d7e8 | ||
|
|
b17d07bf2d | ||
|
|
6f14b66191 | ||
|
|
4725396c81 | ||
|
|
3b752b48f2 | ||
|
|
166a1c8bcb | ||
|
|
db95d5106e | ||
|
|
64b7436015 | ||
|
|
459dad954b | ||
|
|
b908b9a8e5 | ||
|
|
fbe113011d | ||
|
|
8abc1b8852 | ||
|
|
c8d7ec6bb5 | ||
|
|
3e39dbb4f8 | ||
|
|
a621f4c5a0 | ||
|
|
437f096d9d | ||
|
|
1e205840dc | ||
|
|
2b17419e60 | ||
|
|
2da4c38773 | ||
|
|
a356d6def4 | ||
|
|
cff9284453 | ||
|
|
038f0ecd5d | ||
|
|
281c7b7e3f | ||
|
|
05be509e6e | ||
|
|
9b4ae3df28 | ||
|
|
a2374ca679 | ||
|
|
26c6d1e4d1 | ||
|
|
3a9be73875 | ||
|
|
44cdf82631 | ||
|
|
2a4affc3a8 | ||
|
|
76f9cb5977 | ||
|
|
f43697c881 | ||
|
|
8e6d00a57f | ||
|
|
20e03902c8 | ||
|
|
4777f8a221 | ||
|
|
4cd1d024e9 | ||
|
|
a2a76e7624 | ||
|
|
b566fbe57f | ||
|
|
238ec02754 | ||
|
|
1c37a1689e | ||
|
|
8572a0f5f4 | ||
|
|
9370b6751d | ||
|
|
d96e5bddd0 | ||
|
|
d17b5ab45d | ||
|
|
77fa5af592 | ||
|
|
4d30444914 | ||
|
|
e8dab27653 | ||
|
|
239f5f6440 | ||
|
|
d7d68fdd83 | ||
|
|
5ab159521d | ||
|
|
716cbcd705 | ||
|
|
b66c137aa7 | ||
|
|
7fb7ef5c3d | ||
|
|
109dd853a3 | ||
|
|
9ff0410755 | ||
|
|
9580c8c75d | ||
|
|
bbb284e378 | ||
|
|
237b85f420 | ||
|
|
bc4f2b9168 | ||
|
|
bb1692cbfb | ||
|
|
7336a87514 | ||
|
|
13a02e4871 | ||
|
|
5f87e9099e | ||
|
|
c1d04c28c1 | ||
|
|
d1bd44d2d5 | ||
|
|
7ae82e1d5d | ||
|
|
832b73e6e8 | ||
|
|
15f7529a1c | ||
|
|
9e3513557f | ||
|
|
e57fbc9b80 | ||
|
|
4734d4a132 | ||
|
|
790adb07a9 | ||
|
|
a43e9813ad | ||
|
|
621a09a1c5 | ||
|
|
79ba2f158a | ||
|
|
e17385a332 | ||
|
|
1799fcd0e4 | ||
|
|
cfebdf5cd4 | ||
|
|
de34d170d2 | ||
|
|
e5a18194f4 | ||
|
|
e4dfe5ccc7 | ||
|
|
89e9f98a95 | ||
|
|
588fed2aa9 | ||
|
|
8ecf8929fb | ||
|
|
aa4dfb75ea | ||
|
|
9a0e7e168b | ||
|
|
570410cd1a | ||
|
|
fe7b3c4ab7 | ||
|
|
c17cfde236 | ||
|
|
4d9e76e66a | ||
|
|
33dfd9afb3 | ||
|
|
e51b3a2f66 | ||
|
|
3fecf4c713 | ||
|
|
fae11020a5 | ||
|
|
47dd0bd244 | ||
|
|
6e036e4c75 | ||
|
|
57405c1a99 | ||
|
|
9f6753de0f | ||
|
|
6e86483185 | ||
|
|
7eea638b92 | ||
|
|
697e67c85e | ||
|
|
dcd8a8eff4 | ||
|
|
766500dfc3 | ||
|
|
faba8e4ff7 | ||
|
|
280a7f80e5 | ||
|
|
b527430ec7 | ||
|
|
e548ab57fe | ||
|
|
66bd015ee5 | ||
|
|
77b4f21450 | ||
|
|
9d6e49236c | ||
|
|
27b68f043b | ||
|
|
1af21a6cac | ||
|
|
05c450147c | ||
|
|
605ce25f58 | ||
|
|
237452c8c6 | ||
|
|
11c91a0dde | ||
|
|
e68a7af4d3 | ||
|
|
ab77fa3544 | ||
|
|
f0fee7e193 | ||
|
|
43b7e96905 | ||
|
|
7dcc80fde8 | ||
|
|
a042316d6d |
No files matched your search
@@ -0,0 +1,46 @@
|
||||
---
|
||||
paths: ["internal/backup/**", "internal/pbs/**", "internal/pbsdr/**", "internal/dr/**"]
|
||||
---
|
||||
|
||||
# Backup, PBS and DR
|
||||
|
||||
`internal/backup/` is the vzdump runner, restore-test scheduler and report store. `internal/pbs/` is
|
||||
the fingerprint-pinned PBS-API client plus the verify maintenance loop. `internal/pbsdr/` and
|
||||
`internal/dr/` carry the DR tier and recipe halves.
|
||||
|
||||
## The three PBS laws
|
||||
|
||||
1. **Set-only.** `pvesm remove` **DELETES the encryption key**. Re-apply configuration; never remove
|
||||
and re-add a PBS storage to change it.
|
||||
2. **Secret on stdin.** A token secret is passed on stdin, never as an argv the process table shows.
|
||||
3. **Verify the pin BEFORE consuming the secret.** A fingerprint check after the secret has been sent
|
||||
protects nothing.
|
||||
|
||||
## Verify is server-side, and its default skips the work
|
||||
|
||||
The agent drives verification **remotely** via the PBS API; `proxmox-backup-client` has **no** verify
|
||||
subcommand. `POST .../verify` defaults to **`ignore-verified=true`, which SKIPS already-verified
|
||||
snapshots** — send `ignore-verified=false` to actually re-read and detect corruption. A verify that
|
||||
skipped everything reports success.
|
||||
|
||||
## Presence is not success
|
||||
|
||||
A timestamp recording an **attempt** is not evidence of a **result**. Where a status field travels
|
||||
beside a timestamp, the verdict must consult **both** — or the timestamp must record only successes.
|
||||
Ask of any timestamp: *what exactly must have happened for this to be set?* If the answer is "we
|
||||
tried", it cannot answer "did it work".
|
||||
|
||||
**Corollary:** when a verdict changes which field it counts from, the alarm text changes with it.
|
||||
Leaving a message reading `last run 8h ago` while alarming on a six-day-old **success** turns a true
|
||||
alarm into one the operator dismisses.
|
||||
|
||||
## Prune is server-side now
|
||||
|
||||
`DatastoreBackup` carries **no** `Datastore.Prune`. Boxes set `keep_last: 0` and the off-site endpoint
|
||||
runs the prune jobs. **Box tokens stay write-only — never widen that grant** (R-89).
|
||||
|
||||
<!--
|
||||
The ignore-verified default is the sharpest instance of the "absent log line" class in this repo: a
|
||||
verify that silently skipped every snapshot completes fast, exits clean, and reports the same shape
|
||||
as one that read every byte.
|
||||
-->
|
||||
@@ -0,0 +1,27 @@
|
||||
---
|
||||
paths: ["internal/capability/**", "internal/storage/**", "internal/localapi/**", "internal/hub/**", "internal/guesthook/**"]
|
||||
---
|
||||
|
||||
# A health check issues no block I/O
|
||||
|
||||
No `statfs`, no `getdents`, no read, write or `fsync` — **not even behind a timeout**.
|
||||
|
||||
A probe that touches a wedged device enters uninterruptible sleep, survives `SIGKILL`, and cannot be
|
||||
recovered until the device returns or the host reboots — so `systemctl restart` hangs too. A timeout
|
||||
protects the caller's control flow and nothing else: the blocked thread remains.
|
||||
|
||||
**Liveness is decided from `/proc` and the kernel's own state**, never by reading or writing the
|
||||
filesystem.
|
||||
|
||||
<!--
|
||||
Measured, R-117 spike §6.3 (felhom.eu/documentation/audits/SPIKE-r117-bind-liveness-2026-07-30.md):
|
||||
a probe stayed in D state 3m50s after kill -9; a buffered write with no fsync blocked too (O_CREAT
|
||||
needs journal access); and statfs/getdents returned HEALTHY on a namespace that EIOs every byte —
|
||||
fast, and wrong.
|
||||
|
||||
This rule used to be duplicated verbatim in felhom-agent/CLAUDE.md with a note explaining that
|
||||
felhom.eu/CLAUDE.md "does not load in an agent-only session". That reasoning was correct before
|
||||
path-scoped rules existed. Deliberate scoped copies now live in felhom.eu/.claude/rules/hub.md and
|
||||
felhom-controller/.claude/rules/gates.md (hub.md's comment names them); none is the single source. This
|
||||
file is the copy that loads exactly where agent health checks are written. (2026-08-06; corrected 2026-10-06)
|
||||
-->
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
paths: ["internal/localapi/**", "internal/authz/**", "internal/guesthook/**"]
|
||||
---
|
||||
|
||||
# Local API, authz and guest hooks — the per-guest blast radius
|
||||
|
||||
`internal/localapi/` is the narrow per-guest local API: token store, disks/format, guest binds,
|
||||
controller swap, stale-lock recovery, pinned self-signed leaf. `internal/authz/` is the operator
|
||||
signed-op verifier (SSHSIG) plus the durable nonce store. `internal/guesthook/` installs the
|
||||
pre-start self-heal hookscript.
|
||||
|
||||
> **Overlap note:** `health-checks.md` also matches `internal/localapi/**` and
|
||||
> `internal/guesthook/**`. That is deliberate — both rules apply there and both load. Neither
|
||||
> supersedes the other.
|
||||
|
||||
## Scoping is the whole security property
|
||||
|
||||
This API is reachable **from inside a customer guest**. Every route must be scoped to the guest that
|
||||
called it — a route that can name another guest's id has escaped its blast radius. Fail **safe to
|
||||
protected**: an unrecognised or unresolvable caller gets less access, never more.
|
||||
|
||||
## Replay protection must survive a restart
|
||||
|
||||
**`authz.MemoryNonceStore` on a real host is a defect** — replay protection dies on restart. Use
|
||||
`authz.FileNonceStore`. The memory store exists for tests.
|
||||
|
||||
## The token is a hash on disk, plaintext only at mint
|
||||
|
||||
The store keeps **hashes**. The plaintext token exists in exactly one place, `bootstrap.json` on the
|
||||
PVE host — so a "read the token" step means reading that file, and a lost token is re-minted, never
|
||||
recovered.
|
||||
|
||||
## Binds can brick guest boot
|
||||
|
||||
| Do not | Because | Use |
|
||||
|---|---|---|
|
||||
| `GuestBinder.AttachBind`/`DetachBind` (per-drive `pct set -mpN`) | legacy model; a missing bind source can **brick guest boot** (C1) | `AttachDrive`/`DetachDrive` (intermediary model) |
|
||||
| `isHostMountpoint` to reconcile bind state | a boolean cannot converge stacked double-binds (the `/mnt` doubling bug) | `countHostMounts` normalization inside `AttachDrive` |
|
||||
|
||||
<!--
|
||||
Why fail-safe-to-protected rather than fail-closed: this API also carries the recovery paths. A hard
|
||||
refusal on an unresolvable caller would make a half-broken guest unrecoverable through the very
|
||||
interface built to recover it. Less access, never none.
|
||||
-->
|
||||
@@ -0,0 +1,44 @@
|
||||
---
|
||||
paths: ["internal/proxmox/**", "internal/reconcile/**", "internal/signedjobs/**"]
|
||||
---
|
||||
|
||||
# Proxmox — the API contract, and how destructive work is gated
|
||||
|
||||
`internal/proxmox/` is the API-first `Client` plus the fenced root-CLI `Privileged`.
|
||||
`internal/reconcile/` is the reconcile engine, reversibility gate, op journal and crash recovery.
|
||||
`internal/signedjobs/` holds the operator-signed destructive executors (wipe, decommission).
|
||||
|
||||
## A 200 on the POST is not success
|
||||
|
||||
**Every mutating op is async**: it returns a **UPID**, and `WaitTask` must assert
|
||||
`exitstatus == "OK"`. Authorization can fail at *task execution* long after the HTTP call returned
|
||||
200. Treating the POST's status as the result is how a failed destroy reads as a successful one.
|
||||
|
||||
## The privsep token gotcha
|
||||
|
||||
A `--privsep 1` token's rights are the **intersection** of the backing user's permissions **and** the
|
||||
token's own ACLs. The role must be granted on **both** or every call 403s. The same intersection rule
|
||||
bites on PBS (`token ∩ user`).
|
||||
|
||||
## TLS
|
||||
|
||||
**SHA-256 leaf-cert pinning** against the self-signed host cert. **No insecure default**, ever. The
|
||||
pin is the raw leaf-DER sha — the SAN is never checked, so a cert rotation changes the pin and the
|
||||
agent must be re-pinned.
|
||||
|
||||
## The destructive path — never the direct call
|
||||
|
||||
| Do not | Because | Use |
|
||||
|---|---|---|
|
||||
| `Client.DestroyLXC` / `Vzdump` / `SetConfig` ad-hoc | skips classification, signature, per-guest serialization, crash recovery | `reconcile.Engine` paths / `RunSignedJob`; queue via `Queue.Submit` |
|
||||
| add a method to `proxmox.Privileged` | breaks the 3-exception root-CLI fence (`routing_test.go`) | `proxmox.Runner` + a new sudoers `Cmnd_Alias` + `validate.go`-style checks |
|
||||
| treat `ListLXC` output as "guests we own" | audit A1 — pre-v0.62.0 the stale-lock reaper did exactly this, contained only by the pool-scoped token | intersect with `Client.Pool` membership (`staleLockController.Guests()`); **fail safe on read failure** |
|
||||
|
||||
Full trap table: `REUSE.md` §3. Every guest joins the `felhom` pool — `VM.Audit` comes from the
|
||||
`/pool` grant, not from a per-guest ACL.
|
||||
|
||||
<!--
|
||||
The fence is not stylistic. It is what makes this component auditable: two types, one of which can
|
||||
only speak HTTP and one of which can only shell out, with a test asserting neither crosses. A single
|
||||
convenience method on Privileged that also makes an HTTP call would end that property silently.
|
||||
-->
|
||||
@@ -0,0 +1,49 @@
|
||||
---
|
||||
paths: ["internal/storage/**", "internal/escrow/**"]
|
||||
---
|
||||
|
||||
# Storage and escrow — format safety and zero-knowledge recovery
|
||||
|
||||
`internal/storage/` is the storage observer, durable IDs, role/claim classifiers, `SudoHostOps` and
|
||||
the watchdog. `internal/escrow/` is the PBS-key escrow with its zero-knowledge recovery code.
|
||||
|
||||
> **Overlap note:** `health-checks.md` also matches `internal/storage/**`. Deliberate — both rules
|
||||
> apply there and both load.
|
||||
|
||||
## Never format the device you inspected
|
||||
|
||||
**AGENT-001 is a TOCTOU:** acting on the caller's `req.Device` (or any remembered `/dev` path) after
|
||||
inspection lets `/dev` re-enumeration retarget the node to a **different physical disk**. Format the
|
||||
**re-resolved** device — `Server.reresolveWipe` / `reresolveBlank`.
|
||||
|
||||
**Never exec raw `mkfs.*`** (including `Binaries.MkfsExt4`/`MkfsXfs`): sudoers no longer allowlists
|
||||
raw mkfs, and going direct bypasses the claim filter and the wrapper's re-checks. Use
|
||||
`SudoHostOps.Format`, which routes through `felhom-mkfs-guarded`.
|
||||
|
||||
## The two durable-ID schemes refuse each other
|
||||
|
||||
They are not interchangeable, and each returns a `binding_mismatch` for the other's scheme:
|
||||
|
||||
| Purpose | Scheme | Resolver |
|
||||
|---|---|---|
|
||||
| wipe confirmation | `byid:` / `byuuid:` | `ResolveDurableDevice`, `DiskInfo.WipeDurableID` |
|
||||
| enrolled-storage remount | `uuid:` | `ResolveStorageDevice` |
|
||||
|
||||
Using `DiskInfo.DurableID` (a `uuid:`) as a wipe-confirmation id is F20-BUG2.
|
||||
|
||||
## Drive data is never taken by force
|
||||
|
||||
Plain `umount` only — **never `-l`, never `-f`**, and never any format operation under
|
||||
`/mnt/felhom-drives`.
|
||||
|
||||
## Escrow is zero-knowledge, and a fetch failure is not a wrong code
|
||||
|
||||
The server holds no client key; a no-key restore fails with `missing key`. **A fetch failure must
|
||||
never be reported as a wrong recovery code** — that told a customer their correct code was bad, in
|
||||
hundredths of a second, when checking a code actually takes about one. Distinguish "we could not
|
||||
reach the store" from "the code did not match", always.
|
||||
|
||||
<!--
|
||||
The escrow recovery-code "flake" was a REAL defect, not a flake. "Known flake, re-run" needs evidence
|
||||
before it is said out loud — that phrase cost this project a real finding once.
|
||||
-->
|
||||
@@ -0,0 +1,101 @@
|
||||
---
|
||||
unconditional: true
|
||||
---
|
||||
# Unprompted work — rules for any session without a task file
|
||||
|
||||
> Goal sessions, nightly sessions, "work the register" sessions. **A session that starts from
|
||||
> `/goal` or a standing brief inherits these rules exactly as it inherits the gates.** They are the
|
||||
> part of `PROMPT-TEMPLATE.md` that a task file used to carry and a goal does not. Same wording lives
|
||||
> in `felhom.eu`, `felhom-controller`, `felhom-agent` and `app-catalog-felhom.eu` `.claude/rules/`, and in the workspace
|
||||
> root's unversioned `.claude/rules/`; change all five or none.
|
||||
|
||||
## 1. What you may pick up on your own
|
||||
|
||||
- A register row **you or another CC session filed**, with owner CC, at P3 or a bounded P2, that
|
||||
needs **no operator decision**, touches **no customer data by design**, and introduces **no
|
||||
mechanism nobody has measured**. Smallest first.
|
||||
- A defect you find while exercising the product, filed as a row **before** you fix it — **unless it is small**:
|
||||
a small finding is fixed in the session and never filed (the size rule, `OPEN-ITEMS.md` „How a row is filed").
|
||||
- Hygiene: register compression, stale citations, rows with no owner, documents that contradict
|
||||
live source.
|
||||
|
||||
**Not yours, ever, without a task file or an operator word:** money; anything that changes risk to
|
||||
customer data; anything that changes a promise the product makes to a customer; anything that
|
||||
reverses a documented design decision (`documentation/architecture/` — a design decision is not a
|
||||
defect, R-370); anything on DooPlex or ep0; baking or vouching a golden; promoting a
|
||||
catalog version; a new external dependency; **a hub image build or hub deploy in a session the operator does not
|
||||
attend** (operator ruling 2026-10-07, `09` §3 decision 162).
|
||||
|
||||
## 2. When you may decide instead of ask (operator grant, 2026-09-14)
|
||||
|
||||
You may take a decision yourself when **all** of these hold: the architecture folder and the register
|
||||
give a clear direction; your choice follows that direction; it is reversible without customer-data
|
||||
risk; and you can write it in the `09-update-architecture.md` §3 shape — one answerable sentence, the
|
||||
options, what each costs, why this one. **Then record it** as a dated decision in `CONTEXT.md` and
|
||||
the owning architecture document, tagged *decided by CC unattended — operator may reverse*, and put
|
||||
it **first** in the morning note. A decision you cannot write in that shape is one you do not take.
|
||||
|
||||
## 3. The discipline a task file used to carry
|
||||
|
||||
1. **Baselines first.** Read each repo's `main` hash and version from live source before touching it.
|
||||
2. **Read the architecture document for the area, and name it** in the report, before any claim.
|
||||
3. **Red-proof every correctness fix.** A test never seen failing has not been shown to test anything.
|
||||
4. **Live-validate on a Tier-0 box** through the endpoints the UI invokes. `demo-hp` is `ssh hp`.
|
||||
Throwaway apps only; the standing apps and `bentopdf` stay.
|
||||
5. **Evidence off the machine at the end of each phase**, before any revert (R-320).
|
||||
6. **One release per repo per session**, with a CHANGELOG entry (controller: with its `MinAgent`
|
||||
line) and REPORT overwritten. **A controller release needs no golden** (operator ruling 2026-10-15
|
||||
11:26, `09` §3 decision 250; decision 7 stands): push the image only when it is ready for EVERY box (the
|
||||
household's update button offers the highest tag); raise the floor, MinAgent declared, only after it ran
|
||||
on Tester 1, demo-hp and demo-felhom through the normal update path and the box walk's checks are green
|
||||
on all three — not at night unless the night brief says so. **Agent releases still stop for the
|
||||
operator's vouch.** The golden is baked weekly and vouched by the operator; CC never vouches. **No `--no-verify`.**
|
||||
**A version number is pushed once** — `felhom-controller/controller/build.sh` now refuses a second
|
||||
push of a tag that exists, and a registry it cannot reach is a refusal too. This is a FACT about
|
||||
what the build does, not a new rule: a fix for something already pushed is the next patch version,
|
||||
and a box on a bad image is moved by the normal update path, never by re-tagging. Earned on
|
||||
2026-10-15, when 0.315.0 was pushed three times from three different commits and a cache kept
|
||||
serving the first one (`09-update-architecture.md` §0).
|
||||
7. **An enumerated gap becomes a row in the same session — or, if it is small, is fixed in it** (the size rule).
|
||||
Prose is not a record.
|
||||
8. **Hungarian text is searched with ASCII fragments**, with a positive and a negative control.
|
||||
9. **Never leave a half-state.** If time runs out, revert to clean and say what was reverted.
|
||||
10. **Teardown, three layers, stated** — machine, host, hub — or "provisioned nothing".
|
||||
11. **Every helper prompt carries the brief's fences in full** (operator ruling 2026-10-07). A helper session (a
|
||||
subagent, a fork, a workflow agent) gets the brief's fence list word for word — every protected machine, every
|
||||
„no", every delivery and Docker limit — not a summary and not „the usual fences". Earned on 2026-10-06 night: two
|
||||
helpers whose prompts carried only part of the fences ran `docker volume prune` on the bench and a Docker-using
|
||||
gate on DooPlex.
|
||||
|
||||
## 4. The morning note
|
||||
|
||||
One screen, plain language, in this order: **decisions you took** (§2) first; what you exercised;
|
||||
what broke and whether you fixed it; rows opened and closed with the register size before and after;
|
||||
what needs the operator, each with what happens if they do nothing. No file paths, no function
|
||||
names, no row numbers as the subject of a sentence.
|
||||
|
||||
## 5. Instruction files
|
||||
|
||||
**Instruction files (`CLAUDE.md`, `.claude/rules/*`) are kept true by the session that finds them wrong**
|
||||
(operator ruling 2026-10-06, `09` §3 decision 150). A session MAY, without asking: correct a stale fact (a command, a
|
||||
count, a version, a path, a description of what a gate does), add a fact it proved, and remove a reference to something
|
||||
that no longer exists. Each edit is named in the report (file, line, before, after, why). A session MAY NOT, without the
|
||||
operator's word: loosen a safety rule, a fence, a „never", a protected machine, a secret rule, or a review step; or
|
||||
remove a rule. When in doubt, it is a rule change, and it goes to the operator. If Claude Code's own permission check
|
||||
asks before such an edit, wait for the operator's click; if it refuses, record that and file the exact line.
|
||||
|
||||
## 6. The system poster stays true
|
||||
|
||||
**A session that changes a fact listed in `architecture/felhom-system-poster.facts.md` (a machine, a
|
||||
role, a traffic path, a backup tier, a time, a retention, a key, a known gap) updates that file in
|
||||
the same commit.** If the change is text only, it also edits the matching text in
|
||||
`felhom-system-poster.html`. If it needs a new drawing, it adds the line
|
||||
**"System poster needs a refresh: <what changed>"** to `STATUS.md`'s "waiting on the operator" list.
|
||||
**The session report names which of the three it did.**
|
||||
|
||||
Why the three-way split: the poster is a drawing made in Claude Design, and **nothing in this
|
||||
repository renders it** — unlike `where-felhom-stands.html`, which has `render_stands.py`. So the
|
||||
facts file is the source of truth and the drawing trails it. `scripts/poster_facts_gate.py` warns
|
||||
when the facts file has a newer commit than the poster; it **never fails a push**, because a refresh
|
||||
needs the operator and another tool, and a gate nobody can clear is a gate people learn to route
|
||||
around.
|
||||
@@ -0,0 +1,156 @@
|
||||
# gates — re-run this repo's gate entry point on every push, on a machine that does not care who
|
||||
# pushed or what they typed.
|
||||
#
|
||||
# *** THIS REPORTS. IT CANNOT REFUSE. ***
|
||||
#
|
||||
# felhom repos push straight to `main` with no pull request, so there is no merge for a status
|
||||
# check to stand at. The refusing half is `.githooks/pre-push`, which is local to a clone and which
|
||||
# `git push --no-verify` skips; this half is what notices when that happened. Neither half is the
|
||||
# whole thing, and both are named in felhom.eu documentation/backlog/OPEN-ITEMS.md R-168.
|
||||
#
|
||||
# NO `uses:` STEP ANYWHERE, deliberately: JavaScript actions need a node runtime in the runner, and
|
||||
# the runner is a host-mode container with python3 and git and nothing else (see
|
||||
# homelab-manifests/gitea-system/act-runner.yaml for why it is not privileged). Probe P3 measured
|
||||
# that a plain `git fetch` of the pushed SHA from the in-cluster Gitea service is enough.
|
||||
#
|
||||
# A failing run must reach a person — a detector nobody hears is the defect R-29 filed, rebuilt one
|
||||
# layer up. That is the last step, and it runs ONLY on failure.
|
||||
name: gates
|
||||
# R-1129: `workflow_dispatch` is the manual PROOF that the failure mail can be sent. A push run is
|
||||
# unchanged by it: the proof step below is skipped on a push, and `force_alarm` is then empty.
|
||||
on:
|
||||
push:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_alarm:
|
||||
description: "Fail this run on purpose so the alarm step sends its real mail (a proof, not a gate result)"
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
gates:
|
||||
runs-on: felhom-gates
|
||||
steps:
|
||||
- name: Fetch the pushed commit and the sibling clone it needs
|
||||
# This repo's entry point invokes a SHARED checker that lives in the felhom.eu clone next
|
||||
# door and is deliberately never copied here — so CI has to reproduce the workspace's
|
||||
# sibling layout or the gate fails closed with "gate is MISSING". The sibling is also
|
||||
# needed for CONTENT: this repo's REUSE.md cites a path that lives in the hub.
|
||||
run: |
|
||||
# Shallow, and pinned to the exact SHA that was pushed — not to the branch tip,
|
||||
# which can move under us if two pushes race.
|
||||
mkdir -p ws/felhom-agent
|
||||
cd ws/felhom-agent
|
||||
git init -q .
|
||||
git remote add origin http://gitea.gitea-system.svc.cluster.local:3000/admin/felhom-agent.git
|
||||
git fetch -q --depth 1 origin "$GITHUB_SHA"
|
||||
git checkout -q FETCH_HEAD
|
||||
echo "checked out $(git rev-parse HEAD)"
|
||||
cd .. && git clone -q --depth 1 http://gitea.gitea-system.svc.cluster.local:3000/admin/felhom.eu.git felhom.eu
|
||||
echo "sibling felhom.eu present at $(cd felhom.eu && git rev-parse --short HEAD)"
|
||||
|
||||
- name: Run the gate entry point
|
||||
# The ONLY thing CI runs. No go build, no go test, no linting, no deploy. The
|
||||
# exit code IS the result: no `|| true`, no pipe that could swallow it.
|
||||
#
|
||||
# THE FULL SET, NOT `--fast` (R-115, 2026-08-03). `--fast` means "no network and no
|
||||
# container runtime" and exists for `.githooks/pre-push`, where a push must not fail
|
||||
# because Gitea blinked or because someone is on a train. CI is the opposite machine: it
|
||||
# has the network, it is not in anyone's way, and it is the half that emails. The
|
||||
# published-versions gate — the R-115 mechanism, which asks Gitea whether a released
|
||||
# version can actually be downloaded — is network-bound and therefore runs ONLY here.
|
||||
# Leaving `--fast` in place would have registered that gate and never run it, which is the
|
||||
# built-but-never-wired failure this project has shipped four times.
|
||||
env:
|
||||
# In-cluster, so the check does not depend on public DNS or the ingress TLS chain.
|
||||
GITEA_BASE: http://gitea.gitea-system.svc.cluster.local:3000
|
||||
# R-986: the security-scan gate fetches the PINNED gitleaks here (sha256-checked); no Go, so govulncheck says
|
||||
# NOT CHECKED in CI, out loud. Warn-first either way: the scan lists, it never fails this job.
|
||||
FELHOM_SCAN_FETCH: "1"
|
||||
run: cd ws/felhom-agent && python3 scripts/agent_gates.py
|
||||
|
||||
- name: Manual proof - alarm key present, optional forced failure (dispatch only, R-1129)
|
||||
# Runs ONLY on a manual dispatch, never on a push. It prints the key's LENGTH, never the key,
|
||||
# so "CI can see the secret" is an observable rather than an assumption. With
|
||||
# force_alarm=true it then fails on purpose so the alarm step below sends its real mail;
|
||||
# the commit itself stays green, because its push run skips this step.
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
env:
|
||||
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
||||
FORCE_ALARM: ${{ github.event.inputs.force_alarm }}
|
||||
run: |
|
||||
python3 -c 'import os;print("RESEND_API_KEY length: %d" % len(os.environ.get("RESEND_API_KEY","")))'
|
||||
if [ "$FORCE_ALARM" = "true" ]; then
|
||||
echo "force_alarm=true - failing on purpose so the alarm step runs (R-1129 proof)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Alarm on failure
|
||||
# THE POINT OF THE WHOLE THING. Probe P5 measured that a failed run produces NO mail, NO
|
||||
# notification row and NO log line from Gitea itself — a red tick in a web UI nobody watches
|
||||
# is exactly the shape R-29 filed against. So the run sends its own alarm, on the project's
|
||||
# existing transactional path (Resend, the same one the hub uses), and prints the provider's
|
||||
# accepted id so "a message left the machine" is an observable, not an assumption.
|
||||
#
|
||||
# Pure python3 and urllib, NOT curl: the runner image carries python3 and git and nothing
|
||||
# else on purpose, and the first version of this step died on `curl: command not found`.
|
||||
# Reaching for a bigger image to send one HTTP request would have been the wrong trade.
|
||||
if: failure()
|
||||
env:
|
||||
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
||||
# Empty on a push, so a push run's mail is exactly what it was before R-1129.
|
||||
FORCE_ALARM: ${{ github.event.inputs.force_alarm }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import json, os, sys, urllib.request, urllib.error
|
||||
|
||||
key = os.environ.get("RESEND_API_KEY", "")
|
||||
if not key:
|
||||
sys.exit("ALARM FAILED: RESEND_API_KEY is empty — the alarm cannot be sent, and a "
|
||||
"silent alarm is worse than none. Set the user-level Actions secret.")
|
||||
|
||||
repo = os.environ.get("GITHUB_REPOSITORY", "?")
|
||||
sha = os.environ.get("GITHUB_SHA", "?")
|
||||
# R-1135: the link is for a PERSON, so it is the public address and the run ID.
|
||||
# NOT GITHUB_SERVER_URL: the runner registers at the in-cluster address, so that variable
|
||||
# reads http://gitea.gitea-system.svc.cluster.local:3000, which nobody outside the cluster
|
||||
# can open (forced-test mail, job 1997). NOT GITHUB_RUN_NUMBER: Gitea 1.26 reads
|
||||
# /actions/runs/<n> as a run id first, so a run number that is also an older run's id
|
||||
# opens THAT run (measured 2026-10-18: felhom.eu /actions/runs/1116 opened the run whose
|
||||
# id is 1116 = run number 613). The link is printed too, so the job log carries it.
|
||||
run = os.environ.get("GITHUB_RUN_ID", "?")
|
||||
srv = "https://gitea.dooplex.hu"
|
||||
print("ALARM-LINK %s/%s/actions/runs/%s" % (srv, repo, run))
|
||||
|
||||
body = json.dumps({
|
||||
"from": "Felhom CI <monitoring@felhom.eu>",
|
||||
"to": ["admin@felhom.eu"],
|
||||
"subject": ("[felhom CI] FORCED TEST, not a real failure - gates FAILED in %s"
|
||||
if os.environ.get("FORCE_ALARM") == "true"
|
||||
else "[felhom CI] gates FAILED in %s") % repo,
|
||||
"text": (
|
||||
"The gate entry point exited non-zero.\n\n"
|
||||
"Repository : %s\n"
|
||||
"Commit : %s\n"
|
||||
"Run : %s/%s/actions/runs/%s\n\n"
|
||||
"The failing gate names itself in the run log.\n\n"
|
||||
"If the local pre-push hook was GREEN for this commit, then CI and the hook\n"
|
||||
"disagree - that is a finding about the gates themselves, not about CI, and it\n"
|
||||
"outranks whatever the push was for.\n"
|
||||
) % (repo, sha, srv, repo, run),
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
"https://api.resend.com/emails", data=body, method="POST",
|
||||
headers={"Authorization": "Bearer %s" % key,
|
||||
"Content-Type": "application/json",
|
||||
# Cloudflare fronts api.resend.com and BLOCKS the default
|
||||
# "Python-urllib/3.x" agent with its own 403 (error 1010) — which looks
|
||||
# exactly like an auth failure and is not one. Measured 2026-08-02.
|
||||
"User-Agent": "felhom-ci/1.0"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
print("RESEND-ACCEPTED id=%s" % json.load(r)["id"])
|
||||
except urllib.error.HTTPError as e:
|
||||
sys.exit("ALARM FAILED: Resend returned HTTP %s: %s" % (e.code, e.read().decode()[:300]))
|
||||
PY
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/bin/sh
|
||||
# pre-push — refuse a push that carries a broken gate. (2026-08-02, R-29 leg (b) first half.)
|
||||
#
|
||||
# Runs this repo's ONE gate entry point in --fast mode: only checks that touch no network and no
|
||||
# container runtime, so a push stays a push and never pulls images or starts containers. The slow
|
||||
# gates stay deliberate periodic runs; a hook that takes minutes gets bypassed within a week and
|
||||
# the bypass becomes the habit.
|
||||
#
|
||||
# BOTH LINES BELOW ARE DELIBERATE. An absent log line is not evidence a hook ran — a silent pass is
|
||||
# equally consistent with "gates green" and "hook never fired", so a passing push says so out loud.
|
||||
#
|
||||
# HONEST LIMITS, stated so this is not mistaken for enforcement it cannot provide:
|
||||
# * per-clone — core.hooksPath is local config and a clone does not carry it. Arm a clone once:
|
||||
# git config core.hooksPath .githooks
|
||||
# Any manual entry-point run WARNS when the clone is unarmed.
|
||||
# * skippable — `git push --no-verify` bypasses this entirely. That is on purpose: an escape
|
||||
# hatch that cannot be reached is one that gets removed the first time it is
|
||||
# inconvenient. USING IT MUST BE STATED IN THE SESSION REPORT.
|
||||
# The half that is neither per-clone nor skippable is CI — felhom.eu OPEN-ITEMS.md R-168.
|
||||
#
|
||||
# Measured 2026-08-02 (git 2.47.3): a relative core.hooksPath resolves correctly and the hook's cwd
|
||||
# is the repo root whether `git push` is issued from the root or from any subdirectory. The
|
||||
# explicit rev-parse below does not depend on that.
|
||||
set -u
|
||||
|
||||
root=$(git rev-parse --show-toplevel 2>/dev/null) || {
|
||||
echo "pre-push: FAIL - cannot resolve the repo root (git rev-parse --show-toplevel)." >&2
|
||||
exit 1
|
||||
}
|
||||
cd "$root" || exit 1
|
||||
|
||||
# ── WORKSPACE-ROOT ASSERTION (2026-08-05, R-204 rider) ───────────────────────────────────────────
|
||||
# Refuse a push from a clone outside the felhom workspace.
|
||||
#
|
||||
# WHY THIS IS A HOOK AND NOT A LINE IN A DOCUMENT: the workspace root is ALREADY written down, in
|
||||
# documentation/runbooks/workspace-CLAUDE.md and in the workspace-root CLAUDE.md ("stay inside it"),
|
||||
# and work drifted into a home directory anyway. A rule that has failed once as a reminder is not
|
||||
# fixed by writing it down again — it has to be asserted where it can bite.
|
||||
#
|
||||
# A PUSH IS THE RIGHT TRIGGER, deliberately: throwaway clones under /tmp for probes and red-proofs
|
||||
# never push, so nothing legitimate breaks. Reads and builds elsewhere stay unaffected.
|
||||
#
|
||||
# Symlinks are resolved on BOTH sides before comparison, so a symlinked path neither falsely passes
|
||||
# nor falsely fails. If the workspace root does not exist on this machine the check is SKIPPED, not
|
||||
# failed — this hook must not brick a legitimate clone on a different host.
|
||||
#
|
||||
# The only bypass is the documented `git push --no-verify`, whose use is already reportable.
|
||||
FELHOM_WORKSPACE_ROOT=/mnt/5_hdd/felhom.eu
|
||||
if [ -d "$FELHOM_WORKSPACE_ROOT" ]; then
|
||||
ws_real=$(cd "$FELHOM_WORKSPACE_ROOT" 2>/dev/null && pwd -P) || ws_real=""
|
||||
root_real=$(pwd -P) || root_real=""
|
||||
if [ -n "$ws_real" ] && [ -n "$root_real" ]; then
|
||||
case "$root_real/" in
|
||||
"$ws_real"/*) : ;; # inside the workspace — proceed
|
||||
*)
|
||||
echo "pre-push: PUSH REFUSED - this clone is OUTSIDE the felhom workspace." >&2
|
||||
echo " clone: $root_real" >&2
|
||||
echo " expected: under $ws_real (repos live in $ws_real/git/<repo>)" >&2
|
||||
echo " Work in the workspace clone, or bypass with 'git push --no-verify'" >&2
|
||||
echo " and state that you did in the session report." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
echo "pre-push: FAIL - python3 not found, so the gates CANNOT run. This is a failure, never a" >&2
|
||||
echo " pass by default. Install python3, or push with --no-verify and say so." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "pre-push [felhom-agent]: running scripts/agent_gates.py --fast ..."
|
||||
python3 "scripts/agent_gates.py" --fast
|
||||
rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "pre-push [felhom-agent]: PUSH REFUSED - gates exited $rc. Fix the finding above, or bypass with" >&2
|
||||
echo " 'git push --no-verify' and state that you did in the session report." >&2
|
||||
else
|
||||
echo "pre-push [felhom-agent]: gates OK - push proceeding."
|
||||
fi
|
||||
exit $rc
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# build output
|
||||
/felhom-agent
|
||||
/felhom-agent.exe
|
||||
/dist/
|
||||
|
||||
# local config that may carry a real token secret
|
||||
/agent.json
|
||||
*.local.json
|
||||
|
||||
# go
|
||||
/vendor/
|
||||
|
||||
# Python bytecode written by configs/test_felhom_os_apply.py
|
||||
configs/__pycache__/
|
||||
@@ -0,0 +1,18 @@
|
||||
# .security-accepted — hits security_scan_gate.py (felhom.eu/scripts, R-986) lists as ACCEPTED, not WARN.
|
||||
# One per line: <tool> <id> <where> # reason. govulncheck: OSV id + module dir; gitleaks: rule@<sha256[:12] of the line> + file.
|
||||
# The first scan's verdicts (2026-10-19): felhom.eu documentation/audits/night-2026-10-19/P7/triage.md
|
||||
gitleaks generic-api-key@1ad7cc8eb57e internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@46ce5a3eccc4 internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@46fe126846b0 internal/escrow/identity_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@56474f1754f5 internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@631c53997f7e internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@6e790ef41ef3 internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@78334b854242 internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@7b88d323a44f internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@b0eeee93cf24 internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@b1cb2659967a internal/localapi/escrow_stage_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@b66bec43cc8a internal/escrow/recover_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@e0525160fe03 internal/pbsdr/wrapper_read_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@e442e455c360 internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@f23f93f461b2 internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
gitleaks generic-api-key@f471b78e0f6e internal/escrow/recover_retained_test.go # false: a test constant or a decoy value, no live secret
|
||||
@@ -0,0 +1,52 @@
|
||||
# AGENT-001 fix — inline customer-confirmed wipe: anti-retarget re-resolution
|
||||
|
||||
**Branch:** `fix/agent-001-wipe-durable-reresolve` (off `main` @ d17b5ab) — **PENDING REVIEW, NOT deployed.**
|
||||
**Status:** branch-only. No agent build, no deploy, no golden rebake, no re-provision. That is reserved for a supervised session (this fix touches the destructive storage-wipe path).
|
||||
|
||||
## The bug
|
||||
|
||||
`internal/localapi/disks.go` `handleDiskFormat`, the data-bearing **customer-confirmed** branch
|
||||
(`dec.Allowed`), inspected and gate-bound the device by its durable id but then called
|
||||
`s.disks.Format(ctx, req.Device, …)` on the **caller-supplied mutable `/dev` path**. The durable id
|
||||
only bound the *confirmation*; it was never re-resolved to the device just before `mkfs`. If a USB
|
||||
re-enumeration reassigned `/dev/sdbN` to a **different physical disk** between inspection and `mkfs`
|
||||
(the classify→mkfs TOCTOU), the wipe hit the wrong drive — data loss on an unintended disk.
|
||||
|
||||
The signed-jobs path (`internal/signedjobs/wipe.go` `WipeExecutor.Execute`) already does this
|
||||
correctly: resolve durable→device, re-derive the device's durable id and require an exact match,
|
||||
re-inspect, then format the *re-resolved* device. The inline path did not.
|
||||
|
||||
## What changed
|
||||
|
||||
- **New `internal/localapi/wipe_reresolve.go`:**
|
||||
- `antiRetargetResolve(durableID, resolve, derive, inspect)` — pure, injected-deps helper mirroring
|
||||
`WipeExecutor`: refuse empty durable id; resolve→current device; re-derive and require exact
|
||||
match (a `/dev` node now pointing at a different disk derives a different id → refuse); re-inspect
|
||||
and require still-`DataBearing()`. Returns the **re-resolved device** to format.
|
||||
- `(*Server).reresolveDurableForWipe` wires it with the real `storage.ResolveDurableDevice` /
|
||||
`storage.DeviceDurableID` / `s.disks.InspectDevice`.
|
||||
- **`internal/localapi/disks.go`:** the `dec.Allowed` branch now calls `s.reresolveWipe(ctx, deviceDurable)`
|
||||
and formats the **returned** device (never `req.Device`). On any refusal it returns `409 Conflict`
|
||||
with the reason and does **not** call `mkfs`. The response/log now reference the re-resolved device.
|
||||
- **`internal/localapi/server.go`:** new injectable field `reresolveWipe`, defaulted in `NewServer` to
|
||||
`s.reresolveDurableForWipe`. Production behaviour unchanged (same real functions); the seam exists so
|
||||
the inline path is unit-testable without real `/dev/disk/by-*`.
|
||||
- **Tests:** `wipe_reresolve_test.go` covers `antiRetargetResolve` directly — happy path, empty durable,
|
||||
no-longer-resolves, **durable-id mismatch (the core retarget case)**, no-longer-data-bearing,
|
||||
re-inspect error. The existing integration test `TestFormat_DataBearing_UserDataConfirmed_Formats`
|
||||
stubs `reresolveWipe` (the test helper sets it). Full `go build/vet/test ./...` green at the branch HEAD.
|
||||
|
||||
## Why it can't break legitimate wipes
|
||||
|
||||
`storage.DeviceDurableID` produces `byid:`/`byuuid:` ids and `storage.ResolveDurableDevice` accepts
|
||||
exactly those — they round-trip. A normal user-data drive resolves back to itself, re-derives the same
|
||||
id, and inspects data-bearing → format proceeds. Only three new refusals occur, all correct:
|
||||
durable id gone, durable-id mismatch (retarget), or no-longer-data-bearing.
|
||||
|
||||
## Review checklist for the supervised merge+deploy+golden-rebake session
|
||||
|
||||
1. Confirm `antiRetargetResolve` refuses on mismatch/gone/blank/empty (run `go test ./internal/localapi/ -run AntiRetarget -v`).
|
||||
2. Sanity-check `DeviceDurableID`↔`ResolveDurableDevice` scheme round-trip on the real demo USB (`felhom-usb`) so a genuine customer wipe still resolves+matches (read-only `DeviceDurableID` on the live device; do NOT wipe).
|
||||
3. Confirm the `409 Conflict` refusal surfaces sensibly in the controller UI (controller `agentapi` maps the format response — verify it doesn't treat 409 as a hard error that hides the reason).
|
||||
4. Bump agent version + CHANGELOG, build, golden-rebake, re-provision per the agent deploy runbook.
|
||||
5. Live-validate one happy customer-confirmed wipe on a demo scratch drive (re-resolve matches → mkfs) and confirm the audit log line shows the re-resolved device == the confirmed durable id.
|
||||
+6960
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,112 @@
|
||||
# CLAUDE.md — `felhom-agent`
|
||||
|
||||
> Stable orientation only — **current state lives in `CONTEXT.md` and the top of `CHANGELOG.md`**,
|
||||
> never here. Cross-repo conventions (artifact taxonomy, access, clean-tree gate, secrets,
|
||||
> CHANGELOG/REPORT): workspace-root `/mnt/5_hdd/felhom.eu/git/CLAUDE.md`. Path-scoped detail:
|
||||
> `.claude/rules/`.
|
||||
|
||||
## What this repo is
|
||||
|
||||
The operator-tier **host agent**, one per Proxmox host, owning **all** Proxmox interaction:
|
||||
provision/restore guests, host storage, backup/restore orchestration, the hub control loop, and a
|
||||
narrow per-guest local API. It is the **most privilege-sensitive component in the system**.
|
||||
|
||||
- Renamed from `proxmox-controller`.
|
||||
- **Distinct from `felhom-controller`** — that is the *in-guest* controller, Docker-only, holding no
|
||||
Proxmox credentials. Do not confuse them.
|
||||
- **Control plane, not data plane:** if the agent dies, apps keep serving; only management degrades.
|
||||
- Pure Go stdlib + `golang.org/x/crypto`. No web frameworks.
|
||||
|
||||
## Doing X → read Y
|
||||
|
||||
| Doing | Read |
|
||||
|---|---|
|
||||
| writing any new code | `REUSE.md` — helpers, format-safety guards, traps, seams |
|
||||
| needing current state / open threads | `CONTEXT.md` + the top `CHANGELOG.md` entry |
|
||||
| Proxmox, reconcile or signed jobs | loads itself: `.claude/rules/proxmox.md` |
|
||||
| local API, authz or guest hooks | loads itself: `.claude/rules/localapi.md` |
|
||||
| backup, PBS or DR | loads itself: `.claude/rules/backup.md` |
|
||||
| storage or escrow | loads itself: `.claude/rules/storage.md` |
|
||||
| writing a health check | loads itself: `.claude/rules/health-checks.md` |
|
||||
| **release, build, publish, deploy, verify a version** | the **`felhom-build-deploy`** skill — **never hand-roll it** |
|
||||
| writing or reviewing a test, fixing a bug | the **`felhom-testing`** skill |
|
||||
| host addresses, break-glass, node facts | `felhom.eu/documentation/operations/nodes.md` — never restate them |
|
||||
| which box may I break | `felhom.eu/documentation/runbooks/target-selection.md` |
|
||||
| what version is live anywhere | ask the hub (`/hosts`, `/configs`) or the box — **never a doc** |
|
||||
| the authoritative design | `felhom.eu/documentation/architecture/03-host-agent.md` (locked) |
|
||||
|
||||
## The root-CLI fence — API-first, exactly three exceptions
|
||||
|
||||
This is in the core because breaching it is how this component stops being auditable.
|
||||
|
||||
- **API-first** via a scoped `FelhomAgent` token. Raw root-CLI is **fenced to exactly 3 exceptions**:
|
||||
keyctl `pct create` (golden image), USB mount/fstab, SMART/sensors.
|
||||
- **`Client` never shells out; `Privileged` never makes HTTP calls** — asserted by `routing_test.go`.
|
||||
Adding a method to `proxmox.Privileged` breaks the fence; use `proxmox.Runner` plus a new sudoers
|
||||
`Cmnd_Alias` and `validate.go`-style checks (`REUSE.md` §3).
|
||||
- **Destructive ops go through the reconcile gate / signed-jobs path.** Never call
|
||||
`Client.DestroyLXC` / `Vzdump` / `SetConfig` ad-hoc — that skips classification, signature,
|
||||
per-guest serialization and crash recovery.
|
||||
- **Ownership must be PROVEN, never assumed.** A raw `ListLXC` list is not "guests the agent owns";
|
||||
intersect with `Client.Pool` membership and fail safe on a read failure (audit A1).
|
||||
|
||||
## Gates — ONE entry point
|
||||
|
||||
**Run `python3 scripts/agent_gates.py` from the repo root after ANY change here.** It runs every
|
||||
gate in its `GATES` table (that table is the list); the shared ones — `reuse_refs_check`,
|
||||
`instructions_gate`, `observations_gate` — are the copies in `felhom.eu/scripts/`, never copied into
|
||||
this repo (a copy recreates the drift they detect; an absent sibling clone FAILS). `--fast` selects the
|
||||
gates touching no network and no container runtime, and skips `published` (network) and `gate-decoys` (the decoy
|
||||
suite, ~20 s; R-1116), naming them — CI runs the full set. **A missing gate is a FAILURE, never a skip.**
|
||||
|
||||
**The pre-push hook** (`.githooks/pre-push`) runs it with `--fast` and refuses a failing push. It is
|
||||
**per-clone** — switch it on once with `git config core.hooksPath .githooks`, and a manual run WARNS
|
||||
when this clone is unarmed. `git push --no-verify` bypasses it deliberately; **say so in the session
|
||||
report when you use it** — CI re-runs the same entry point on every push and **emails the operator on
|
||||
failure**, so a bypass is noticed even though it is not blocked (R-168, CLOSED 2026-08-02).
|
||||
|
||||
<!--
|
||||
WHY ONE ENTRY POINT (2026-08-02, R-29): a census of all gates across the four repos found every check
|
||||
a CLAUDE.md names was passing, and two of the four nobody is told to run were failing. This repo was
|
||||
the extreme case — nothing ran against it at all, and 90 cited paths were checked by no one.
|
||||
-->
|
||||
|
||||
## Live validation — the fence
|
||||
|
||||
Exercise the **SERVER-SIDE PIPELINE** a real user triggers, end-to-end. **The forbidden shortcut is
|
||||
BYPASSING it** — the F9 episode was a raw guest-attach with hand-set state, and it proved nothing.
|
||||
|
||||
`claude-in-chrome` is NOT available on DooPlex. Invoking the exact endpoint the UI invokes is an
|
||||
acceptable proxy — **say which method was used**. Low-level mechanism tests where the direct call IS
|
||||
the mechanism are exempt.
|
||||
|
||||
## Conventions
|
||||
|
||||
- **Trunk-based — no branches.** All shippable work commits directly to `main`; `main` equals what is
|
||||
deployed. Report-only artifacts (audits, findings, fixspecs) go to `felhom.eu/documentation/`.
|
||||
- **Unattended escape hatch:** if a fix cannot be cleanly verified and shipped, **revert and report**
|
||||
— never park it on a branch.
|
||||
- **Logging**: the slog logger fans out to journald (configured level) plus the always-DEBUG
|
||||
`applog.Ring` (remote pulls). English, keys-never-values, durations on outcomes. Full rules:
|
||||
`felhom.eu/documentation/runbooks/logging-conventions.md`.
|
||||
- Update `REUSE.md` in the same commit that adds, changes or deprecates a shared helper or pattern.
|
||||
|
||||
## End-of-session checklist
|
||||
|
||||
- **`CHANGELOG.md`** (cumulative, newest on top) and **`REPORT.md`** (overwritten with this run only)
|
||||
— in every repo touched.
|
||||
- **`CONTEXT.md`** — decisions, state, what is next.
|
||||
- **`REUSE.md`** — if a shared helper or pattern moved.
|
||||
- **A finding goes in `felhom.eu/documentation/backlog/OPEN-ITEMS.md` first**, never only in a report
|
||||
or an audit.
|
||||
- **Confirm your own last push's CI run went green, by run ID** — CI mails on failure, which is a PUSH
|
||||
signal; this is the PULL check that catches a lost or unread mail. An unchecked green is an
|
||||
assumption, not an observation.
|
||||
|
||||
**A gate ships with a decoy test that has been seen to fail (R-421).** A decoy is the LABEL without
|
||||
the FACT — a directory with the right name and no bake log, a note whose prose mentions the marker it
|
||||
lacks. `felhom.eu/scripts/decoy_coverage_gate.py` (run by felhom.eu's `repo_gates.py`, for all four repos) refuses a new gate that has neither a decoy nor a named
|
||||
exemption carrying its row. The four shapes, the 2026-09-01 sweep that fooled 16 of 29 gates, and the
|
||||
decoys withdrawn as illegitimate: `felhom.eu/documentation/audits/AUDIT-gate-decoys-2026-09-01.md` and
|
||||
`felhom-controller/.claude/rules/gates.md`. **Scope is a fact too** — prefer `os.walk` over
|
||||
`os.listdir`, and a glob over a hand-maintained list.
|
||||
+606
@@ -0,0 +1,606 @@
|
||||
# CONTEXT — felhom-agent working state
|
||||
|
||||
> **2026-10-28 night (unattended) — R-528 + R-1254 on `main` (`6f933d6`, `5ddd124`), NOT released (the next agent
|
||||
> release, the operator's vouch).** `GET /guest/oom-kills` (`internal/localapi/guest_oom.go`): the host-side `oom_kill`
|
||||
> of each Docker container of the token's guest (`Server.cgroupRoot` is the test seam); proven on Tester 1 with a test
|
||||
> build (felhom.eu `audits/night-2026-10-28/r528-oom/`). Escrow ceremony: after a failed upload `HubHolds` compares the
|
||||
> hub's identity blob with the new package (`hubHoldsEscrowPackage`, cmd) — stored / not stored / unknown (504).
|
||||
|
||||
> **2026-10-26 — R-1285 on `main` (`b1bc63d`), NOT released (operator: „not today").** The new agent runs the bundle
|
||||
> policy pass once after its self-update commits (`selfupdate.Manager.SetOnCommitted` → `PolicyPassAfterUpdate`). Live
|
||||
> proof needs a BYO box coming from 0.164.0. Tuesday 2026-10-27: the weekly line test's first run against a seeding box
|
||||
> on demo-hp (felhom.eu R-1250, read, not forced).
|
||||
|
||||
> **2026-10-24 night (unattended) — D43 = A on `main` (`c8d3c17`), NOT released (next v0.165.0, with R-1250).** The sshd
|
||||
> drop-in's bundle policy is `appliance` (install record `/var/lib/felhom-install/state.json` mode); BYO: nothing written,
|
||||
> our copy removed; no proof: untouched. X3: `lan_addresses()` for `sshd -T`. X4: `undo_sshd_step` — only the SSH file is
|
||||
> put back. Release + the operator's vouch: Monday's list (felhom.eu STATUS).
|
||||
|
||||
> **2026-10-24 (attended release run) — v0.164.0: the weekly upload line test (R-1247; operator ruling 07:26).**
|
||||
> `internal/linetest`: the agent's own iperf3 client (operator's pick — nothing installed on hosts) to ep0's tunnel-only
|
||||
> server `10.77.0.1:5201`, Tuesday 10:00-15:59, after 10 quiet minutes, under the heavy-op gate, best of 3; `line_test`
|
||||
> stanza + `GET /host/line-test` (the controller's qBittorrent limit, D41 = 50 %). demo-hp measured 90.7 Mbit/s. Release
|
||||
> needs the operator's vouch.
|
||||
|
||||
> **2026-10-22/23 night — D30 on `main` (`ae69ad4`), unreleased; the D4 patches v2 ready.** Operator ruling D30
|
||||
> (2026-10-22): a handshake stamped in the future for 15 min → a new WG key pair (`wgtunnel/rekey.go`; old key kept as
|
||||
> `wg/private.key.prev`), registered on `POST /hosts/<id>/wg` (the hub re-keys in place; its full-list push swaps the
|
||||
> key on ep0). Live on a test box (felhom.eu `audits/night-2026-10-23/P2-D30/`): with ep0 free to start a handshake the
|
||||
> tunnel heals by itself (no re-key, correct); with ep0's starts dropped it stays dead and the agent re-keyed at 15 min.
|
||||
> The interface switches at the next desired-state fetch. Gap: a reboot forgets the future stamp (R-1223). The D4 bundle
|
||||
> patch v2 (`P3-review/felhom-agent-bundle-D4-v2.patch`): `try-reload-or-restart`, an `sshd -T` read-back reported as
|
||||
> `sshd_dropin_not_in_force`, a reload in undo. Next agent release = D30 + D4 v2 + R-1204's field, the operator vouches.
|
||||
> **2026-10-20 — v0.161.0 RELEASED (operator yes), NOT VOUCHED; on demo-hp, Tester 1, demo-felhom** (signed jobs, all
|
||||
> committed, 76/76). R-921 `host_busy` (proven live on demo-hp during a press), R-1070 the tunnel `absent` state (9202's
|
||||
> real Docker 29 answer is the fixture row), R-1010 the janitor's one `force=1`. Vouch: version 0.161.0, sha256
|
||||
> `ff4e6dc5…99d3b5`, bundle `485f36c9…12c47`. R-1083 left out on purpose: an agent key for 9202 would make its
|
||||
> controller book whole-guest backups through the agent, one tier writing to ep0 — the R-1083 row says what is needed.
|
||||
> **2026-10-19 — v0.160.0 RELEASED (operator yes), NOT VOUCHED; on demo-hp, Tester 1, demo-felhom.** R-1093, R-978,
|
||||
> R-973 (below) plus R-1149 (the local tier prunes `keep-daily=N`, `BackupTier.PruneSpec`) and R-1151 (a claim with
|
||||
> `{"defer_upload": true}` keeps the package until `POST /escrow/ceremony/confirm` brings the two words at the drawn
|
||||
> positions; only those two words are held, 30-min window; a claim without it keeps the R-978 order). Vouch: version
|
||||
> 0.160.0, sha256 `7ebb547f…dfca1`, bundle `6b060901…e202f`. After it, unreleased: `4545819` (two texts). felhom.eu
|
||||
> `REPORT-day-2026-10-19.md`.
|
||||
> **2026-10-18 (night) — UNRELEASED on main: R-1093, R-978, R-973** (release + vouch: the operator's). R-1093 an empty
|
||||
> partition is blank (lsblk's PTTYPE on a partition is the parent's; set aside only on a clean blkid exit 0 with a
|
||||
> matching `PART_ENTRY_SCHEME`; an unparsable lsblk is unprobed). R-978 the escrow package reaches the hub only at the
|
||||
> claim (ceremony output v2, `PendingUpload`; `EscrowCeremonyConfig.Upload`). R-973 the `POST /backup` cadence cap.
|
||||
> **Decision (decided by CC unattended — operator may reverse): the cap's numbers.** Q: how many whole-guest runs may
|
||||
> the guest ask for? Options: (a) 2 per 24 h on the daily tier — keeps one local point older than a day, but refuses
|
||||
> the NIGHT after a press (R-899's night can fall inside the same 24 h); (b) 3 per 24 h + 1 per 6 days on the weekly
|
||||
> tier — the schedule never refused, a second press in a day refused, the local tier still replaceable in a day;
|
||||
> (c) none — status quo. Chose (b): the off-site tier is the one `07` §8 row 10 relies on and it is fully protected
|
||||
> for ~6 days; the local half's real fix is a time-bucketed prune (keep-daily), which is a retention change for the
|
||||
> operator. No sudoers or bundle change in any of the three.
|
||||
|
||||
> **2026-10-17 — v0.159.1 RELEASED (0.159.0 superseded, do not vouch).** Go 1.27.2; R-1124 a decommissioned drive that is
|
||||
> gone is forgotten (`ForgetRetiredDrives`, root verb `unit-forget`); R-1012 the read-only `felhom-thin-check` names an
|
||||
> orphan thin device in a failed backup's text; the signed decommission prunes the guest-bind record. Bundle adds a path →
|
||||
> step bundle `0.159.0-step1`. Proven on demo-hp; delivered by signed jobs on the operator's yes. Vouch: the operator's.
|
||||
|
||||
> **2026-10-16 — v0.158.1 RELEASED and VOUCHED (storage identity build B; 0.158.0 superseded, do not vouch).** The internal
|
||||
> data place (root helper `felhom-internal-place`, the record in the state dir, mounted at /mnt/belso like a drive), the
|
||||
> signed `add_internal_place`, partitions in `/disks/candidates` (`formattable` only while nothing on the disk is mounted),
|
||||
> the partition wipe id `byid:<…-partN>+fsuuid:<uuid>`, the strict mkfs wrapper. Delivered through step bundle
|
||||
> `0.158.1-step1` (a bundle that adds a path, R16). Next: R-1093 (an empty part's probe reads the parent's partition table).
|
||||
|
||||
> **2026-10-15 (night) — v0.157.0 RELEASED, NOT VOUCHED (storage identity build A, felhom.eu `16` §13).** Every storage
|
||||
> place carries `kind`, `physical_disks`, `identity` (+ reason) in `GET /disks` and the hub report; `POST /disks/same-disk`
|
||||
> answers the controller (v0.317.0, MinAgent 0.157.0). Computed on every read, never cached. Delivered by signed jobs to
|
||||
> Tester 1, demo-hp, N100 only. Next: the operator vouches 0.157.0 → the global floor 0.317.0 / MinAgent 0.157.0. Build B
|
||||
> (partition candidates, the internal place) waits for a task file (R-1079).
|
||||
|
||||
> **2026-10-08 (day) — UNRELEASED on main, ships with tomorrow's release (decision 178).** R-899: `POST /backup?trigger=manual`
|
||||
> runs no OS leg (`localapi/server.go`); after-boot kernel reports carry the saved block's ring (`kernelReportRing`). R-304:
|
||||
> `escrow.ErrRetainedUnchecked` → HTTP 424 `older_unchecked` when not every earlier package was tried (withheld, caps,
|
||||
> malformed, unreadable list). Binary-only delivery; no wrapper or root file changed.
|
||||
|
||||
> **2026-10-07 (evening) — v0.152.0, the kernel lane (R-836, decision 172, `11` §5.11).** Wrapper layer `kernel` + two GRUB generators in the bundle (delivered with step bundle `0.152.0-step1` — the bundle adds paths, R-880); `osupdate/kernel.go`: night step on told nights only, after-boot judge (host rule + hub reached, 20 min), ONE self-revert, `os_kernel_step` stages only. Proven on Tester 1 (panic → fell_back, held guest → self_reverted, healthy → 7.0.14-22 default). Open: R-898, R-897; the ring-0 night run.
|
||||
|
||||
> **2026-10-04 night — v0.143.0 RELEASED + vouched (R-840, decision 96): the config bundle.** `felhom-os-apply` mode
|
||||
> `bundle` (signed `agent_config_update`, verified by the wrapper itself; trust files never bundle paths) +
|
||||
> `--install-bundle` (installer 1.31.0); `BUNDLE_FILES` is the one table; `scripts/build-config-bundle.py`;
|
||||
> `release-agent.sh` publishes it. A box whose `felhom-os-apply` predates 0.143.0 needs ONE by-hand bootstrap
|
||||
> (`felhom.eu/scripts/felhom-bundle-bootstrap.sh`) — done on both demo boxes; Tester 2 waits for the operator (R-862).
|
||||
> Both demo boxes: agent 0.143.0, bundle 0.143.0 (record `/etc/felhom/config-bundle.json`). R-861 found: the sudoers is
|
||||
> root-equivalent. `build-golden.sh` 3.2.0 (`GOLDEN_GUEST_PKGS`). Runbook `felhom.eu/documentation/runbooks/config-bundle.md`.
|
||||
|
||||
|
||||
> **2026-09-25 night — v0.133.0 AND v0.134.0 DELIVERED to both demo boxes (CC-signed `agent_update`, ruling 1);
|
||||
> restore test back ON (the `-1` config kept as `agent.json.night-0925-off`). v0.134.0 = R-685:** `backup/runner.go`
|
||||
> `spaceFits` — a vzdump to a LOCAL target needs free ≥ newest archive of that guest × 1.25 + 1 GiB (PVE prunes
|
||||
> only after success); a shortfall is a named skip (`BackupSkipNoSpacePrefix`), fail-open on PBS / first backup /
|
||||
> unknown usage. **Free space comes from `NodeStorage` (`GET /nodes/<n>/storage`) — `ListStorage` (`GET /storage`)
|
||||
> has NO usage**; the first build read it and a real vzdump started in its own live test (aborted, no archive).
|
||||
> demo-hp: `local_backup_retention` 1 (operator option A, saved `agent.json.pre-a4-retention`). Peti's box: nothing.
|
||||
|
||||
> **2026-09-24 — v0.133.0 RELEASED, NOT DELIVERED (R-672, R-673).** Restore-test space preflight
|
||||
> (`reconcile/restoretest_space.go`, `internal/restorespace`): uncompressed size from the vzdump log / PBS size,
|
||||
> × 1.2 + 5 GiB, thin metadata, off the tested guest's pool, unknown refuses, reported as `skipped` non-pass.
|
||||
> Janitor (`cmd/felhom-agent/janitor.go`) every 10 min: `Engine.RetryScratchTeardown` + stale-lock sweep under
|
||||
> the heavy-op gate. Thin pool ≥ 90 % → immediate report (hub v0.124.0 alarms). **Operator rulings 2026-09-24
|
||||
> (evening):** the scheduled restore-test is OFF on both demo hosts (`backup.restore_test_eval_interval_seconds:
|
||||
> -1` — note: 0 means the 6 h DEFAULT, only a negative disables) until v0.133.0 is delivered there; saved configs
|
||||
> `/etc/felhom-agent/agent.json.pre-r672`. demo-hp 9201 was repaired (stop, fsck, start; two Redis AOF tails cut).
|
||||
> Snapshot of the current state + open threads. Authoritative history lives in `CHANGELOG.md` (top
|
||||
> entry = current); the end-of-task detail lives in `REPORT.md`.
|
||||
|
||||
## R-199 (v0.125.0) — links 6–8 of the recovery chain, assembled and walked
|
||||
|
||||
`POST /escrow/recover-offsite-password` (pinned local API, `withGuest`): the controller supplies the
|
||||
customer's recovery code, the agent fetches THIS host's own sealed blob from the hub
|
||||
(`hub.Client.FetchIdentityEscrow` → `GET /hosts/{id}/escrow`, hub >= v0.94.0, self-scoped by the
|
||||
per-host key), unseals it via `escrow.OffsiteKeyRecoverer`, and returns **only** the offsite restic
|
||||
repository password plus its sha256.
|
||||
|
||||
**Rules that must not erode:**
|
||||
- **Only that field.** Not the tunnel token, not the PBS token, not the WG key — the controller is a
|
||||
trust tier down and needs none of them. Narrowing cost nothing and is not recoverable later.
|
||||
- **The unseal stays in the agent.** `age` is an agent runtime dependency (`/usr/bin/age` — hardcoded,
|
||||
no config override; 1.2.1 on demo-felhom) and is deliberately absent from the controller image.
|
||||
- **R:** in memory for one call, cleared on the success path AND every failure path, never on disk,
|
||||
never in argv, never logged at any level including inside an error, never echoed. Verified live: 0
|
||||
log lines, 0 files, 0 leftover `felhom-idesc-*` dirs, with a positive control proving the search worked.
|
||||
- **Three distinct outcomes**, not one generic failure: no blob (404), a bundle that opens but predates
|
||||
the field (409 — pre-fork-4, cannot be retro-fitted), a code that does not open it (400 — fail-closed
|
||||
at age's KDF, nothing written).
|
||||
- **The wiring is pinned by an AST walk** (`cmd/felhom-agent/escrow_recover_wiring_test.go`):
|
||||
`main` → `runDaemon` → `buildLocalAPIServer`, an `escrow.OffsiteKeyRecoverer` constructed there, the
|
||||
`Options.EscrowRecovery` field present, and the fetcher calling the DAEMON's own `hubClient` (the
|
||||
self-scoping that makes cross-host retrieval impossible is a property of WHICH key is used).
|
||||
Links 6 and 7 were two of this project's six built-but-never-wired instances.
|
||||
|
||||
**Proven live on demo-felhom 2026-08-04:** recovered sha256 == on-disk sha256 == the hub's stored hash.
|
||||
A wrong code five minutes earlier failed closed. **The chain stops at link 8** — nothing installs a
|
||||
recovered password, reopens a repository, or restores a file.
|
||||
|
||||
**§8.6, fixed while here:** `runSelftestIdentityConsume`'s success line used to recite
|
||||
"tunnel_token + pbs_token", which became a misstatement when v0.77.0 sealed the repository password
|
||||
into the same bundle — anyone reading it would conclude the password was not there. It now names what
|
||||
THIS bundle carried and what it did not.
|
||||
|
||||
## Current
|
||||
|
||||
- **2026-08-03 — v0.123.0 (R-185): a tier the box cannot READ now says so.** The agent's token had
|
||||
`FelhomAgentStore` on `local`, `local-lvm`, `felhom-pbs` and **not** on `felhom-backup` — the
|
||||
storage both demo boxes configure as `local_backup_target`. That storage answered `{"data":[]}`
|
||||
through the token while root listed three archives, and `pickForThisRun` skipped it as *"no settled
|
||||
archive yet"* — **which is what a brand-new tier reports**, so the host tier was never
|
||||
restore-testable and nothing said so.
|
||||
- **The permission question is asked directly**, because unlike the listing it has a definite
|
||||
answer: `Client.Permissions` reads `/access/permissions?path=/storage/<target>` **as the agent's
|
||||
own token**, and `storeGrantStatuses` emits one `capability.Status` per configured tier. It
|
||||
composes AROUND the sudo prober, the way `poolReadStatus` already does — an API read does not
|
||||
belong inside a sudo-policy probe. `Status`'s wire shape is untouched, so the hub's critical
|
||||
degraded alert applies with **no hub change**.
|
||||
- **MEASURED FIRST, and the obvious reading is wrong:** an ungranted path answers neither empty nor
|
||||
403 — it carries the privileges INHERITED from the box-wide `/` grant
|
||||
(`Sys.Audit, SDN.Use, Datastore.Audit`). Checking path-presence, or `Datastore.Audit`, reports a
|
||||
blinded storage HEALTHY. The probe tests **`Datastore.AllocateSpace`**; re-measure before ever
|
||||
changing that constant (`storeGrantRequiredPriv`, red-proved).
|
||||
- **The probed set comes from `BackupTiers()`, never a fixed list** — a hardcoded probe list is the
|
||||
defect reproduced inside the fix. Critical, EXCEPT the `local` fallback target (reported, but it
|
||||
does not page). It never consults content, so it cannot alarm on a newborn tier; it never reports
|
||||
ok when it could not ask.
|
||||
- **LIVE:** degraded observed on the still-blind box (hub emailed `agent_capability_degraded`) →
|
||||
grant applied on **both** demo boxes → token lists 3 and 4 archives → `ok=70 total=70 degraded=0`
|
||||
and `degraded → ok` at the hub → **the host tier became a due-check candidate for the first time**,
|
||||
correctly picking the 08-02 archive (08-03 had not settled 24 h).
|
||||
- **The installer's real defect was NOT `PVE_STORAGES`** — see `felhom.eu` CONTEXT S-22: Case A
|
||||
grants, the Scenario-F reuse arm did not. Fixed in installer **1.24.0** with a gate.
|
||||
|
||||
- **2026-08-03 — v0.122.0 (R-189 · R-188 · R-186): three signals that lied about their own work.**
|
||||
None touches data; all three cost attention, which every other signal depends on.
|
||||
- **R-189 — a passing restore-test no longer vanishes on a restart.** `restore_tests[]` came only
|
||||
from the in-memory `backup.Store` (*"lost on restart; the cadence re-populates"* — true under a
|
||||
timer, FALSE since R-86, because the agent will not re-test a proven archive). **Observed live:**
|
||||
a 14.5 GB offsite PASS at 15:25:14, agent restarted 2 m 43 s later, hub logged `0 restore-tests`
|
||||
twice. `RestoreTestState` now stores `tier` + `verified` beside the archive (v3 shape; v1/v2
|
||||
still read, and a record missing archive-or-tier is NOT reported), exposes
|
||||
`ProvenRestoreTests`, and `Collector.SetProvenRestoreTests` merges it — **one entry per tier,
|
||||
newest by `TestedAt` wins**, so a fresh failure beats a stored success and a tier never appears
|
||||
twice. Wiring pinned by an AST test: the method this replaces (`Snapshot`) claimed a
|
||||
"host-report gauge" in its doc comment and had **no caller** for weeks.
|
||||
- **ONLY SUCCESSES ARE PERSISTED, and the reason is now in the code:** a success *suppresses*
|
||||
future work (a proven archive is never re-tested, so a lost proof leaves the box quietly less
|
||||
tested than it believes); a failure *causes* future work and heals itself at the next evaluation.
|
||||
- **R-188 — the release stopped emailing false failures.** Only the tag PUSH moved (build → tag
|
||||
locally → publish → push tag): the push is what wakes CI, and a tag visible before its package
|
||||
made the gate correctly fail a correct release ~half the time. The old order's invariant is now
|
||||
asserted directly — `check-published-versions.py` refuses a **published version with no tag**, as
|
||||
a bounded, printed probe (the package listing api is still 401 without a token, re-measured).
|
||||
- **R-186 — a released binary is verifiable.** `-trimpath -buildvcs=false`: same source → same
|
||||
bytes whether or not the tag exists. Measured. `publish-agent.sh`'s fallback also forced
|
||||
`CGO_ENABLED=0` and built a **74 KB different** binary for the same version — both paths now
|
||||
identical. The verification command is in `CLAUDE.md`.
|
||||
|
||||
- **2026-08-03 — v0.121.0 (R-86): the restore-test follows the BACKUP, not the clock.** The ticker is
|
||||
now only the **evaluation interval**; a tier is **DUE** when its newest archive that has settled for
|
||||
`settle` (default 24 h) **has not been proven**. Daily tier → proved daily on yesterday's archive;
|
||||
weekly tier → weekly on its own; newborn → UNKNOWN. **The trap, so it is not reintroduced:** the
|
||||
literal reading of R-86 — *"due when the newest archive is ≥ 24 h old"* — is NEVER true on a daily
|
||||
tier (a new archive resets the age before it reaches the lag), so it switches restore-testing off
|
||||
where it matters most. Red-proved at 0 runs over 5 simulated days.
|
||||
- **The state now records WHICH archive was proven**, not just when a tier passed. A pre-R-86 file
|
||||
keeps its time (ordering survives) and yields no proven archive → each tier is due once after the
|
||||
upgrade, deliberately.
|
||||
- **The old cadence key:** `restore_test_cadence_seconds` is DEPRECATED. Negative still DISABLES
|
||||
(verbatim); a positive value now seeds the **settle lag** and the daemon WARNs once at start-up
|
||||
naming `restore_test_eval_interval_seconds` (default 6 h) and `restore_test_settle_seconds`
|
||||
(default 24 h). It is NOT carried into the evaluation interval.
|
||||
- **6 h is bounded from both ends:** measured evaluation cost (local 18 ms, PBS-over-WAN 392 ms,
|
||||
both 430 ms) says cost is irrelevant; the ceiling is that a FAILING tier stays due, so the
|
||||
evaluation interval is also its retry interval for a multi-GB restore.
|
||||
- The due-check now runs **before** the heavy-operation gate is taken (a frequent poll must not be
|
||||
able to make a starting backup record a failure — F-A1), and the candidate picker skips archives
|
||||
failing `archivePlausiblyComplete` (a phantom would be due forever and fail forever).
|
||||
- New read-only `--selftest=restore-test-due` prints the per-tier verdict + its cost.
|
||||
- **v0.121.1 — a quiet evaluation is AUDIBLE.** "Nothing is due" is now the NORMAL outcome, and at
|
||||
DEBUG it was silent: an empty journal would have been equally consistent with a healthy loop and
|
||||
a dead goroutine (standing rule 3 — the shape the R-88 watcher was retired for). A not-due
|
||||
evaluation logs ONE INFO line naming every tier's verdict; an unlistable tier reads `UNKNOWN`
|
||||
with its error in that same line.
|
||||
- **PROVEN LIVE 2026-08-03 on demo-felhom:** due-triggered offsite restore-test of a 14.5 GB
|
||||
encrypted PBS archive — restored, booted, verified, scratch destroyed, **635 s**; the state then
|
||||
named that archive, a second evaluation ran nothing, and an agent restart ran nothing.
|
||||
- **R-185 (filed, NOT fixed here):** on demo-felhom the agent token has no ACL on
|
||||
`/storage/felhom-backup`, so its content listing comes back EMPTY (root sees 3 archives) — the
|
||||
host tier has never been restore-testable there, and the due-check cannot distinguish that from
|
||||
a newborn tier.
|
||||
|
||||
- **2026-07-28 — v0.107.0: F-REBOOT fixed — a guest rebooted mid-backup now comes back by itself.**
|
||||
New `internal/localapi/guestpower.go`: a 60 s watchdog that starts a guest which is `onboot:1`,
|
||||
stopped, unlocked, and has no vzdump in flight. It closes the two narrow gaps that let
|
||||
`RecoverStaleLockedGuests` miss campaign fault 11 — that recovery acts only on a **stale vzdump
|
||||
lock** (fault 11's guest was unlocked) and runs **once at agent startup** (fault 11's guest went
|
||||
down while the agent was already up). `onboot` is the deliberate-stop discriminator and is *not*
|
||||
invented here: it is already what `stalelock.go` uses for this decision, it is 0 on scratch/golden
|
||||
guests, and it is what `pve-guests` consults at host boot — so the agent agrees with the platform
|
||||
instead of keeping a second private definition of "should be running". Retry bounded at 3
|
||||
(1m/2m/4m) then escalates **once**; an unbounded silent retry loop is the over-correction here.
|
||||
Live on demo-hp: **120 s unattended** recovery vs the incident's **587 s** with a human; Scenario B
|
||||
proven (an `onboot:0` guest left stopped throughout). Detail: `REPORT.md`.
|
||||
|
||||
- **2026-07-28 — F-LEAK took THREE attempts; v0.108.0 and v0.110.0 are the corrections.** The cause is
|
||||
structural: `FelhomAgentGuest` is granted at `/pool/felhom` and a guest joins that pool only when its
|
||||
restore **completes**, so a *failed* restore-test leaves a pool-less guest out of reach (403).
|
||||
**(1) v0.107.0 pool adoption — REFUTED LIVE:** `PUT /pools/{pool}` also requires `VM.Allocate` on the
|
||||
VM being added, so membership cannot bootstrap its own authority; removed in **v0.108.0**.
|
||||
**(2) host-install v1.21.0 per-path `/vms/990000..990009` ACLs — works, but exactly ONCE per slot:**
|
||||
PVE's destroy calls `AccessControl::remove_vm_access` (`API2/LXC.pm:906`) which deletes every ACL at
|
||||
`/vms/<vmid>` (`AccessControl.pm:1898`) — **the grant is consumed by the op it authorises**. Caught by
|
||||
counting ACL rows after the fix, not by reasoning. **(3) v0.110.0 SHIPPED —
|
||||
`Privileged.DestroyScratchLXC`, the FOURTH root-fenced exception** (was exactly three: keyctl
|
||||
`pct create`, USB mount/fstab, SMART/sensors). Band enforced in **sudoers literally**
|
||||
(`pct destroy 99000[0-9] --purge`) + re-checked in code + journal provenance at the caller; none is
|
||||
consumed by use. API destroy still tried FIRST; band ACLs stay provisioned so the common case needs no
|
||||
privileged call. **Ships with a sudoers change — deploy `configs/felhom-agent.sudoers` WITH the
|
||||
binary.** Live: token 403 on a stranded scratch → fenced path removed the guest and all 3 LVs; sudo
|
||||
PERMITS the band and REFUSES `9201`/`9100`/`9999`/`990010`/`1`, and refuses `pct start 990000` too.
|
||||
|
||||
- **2026-07-28 — v0.109.0: the guest-power watchdog got the observable it shipped without.** A
|
||||
self-correction: v0.107.0's watchdog logged only at startup and when it *acted*, so on a healthy box
|
||||
its health could be read only from **absence** — F-OBS's exact shape, shipped in the same session
|
||||
F-OBS was fixed in the controller. Now an INFO summary every 10th sweep carrying
|
||||
`sweeps_since_boot`/`guests_evaluated`/`currently_stopped`. An **aborted** sweep (unproven
|
||||
ownership) does not count, or the heartbeat would claim liveness for a watchdog examining nothing.
|
||||
|
||||
- **2026-07-28 — v0.106.0: F-CRIT-2 fixed — a failed backup no longer looks like a fresh one.**
|
||||
`NewestArchiveTime` counted an aborted PBS upload (1 byte, manifest-less, NEWEST) as a successful
|
||||
backup, so the tier reported fresh, went **not due**, and was never retried — 7 days of silence on
|
||||
the real 168h cadence, invisible to both the R-88 breaker (defers only DUE tiers) and the hub
|
||||
deadline monitor (reads the same freshness). Now only *plausibly complete* entries count, via a
|
||||
measured floor `minPlausibleArchiveBytes` = 1 MiB; undecidable ⇒ not counted.
|
||||
**Size is the only tier-agnostic discriminator** — `verification` and `encrypted` are absent on
|
||||
every local (dir) archive and on a good PBS snapshot until verify-new catches up, so gating on
|
||||
either would reject 100% of local backups and cause fleet-wide backup THRASH. Floor measured:
|
||||
smallest real backup on the fleet is 612,397,450 B, so 1 MiB leaves 584x headroom (asserted by a
|
||||
test). Rejections logged at WARN once per volid. Re-tested live by replaying campaign fault 2 on
|
||||
demo-hp — both directions, incl. a no-thrash window with 91 scheduler ticks as the positive
|
||||
observable. Deployed on both boxes. Detail: `REPORT.md`.
|
||||
**Also established:** server-side prune does NOT count phantoms toward `keep-last` (dry-run kept
|
||||
2 real + the phantom) ⇒ **no retention/data-loss bug** — but it never removes them either, so they
|
||||
accumulate. Filed as R-99 (LOW).
|
||||
|
||||
- **2026-07-25 — v0.95.0 (additive): SMART coverage fixes (spike B+A) + device model.** Union-path
|
||||
drives (USB/registry) now get SMART via `storage.SmartReader.SMARTForBacking` wired into the localapi
|
||||
`/disks` union (localapi `Smart` seam); `smartDeviceFor` resolves dm/LVM to the whole disk via
|
||||
`/sys/block/<dm>/slaves` (recursive, skips >1-disk); the builtin `local` dir on the LVM root gets a
|
||||
**SMART-only** device from its containing filesystem (never touches backing/durable_id — the
|
||||
removable-safety guard in build() stays intact); `SmartSummary.ModelName` captured from smartctl. The
|
||||
watchdog `Known` path stays enrich-free. Consumed by controller v0.171.0. Source of WHERE:
|
||||
`felhom.eu/documentation/audits/SPIKE-smart-coverage-2026-07-25.md`.
|
||||
- **2026-07-24 — v0.94.0 (additive): SMART serialized into /disks.** `localapi.DiskInfo` gains
|
||||
`Smart *hub.SmartSummary` (omitempty), copied from the target's already-computed Observe-time
|
||||
enrichment when `Health != ""` — no new smartctl load, no endpoint, no sudoers/MinAgent change. The
|
||||
controller v0.169.0 renders a "Lemezek állapota" card + 6h degradation alert from it; old controllers
|
||||
ignore it. **NOTE: at the remote-site vacation window the agent is DOWN (localapi binds .162 → fails),
|
||||
so live /disks-from-real-agent validation is deferred — the field is unit-proven; publish only.**
|
||||
- **2026-07-22 — v0.93.0 is the FLEET AGENT.** Built, published (sha `a68b2ff73200622e…`),
|
||||
Day-0-manifest-vouched (MinAgent also 0.93.0, operator-ruled) and deployed to BOTH boxes
|
||||
(`demo-felhom-8363b5` + `demo-hp-bb76ea`, the latter over G1 break-glass — still no key baked);
|
||||
clean-restart 5/5 on both, `.bak-0.92.1` retained. Discharges the onboarding runbook §A5
|
||||
ceremony gate. Record: `felhom.eu/documentation/pilot/RUNBOOK-publish-agent-0.93-2026-07-22.md`.
|
||||
**The bullet below ("agent is DOWN … deployed 0.90.0") is SUPERSEDED history** — vmbr0 was made
|
||||
static .162 on 2026-07-20 (F1 mitigation) and the agent has been up since; kept for the record.
|
||||
|
||||
- **2026-07-20 — REMOTE SITE until ~2026-08-02; the agent is DOWN there and cannot self-recover.**
|
||||
felhom-pve moved off the home LAN; `ssh felhom-pve` = tailnet `100.70.170.35` (direct, ~37 ms). The
|
||||
host is on DHCP and holds `192.168.0.147`, so `localapi`'s literal `192.168.0.162` bind fails with
|
||||
`bind: cannot assign requested address` — the daemon exits ~1.1 s after start, systemd gave up after
|
||||
4 retries, and a manual restart reproduces it exactly. Deployed binary is **0.90.0**. Fix needs
|
||||
`listen_addr` in `/etc/felhom-agent/agent.json` **and** the guest bootstrap endpoint (plus the
|
||||
pinned leaf's SAN) → **Viktor GO**; re-pinning to another literal just re-breaks on the next lease.
|
||||
Also re-observed each start: `pbs: cannot read token secret … /etc/pve/priv/storage/felhom-pbs.pw:
|
||||
permission denied` (R-39-adjacent). Evidence + ranked findings:
|
||||
`felhom.eu/documentation/audits/AUDIT-vacation-remote-ops-2026-07-20.md`
|
||||
|
||||
- **v0.90.0** (2026-07-17) — **agent train: guest RAM resize (R-24) + fast-tick (R-28); LIVE on BOTH
|
||||
demo hosts (felhom-pve + nested demo-vm-felhom-4846bc).** MinAgent coupling: felhom-controller
|
||||
v0.143.0 gates its resize UI on this agent. (1) **R-24 guest RAM resize (controller-direct)** —
|
||||
self-scoped `GET`/`POST /guest/memory` (`internal/localapi/guestmemory.go`); the AGENT enforces
|
||||
every bound fresh per request (min 2048 / max host_total−2048 / shrink floor max(2048, usage+512)) and
|
||||
applies via PVE `SetConfig` — **live cgroup apply, no reboot** (Phase-0 PROVEN on the nested box; the
|
||||
break-glass access path + the proof are in `~/.claude/.../nested-vm-access-breakglass.md`). Verify-
|
||||
after-apply re-reads maxmem before claiming success. New narrow `MemoryOps` seam (GuestAPI untouched);
|
||||
memory only. (2) **R-28 fast-tick** (`internal/fasttick/`) — while any desired-state item is unapplied
|
||||
(esp. the pre-tunnel WG-registration window a hub poke can't reach) pulse the shared out-of-band
|
||||
trigger every 30 s, self-disarm on convergence; four cached sources (desired-gen==0, reconcile
|
||||
Planned−Pending>0, pbsdr waiting_secret ONLY, wgtunnel desired-not-operational). Seams:
|
||||
`reconcile.Engine.LastResult()` + `wgtunnel.Manager.TunnelConvergence()` (cached — no per-tick exec).
|
||||
(3) **Guests-0/0** REFUTED live: the 0/0 was the pre-provision window (guest not yet created), not a
|
||||
pool-membership bug; the fast-tick shortens that window. **OPEN (operator GO):** publish 0.90.0 +
|
||||
hub Day-0 manifest vouch + MinAgent-floor raise to 0.90.0 (password-gated UI; the safety gate — both
|
||||
agents on 0.90.0 — is satisfied and the coupling is proven live via the version header). See REPORT.md.
|
||||
|
||||
- **v0.89.0** (2026-07-16) — **agent train: three bundled agent-plane items; built + published to
|
||||
Gitea (sha256 `3969fd91…`); paired with hub 0.59.0 (LIVE).** (1) **pbsdr self-grant (R-22)** —
|
||||
closes the F4 self-deadlock: a 403 on the token-auth `StorageEntry` pre-check now self-grants via
|
||||
the root wrapper + re-reads instead of aborting before the grant (the demo's `felhom-offsite`
|
||||
case). (2) **escrow config live-reload** — `/escrow/preflight`'s `pbs_storage_id` row now reads
|
||||
the live agent.json (late-bound `CurrentPBSStorageID`) so a pbsdr-seeded id flips green with no
|
||||
restart. (3) **agent-plane poke listener (Direction-2a)** — `internal/poke`: contentless UDP poke
|
||||
bound to the box WG /32 (port **51822**), leading-edge debounced, fires the hub-loop out-of-band
|
||||
trigger for an immediate desired-state cycle; enabled with `wg_tunnel.enabled`; first slice of
|
||||
R-13. Red-proofs for all three (run-fail-revert). **ALL THREE LIVE LEGS PROVEN on the demo
|
||||
(2026-07-17), demo now LIVE on 0.89.0:** Scenario 4 floor-driven A/B train 0.88→0.89 (operator
|
||||
signed+enqueued the `agent_update` op — the vouch+floor alone does NOT trigger it; committed, no
|
||||
rollback); Scenario 1 R-22 self-heal (marker aside + ACLs revoked → `pre-check 403 … self-granting
|
||||
(R-22)` → `converged state=adopted` in ~3 s, ACLs restored, offsite active); Scenario 3 poke→tick
|
||||
~31 ms ep0→box + immediate report cycle (save→tick ≈ ~0.45 s). Details: REPORT.md.
|
||||
|
||||
- **v0.88.0** (2026-07-13 eve) — **controller-driven escrow ceremony (agent half), LIVE on demo
|
||||
host + drill VM (63/63 capabilities both).** `--output=json` machine mode (text mode
|
||||
byte-identical; extraction into `escrowCeremony()`); the ONE fixed argv
|
||||
(`escrow.CeremonyArgs()` — shared by the localapi exec + the `escrow-ceremony` capability
|
||||
(Critical, pbs_dr-gated EXPLICIT) + the new `FELHOM_ESCROW` sudoers alias, three-way
|
||||
pin-tested); localapi job endpoints (`POST /escrow/ceremony` single-flight 60 s,
|
||||
status, ONE-SHOT claim → 410, 10-min TTL → `unclaimed_void`, `GET /escrow/preflight`).
|
||||
R in-memory ONLY (never the job struct — snapshot-hygiene-tested; restart loses it safely).
|
||||
Live-proven on drill endpoint-exact: stage → preflight all-green (live FELHOM_ESCROW
|
||||
list-probe) → job ~4 s → hub blob `restic_pw_sha256` covering (repaired the spike's hash-less
|
||||
blob) → claim 200 once → 410. Coupled: controller v0.127.0 (MinAgent 0.88.0 for the wizard).
|
||||
**OPEN: publish 0.88.0 + Day-0 manifest vouch (operator) at the next train; deployed hosts got
|
||||
direct deploys.** Details: REPORT.md + felhom.eu RUNBOOK-escrow-ceremony.md (F1 threat model).
|
||||
|
||||
- **v0.87.0** (2026-07-13) — **SystemDisks device-mapper walk (IA finding 2, MEDIUM):
|
||||
legacy-boot hosts get a working drive wizard.** Operator ruling (approved 2026-07-13,
|
||||
verbatim): *resolve device-mapper/raid parents — for the root filesystem's backing block
|
||||
device, walk `/sys/block/<dev>/slaves` recursively down to physical disks; those, plus any
|
||||
ESP holder when present, are system. Disks outside that set become wizard candidates (still
|
||||
subject to the existing data-bearing guards). The all-system fail-safe remains ONLY for walk
|
||||
failure — it returns to being the error case, not the legacy-boot common case.* Implemented as
|
||||
`physicalDisksOf`/`walkSlaves` + `HostReader.BlockSlaves` (one seam method); per-branch
|
||||
conservatism (any unresolvable slave → ok=false → unchanged all-system path); signature test
|
||||
`TestSystemDisks_WalkTopologies` (root-backing disk ALWAYS system — never weaken). §3 spike
|
||||
transcripts: drill (legacy) dm-1→sda3→sda; felhom-pve (EFI+LVM) ESP+walk agree on sda →
|
||||
byte-identical regression. §13.2 wizard leg COMPLETE (offered → enrolled → formatted →
|
||||
torn down, boxes as found) + Day-0 manifest vouched to 0.87.0 (operator). The leg also
|
||||
surfaced two CONTROLLER bugs (fixed same-day: v0.126.3 claimed-box wizard CSRF, v0.126.4
|
||||
502-through-CF + native-alert ban).
|
||||
|
||||
- **v0.83.0** (2026-07-11, LIVE on felhom-pve; NOT published — Peti stays 0.81.0) — **observability
|
||||
pass** (pairs with controller v0.116.1 + hub v0.46.0). `applog.New` → `(logger, *Ring)`: slog
|
||||
fan-out, journald at the configured level, ~1000-entry ring FIXED at DEBUG. `GET /debug/logs`
|
||||
(local API, token-authed; the controller Debug page's Ügynök tab) + request-level DEBUG
|
||||
middleware. Heartbeat log-pull: envelope `log_tail_requested` → next heartbeat ships `log_tail`
|
||||
(128 KB, consume-once; failed push re-armed by the next envelope; `operator log pull served` INFO
|
||||
on fulfillment). Gap-fill sweep: netverify phase/verdict lines (job start, trigger outcome,
|
||||
/proc/mounts verdict, journal bytes, classification code, rollback outcome, durations), netmount
|
||||
unit steps, signedjobs op-received (class/host/expiry — never signatures) + fetch duration,
|
||||
selfupdate invariants + download sha/duration, disks outcome INFOs, controller-swap pre-pull +
|
||||
health verdicts, desired/loop per-exchange DEBUG. Logging conventions:
|
||||
`felhom.eu/documentation/runbooks/logging-conventions.md`. OPEN: the hub-side live pull awaits
|
||||
the operator's button click (hub UI password-gated); pre-existing lanresolver
|
||||
permission-denied WARN on /var/lib/felhom-agent/guests noted in REPORT.
|
||||
|
||||
- **v0.77.0** (2026-07-09) — **fork-4: escrow the offsite restic repo password under R.** `IdentityBundle`
|
||||
gains `ResticRepoPassword` (rides the existing age-under-R `WrapIdentityBundle` path — validated by the
|
||||
custody spike `febdc56`). New `POST /escrow/stage-secret` (`withGuest`) transiently stages the
|
||||
controller-pushed password (0600, never logged), which the `--selftest=escrow-create` ceremony
|
||||
auto-injects into the bundle and then wipes. `AttachResticPassword`/`StagedResticPasswordPath`/
|
||||
`WipeStagedResticPassword` added. Pairs with controller v0.105.0 (push + atomicity gate + DR inject +
|
||||
`DRResticCoord`). **NOT yet live-validated** — the supervised escrow ceremony is operator-run.
|
||||
|
||||
- **v0.76.0** (2026-07-08, LIVE on felhom-pve + **PUBLISHED sha `9828c5f7…f50b`** — THE Day-0
|
||||
manifest bump target; **0.75.0 superseded unpublished**) — **GL-5b / G12: restore-test
|
||||
full-fidelity**. Params derive from the ARCHIVE's embedded config (`drRestoreOverrides`, same as
|
||||
DR — the old live-source-config path verified the wrong object AND dropped storage mpN per PVE's
|
||||
all-or-nothing rule; deleted with `bindMountOverrides`/`archiveVMID`). NEW mount-parity assert
|
||||
(restored mpN vs archive; miss/mispath/undersize/extra = FAIL naming the delta) + `MountParity`/
|
||||
`MountInventory` on the wire record (additive). Live-proven: scratch 990000 ← 6.5GB 9201 archive,
|
||||
parity ok, inventory mp0 200G+mp1 50G+2 throwaways, **3m4s local tier** (cheaper than feared);
|
||||
rotated-out archive volid → clean up-front refusal (nice failure mode). bringup.go untouched.
|
||||
|
||||
- **v0.75.0** (2026-07-08, LIVE on felhom-pve) — **GL-5 / go-live G8: guest-loss DR bring-up
|
||||
actually restores** (closes the v0.74.0 OPEN item + SPIKE-dr-bindmount-source §8). DR passes the
|
||||
COMPLETE explicit restore param set derived from the archive's embedded config (NEW
|
||||
`Client.ExtractArchiveConfig`, 200 under the scoped token) — **two live-discovered PVE rules: mpN
|
||||
params need an explicit rootfs, AND unlisted mountpoints are silently DROPPED** (first run booted
|
||||
without mp0/mp1!) — storage mpN passed through, structural mp8/mp9 → throwaways, then step 4d
|
||||
swaps the REAL binds in via the host runner (root pct; new `EngineOptions.HostRunner`+`StateDir`
|
||||
seam) and deletes the unusedN residue. Scratch-DR live-proven end-to-end (9310 from a real 9201
|
||||
archive: mp0 200G + mp1 50G + real binds + no residue + clean teardown). Provision = nil
|
||||
overrides (regression-tested). NOTE: published/vouch-pending agent is 0.74.0 — publish 0.75.0
|
||||
before/with the manifest bump. OBSERVATION: the DR selftest hardcodes KeepMAC=true — a scratch DR
|
||||
while the SOURCE guest is live briefly duplicates its MAC on the bridge (pre-existing; fine for
|
||||
supervised runs, worth a -keep-mac flag someday). Full customer-data DR drill = GL-6/S5 family.
|
||||
|
||||
- **2026-07-07 — v0.74.0 Gitea-PUBLISHED (RUNBOOK GL-1)** — the LIVE felhom-pve binary's exact
|
||||
bytes, sha256 `1ec3f58842edce1e…76af05`, anon-fetch-verified. This supersedes/closes every
|
||||
standing "publish 0.6x + Day-0 vouch" OPEN item below (0.64→0.73 were never published; 0.74.0
|
||||
is the vouch target). Golden 0.103.0 published in the same run (felhom.eu execution record
|
||||
`documentation/pilot/RUNBOOK-GL1-publish-2026-07-07.md`). **Day-0 manifest vouch = operator
|
||||
step** (agent 0.74.0 / golden 0.103.0).
|
||||
|
||||
- **v0.74.0** (2026-07-07) — **campaign-2 R2 CLOSED; the mislabelled "R1" was a symptom** (LIVE on
|
||||
felhom-pve). Pool membership is what lets the pool-scoped token reach a guest; `pct restore --pool`
|
||||
sets it only at CREATE, so a restore-over-existing dropped 9201 from the `felhom` pool → no
|
||||
`VM.Audit` → restore-test's *existing* `bindMountOverrides` never ran → "mp8 … only possible for
|
||||
root". Fix: `Client.PoolAddVMID` + bring-up re-asserts membership post-restore (warn-not-fail).
|
||||
Role/ACL + `bindMountOverrides` untouched (both correct). **Live restore-test PASSED for the first
|
||||
time** once the pool was healed (Part A one-liner): read config → neutralize 2 binds → restore →
|
||||
boot+running → clean teardown, 4m35s. B3 (scratch-teardown 403) confirmed a cascade — no code.
|
||||
OPEN: DR `bring-up -mode dr` bind-override gap (spike `SPIKE-dr-bindmount-source-2026-07-07.md`:
|
||||
small known-constant override reusing `bindMountOverrides`; mp8/mp9 are structural constants).
|
||||
|
||||
- **v0.73.0** (2026-07-06) — **F2 mount-role fallback CLOSED** (LIVE on felhom-pve). `roleForMountPath`
|
||||
gained a mount-table fallback (Impl-2b style): a bind-mounted RAW enrolled user-data drive is not a PVE
|
||||
storage, so it fail-safe'd to `system` and the eject/decommission gates 403'd EVERY user-data drive
|
||||
(campaign F2, `where=/mnt/teszt_enroll role=system`). Device-keyed classification + whole-disk containment
|
||||
(`storage.SameWholeDisk`); Observe-error keeps the fail-safe BEFORE the fallback. Only `roleForMountPath`
|
||||
touched. Live-proven full lifecycle on teszt_enroll (eject/decommission 200, no-rebind across restart,
|
||||
end==pre). OPEN follow-up: the `deviceRole`/`roleForMountPath` unification refactor (deferred).
|
||||
|
||||
- **v0.72.0** (2026-07-05) — **OOB operator access (merged E1+H1)** — TASK H1, provenance both
|
||||
`SPIKE-{felhom-sshd,oob-wg-operator-peer}-2026-07-05`. Operator `/32` RENDERED into wg-felhom
|
||||
AllowedIPs (survives self-heal, [OF-1]); dedicated `internal/felhomsshd` (port claim + config
|
||||
render→sshd -t→reload + operator authorized_keys + heal + oob heartbeat stanza); static
|
||||
`inet felhom_oob` belt (agent mutates SET ELEMENTS ONLY); `configs/felhom-sshd.service` (NO
|
||||
RuntimeDirectory [SF-1]) + `felhom-oob.nft` + `felhom-op.sudoers`; `FELHOM_SSHD`+`FELHOM_OOB`
|
||||
grants; `oob.enabled` DEFAULT FALSE. Live on felhom-pve (8822, belt filled, operator SSH as
|
||||
felhom-op with scoped sudo); hub v0.35.0. Rollback `.bak-0.71.0`. 5 live-found bugs fixed (port
|
||||
path, self-listen flip-flop, nil-block lockout, reachable-via-dial, operator-configured source).
|
||||
- **v0.71.0** (2026-07-05) — **management-plane break-glass: privsep-dir watchdog + mgmt_plane
|
||||
health** — TASK G1 (prereq for felhom-sshd/H1), provenance `SPIKE-felhom-sshd-2026-07-05` §8.
|
||||
Host artifacts (`configs/felhom-privsep.tmpfiles` + `felhom-mgmt-watchdog.{sh,service,timer}`) make
|
||||
`/run/sshd` boot-persistent AND auto-heal it every ~60s **agent-independently** (heals with the
|
||||
agent stopped — proven live: `/run/sshd` removed → restored in 30.0s, `:22` back, no login).
|
||||
`internal/mgmtplane` reports the additive `mgmt_plane` heartbeat stanza; hub v0.34.1 raises
|
||||
`mgmt_plane_healed`. **NO unit declares `RuntimeDirectory=`** (the incident cause). H1 may now
|
||||
assume `/run/sshd` is guaranteed present. Live on felhom-pve; rollback `.bak-0.70.0`.
|
||||
- **v0.70.0** (2026-07-05) — **agent self-update (operator-signed A/B slots + crash-loop
|
||||
auto-rollback)** — TASK D1, provenance `SPIKE-agent-selfupdate-2026-07-05`. An operator-signed
|
||||
`agent_update` op (version+sha256, sha is the only integrity root) rides the signed-jobs gate;
|
||||
`internal/selfupdate.Executor` downloads+verifies+hands to `felhom-selfupdate-guarded apply` (root
|
||||
re-verify → A/B atomic flip → pending marker → detached restart); the new binary commits after a
|
||||
60s dwell; a crash-looping binary is auto-reverted by `OnFailure=felhom-agent-rollback.service`
|
||||
(first-crash trigger [SF-1]) with the tuned `[Unit]` start-limit (120s/4) as backstop. Host
|
||||
artifacts + sudoers `FELHOM_SELFUPDATE` + `felhom-host-install.sh` day-0 install + report field
|
||||
`selfupdate_pending`. Green tests + companions. **LIVE-VALIDATED on felhom-pve (2026-07-05): all 4
|
||||
drills PASS** — happy path (0.70.0→0.70.1 signed op → download+verify+flip+commit), crash-rollback
|
||||
(0.70.2-crash → OnFailure → **~2s crash-to-recovered**, byte-identical revert, no loop), no-pending
|
||||
guard, gate refusal (non-pinned key). Full agent-side pipeline ran real (envelope injected into the
|
||||
hub `signed_jobs` queue — CC lacks the hub global operator key; hub enqueue-auth is hub-unit-tested).
|
||||
Box restored to canonical **v0.70.0** (host artifacts KEPT installed; scratch operator key REMOVED —
|
||||
self-update dormant until an operator pins a real key, a Day-0-vouch-style follow-up). Rollback
|
||||
`felhom-agent.bak-0.69.0`. OPEN (v1 scope-outs): no hub-floor auto-update, no failed-update
|
||||
auto-retry, no pending-timeout auto-rollback; per-crash OnFailure can double-fire (idempotent —
|
||||
future: serialize the rollback oneshot). Detail: REPORT.md.
|
||||
|
||||
- **v0.69.0** (2026-07-04, live on felhom-pve) — **S5: host-loss DR — safe halves shipped**.
|
||||
**Part 1** `wgtunnel.InstallRecoveredKey` — writes an escrow-recovered WG privkey (create-only,
|
||||
refuse-overwrite) so the tunnel re-establishes with the SAME identity/pubkey (same /32), no keygen;
|
||||
wired into `--selftest=identity-consume -install-wg-key` (opt-in; pre-S3 blob → logged fresh-keygen
|
||||
fallback). **Part 2** new `internal/dr` — consumes the host_loss `restore_directive` (was
|
||||
logged-ignored) into an inspectable RestorePlan via AddConsumer: per-guest {vmid,archive,target,
|
||||
sizing} + per-drive {durable_id→mount} + offsite PBS coord; DERIVE-AND-SURFACE only (Consumer has
|
||||
no restore/destroy dep — execute-nothing is structural). Tests + red-proofs (WG create-only; plan
|
||||
mode-gate). **Part 3** hub escrow-GET NOT needed (operator exports the blob via `sqlite3 writefile`
|
||||
on a cp'd hub.db). **Part 4-A** re-attach wrong-disk safety already unit-proven
|
||||
(`ResolveStorageDevice`: match resolves, absent/mismatch ERRORS, non-uuid scheme refused — never a
|
||||
near disk). **Part 4-B (destructive in-place 9201 restore) PREPARED + OPERATOR-GATED, NOT executed**
|
||||
— pre-flight green (offsite ct/9201 restorable per S4.1); the operator runs the R-consume steps +
|
||||
confirms the destroy (§9-4a: CC never runs a consume/R command — see [[operator-present-one-time-secrets]]).
|
||||
OPEN: the operator-run 4-B drill; guest_loss DR; hub-driven full-auto DR. Rollback
|
||||
felhom-agent.bak-0.68.0. Detail: REPORT.md + doc-06 §3.5/S5.
|
||||
- **v0.68.0** (2026-07-04, live on felhom-pve) — **S4.1: unattended offsite restore-test**.
|
||||
**Tier-aware restore-task deadline:** `RestoreTestSpec.RestoreTaskTimeout` (0→10m default) from
|
||||
`config.RestoreTestPBSRestoreTimeoutSeconds` (accessor default **120m**), set only when
|
||||
`SourceTier=="pbs"` (`main.restoreTaskTimeout`); local tier UNCHANGED. Fixes the WAN restore being
|
||||
killed at 10m → mid-restore teardown → leaked scratch. **Teardown "VM.Allocate" follow-up =
|
||||
PHANTOM (diagnosed, not blind-fixed):** ran the restore-test on the AGENT-TOKEN path sourcing the
|
||||
offsite (pbs) backup → `pass:true verified:boot+running`, teardown succeeded (`torn down
|
||||
vmid=990000`, no 403), scratch band clean. The earlier 403 was the 10m-timeout consequence (guest
|
||||
not yet pool-associated); the scratch is restored INTO `/pool/felhom` (ACL already grants
|
||||
VM.Allocate) so teardown is authorized once the restore completes. **No ACL/host-install change.**
|
||||
OPEN: publish 0.68.0 + Day-0 vouch; Tier-1/Tier-2 split for offsite-as-default; S5 DR consume.
|
||||
Rollback `felhom-agent.bak-0.67.0`. Detail: REPORT.md.
|
||||
- **v0.66.0 + v0.67.0** (2026-07-04, live on felhom-pve) — **S4: PBS over the tunnel**. **v0.66.0**:
|
||||
wgtunnel **v4-pin** (renderConf writes the resolved A LITERAL, never DNS/AAAA; `Resolver` seam,
|
||||
lowest addr; cached → steady-state zero-DNS/zero-exec) + **re-resolve watchdog** (`Manager.Watchdog`,
|
||||
loop-only; handshake stale > `stale_after_seconds`=180 → re-resolve → IP-changed re-render+restart)
|
||||
+ FELHOM_WG **Critical** flips (conf-install/enable/restart/handshake-read). **v0.67.0**:
|
||||
**namespace-aware PBS client** (Config.Namespace → `Snapshots ?ns=`, `Verify ns=`; root-ns
|
||||
unchanged) — the operator-approved fix after Phase-1 showed the ns-unaware datastore-root 403s a
|
||||
per-tenant token. **Live Scenario-D (all green):** real vzdump of 9201 → **ciphertext** in ns
|
||||
`demo-felhom-01` over the tunnel; ns-scoped verify=ok under the box's own `felhom@pbs!demo-felhom-01`
|
||||
token; WARN gone; restore round-tripped (decrypt with box-born key → boot → teardown).
|
||||
**Confirmed tenant ACL (felhom-hetzner):** `DatastoreBackup` on `/datastore/felhom-offsite/<ns>`
|
||||
(NOT `/ns/<ns>`) to BOTH user `felhom@pbs` AND token (privsep=intersection; cross-ns 403);
|
||||
DatastoreBackup can't prune (safety). **FINDINGS:** retarget field is `local_backup_target` (not
|
||||
`backup_target`); retarget REVERTED to `local` (controller backs up ~every 30 min → single-target
|
||||
offsite = near-continuous 20-min uploads; needs Tier-1/Tier-2 split); restore-test scheduler needs
|
||||
a WAN restore deadline + scratch-band `VM.Allocate` before it runs offsite unattended. **OPEN:**
|
||||
escrow-create (OPERATOR-PRESENT, new R); publish 0.66/0.67 + Day-0 vouch; S5 DR consume. Rollback:
|
||||
`felhom-agent.bak-0.65.0`/`.bak-0.66.0`. Detail: REPORT.md + doc-06 §3.4/§4.2 + runbook §4a/§4b.
|
||||
- **v0.65.0** (2026-07-04, live on felhom-pve) — **S3.1 offsite-tunnel client MTU 1420 → 1280**:
|
||||
resolves `06 §4.3`'s OPEN DECISION left by the CGNAT smoke test. 1420 **silently black-holed bulk
|
||||
TCP** on sub-~1480 paths (mobile ~1400, DS-Lite ~1452) — handshake+ping healthy, PBS TLS page
|
||||
(and at S4 the backup itself) drops. New `const clientMTU = 1280` (RFC 8200 IPv6-minimum floor;
|
||||
outer 1340 v4 / 1360 v6 fits every realistic path), **permanent + fleet-wide + family-agnostic**.
|
||||
**Client-only by construction** — interface MTU caps box→PBS, advertised MSS caps PBS→box, so the
|
||||
endpoint's `wg0` is untouched (zero live-endpoint risk). Golden pins exact `MTU = 1280`
|
||||
(red-proofed vs a 1420 flip); no wire/JSON change. Live: agent re-rendered on restart (hash-gated
|
||||
apply), conf + live iface both 1280, PBS page loads at 1280 (no regression on wired). OPEN:
|
||||
true-CGNAT-SIM retest (low risk); publish 0.65.0 + Day-0 vouch (operator); S4 PBS-over-tunnel.
|
||||
Rollback: `felhom-agent.bak-0.64.0` on the box. The v4-pin (§4.2 determinism) is a separate,
|
||||
optional future note — NOT needed for MTU correctness.
|
||||
- **v0.64.0** (2026-07-04, live on felhom-pve) — **S3 offsite WG tunnel**: new `internal/wgtunnel`
|
||||
(keygen 0600/0700, marker-gated one-shot registration, agent-managed `wg-quick@wg-felhom` from
|
||||
the hub's desired-state `wireguard` block via the new `desired.Syncer.AddConsumer` seam,
|
||||
revoked-stays-revoked teardown, report stanza) + `FELHOM_WG` sudoers/capabilities +
|
||||
`IdentityBundle.WGPrivateKey` escrow auto-inject. **`wg_tunnel.enabled` DEFAULTS FALSE** (safety
|
||||
gate — rollout to Peti's box is a no-op until the production endpoint exists; enabled explicitly
|
||||
on felhom-pve only). Live: tunnel to ep0.felhom.eu:443 up 3 s after enable (PBS page through
|
||||
10.77.0.1:8007), reboot-persistent, revocation drill clean, 30-min keepalive soak. GOTCHAS:
|
||||
hub envelope poll_interval_seconds (hub-side const 900 s) silently overrides agent poll_seconds
|
||||
on cycle 1; `wg show <if> dump` leaks the PRIVATE key (forbidden everywhere — sudoers only
|
||||
grants `latest-handshakes`). OPEN: CGNAT/mobile-hotspot smoke (operator-assisted appendix);
|
||||
publish 0.64.0 to Gitea + Day-0 vouch (operator); S4 points PBS at the tunnel.
|
||||
- **configs: build-golden.sh v2.0.0** (2026-07-03, @ `ceca355`; no agent version change) — **drill
|
||||
findings B5 + B1 FIXED** (`DRILL-golden-098-2026-07-03.md`): the controller tag is a MANDATORY
|
||||
argument (the default rotted twice — a fresh install booted a pre-floor controller, forcing the
|
||||
guide's manual D.1b update) and the golden now bakes a `felhom-controller-bootstrap.path` unit
|
||||
(controller deploys the moment the back-half hot-plugs the bootstrap mount — no reboot; installer
|
||||
v1.9.1's reboot is a redundant belt, kept). **Golden 0.98.3** baked on the drill VM, clean-room
|
||||
validated (bake integrity → isolated hot-plug proof → local-golden Day-0 → published-artifact
|
||||
Day-0), published (sha256 b9a02ef1…fd01) + operator-vouched — Day-0 manifest now vouches
|
||||
**agent 0.63.0 + golden 0.98.3** (the v0.63.0 vouch follow-up below is DONE). Fresh installs land
|
||||
current and self-manage. NEW operator follow-up (SECURITY): the customer-config `git.token` has
|
||||
Gitea package-WRITE rights — scope down + rotate (evidence-doc observation O1).
|
||||
- **v0.63.0** (2026-07-03, live on felhom-pve + Gitea-published sha256 b4a89c81…) — **drill findings
|
||||
B3 + B2 FIXED** (`DRILL-day0-cleanroom-2026-07-03.md`): `TokenStore.Lookup` reloads the append-only
|
||||
store once on a miss (cross-process coherence with the one-shot provisioner — no more fresh-install
|
||||
`/controller/swap` 401 / manual restart; size short-circuit bounds the cost; behind the
|
||||
`TokenAuthority` seam) + `guesthook.InstallSnippet` issues a fenced `mkdir -p /var/lib/vz/snippets`
|
||||
first (fresh boxes lacked the dir → the self-heal hook silently never installed). Sudoers gained
|
||||
exactly that one grant — **ship sudoers WITH the binary** (done on felhom-pve). Red-proofed both;
|
||||
Scenario-E method: compiled test suite run ON felhom-pve + live channel-health hit-path.
|
||||
**OPERATOR FOLLOW-UP: bump the hub Day-0 manifest to agent 0.63.0** — until then fresh installs get
|
||||
0.62.0 and the guide's D.1b restart-first step still applies (narrowed to "< v0.63.0" in the guide).
|
||||
- **v0.62.0** (2026-07-03) — **audit A1 RESOLVED**: the stale-lock reaper's scan is now
|
||||
pool-intersected (`staleLockController.Guests()` = `ListLXC` ∩ `Client.Pool("felhom")` members),
|
||||
fail-safe skip on pool-read failure; `pve:pool-read` capability (non-critical) + `--selftest`
|
||||
"pool read" line. Companion host-install **v1.9.0** adds `Pool.Audit` to `FelhomAgentGuest` —
|
||||
**deploy order on any box: rescope ACL first, then this agent.** Per
|
||||
`SPIKE-a1-pool-membership-read-2026-07-03.md`; red-proofed tests in stalelock_pool_test.go.
|
||||
- **2026-07-03 — CLAUDE.md refreshed**: version narrative removed (state lives HERE + CHANGELOG top), layout completed (all 17 internal packages + cmd/felhom-opsign); deploy runbook now in the `felhom-build-deploy` skill (`felhom.eu/skills/`).
|
||||
- **2026-07-03 — `REUSE.md` exists at the repo root** (canonical helpers / format-safety guards / traps / seams, code-verified); maintenance rule active: update it in the same commit that changes a shared helper.
|
||||
- **v0.61.0** (2026-07-03) — blast-radius audit fixes **B1 + D1 + D2 + D3** from
|
||||
`felhom.eu/documentation/audits/AUDIT-blast-radius-hostroot-localapi-2026-07-02.md`: random temp
|
||||
staging for root-installed scripts (+ sudoers/manifest glob updates), mkfs-wrapper member/RO
|
||||
re-checks (validated by `scripts/mkfs-guarded-harness.sh`), classifyClaim empty-lsblk fail-safe,
|
||||
and the blank-format anti-retarget (durable-id-bound, AGENT-001's benign-branch twin).
|
||||
- Deployed on demo host `felhom-pve` (node `demo-felhom`), non-root `felhom-agent` service user,
|
||||
pool-scoped token (`felhom` pool).
|
||||
|
||||
## Open threads
|
||||
|
||||
- Deferred audit items (housekeeping/design, all INFO): C1 (controller-swap version floor), C2 (NAS
|
||||
server allowlist), A2 (gate journal cross-check), B2–B5, E1/E2.
|
||||
- Drive-enrollment leftovers: (a) `runStorageInit` slow-device detached-format polling; (b) Impl-3
|
||||
shared-box operator format gate.
|
||||
- BUNDLE leftover: non-root agent can't read the PBS key; migration must preserve cert/key/tokens.
|
||||
- Not run (needs a supervised session): the destructive D1/D3 live proofs (real mkfs on a crafted
|
||||
member; a live /dev re-enumeration race during a real format).
|
||||
@@ -0,0 +1,136 @@
|
||||
# Slice 6 Phase A — backup + restore-test orchestration (felhom-agent)
|
||||
|
||||
## Context
|
||||
|
||||
The agent can observe/report storage (slice 5) and reconcile benign guest ops behind a
|
||||
signed-op gate (slice 4), but it has no guest-level **backup/restore** layer and — the point
|
||||
of this slice — no **self-restore-test**, which closes the "a backup you haven't restored
|
||||
isn't a backup" theme (doc 03 §8). This adds: a vzdump backup primitive that resolves the
|
||||
produced archive, a benign restore-to-a-**new** guest, and a journaled **self-restore-test**
|
||||
(restore → boot → verify → teardown) that inherits the slice-4 journal/serialization/crash
|
||||
recovery so a mid-test crash can't leak a scratch guest. Provisioning/identity-reset/golden
|
||||
base (§9) are **slice 7**; PBS/offsite/zero-knowledge (§8 offsite tier) are **Phase B**.
|
||||
|
||||
Everything here is **benign** (backup, restore-to-new, scratch teardown): reuse the slice-4
|
||||
classifier/gate/journal — **no new destructive class, no new crypto**. Version → **v0.6.0-rc1**,
|
||||
stop at the checkpoint.
|
||||
|
||||
**Locked decisions (from the task + clarifications):**
|
||||
- Backups are **crash-consistent only** (marked so in the report); app-consistency needs the
|
||||
controller quiesce (slice 8).
|
||||
- Restore is **to a NEW guest only** (no overwrite anywhere this slice).
|
||||
- Restore-test verify = **net link-down (benign SetConfig) → boot → reaches `running`** (no
|
||||
in-guest probe — slice 8). Link-down avoids the cloned source MAC/IP conflicting on a live
|
||||
host; it is test-safety, **not** slice-7 identity reset.
|
||||
- Restore-test cadence **defaults ON at 24h**, configurable; runs only when a valid scratch
|
||||
VMID band is set; also on-demand via the selftest harness.
|
||||
- Scratch VMID band **990000–990009** (lowest-free, excludes 9999/real guests); refuse to run
|
||||
if unset/invalid.
|
||||
- **Local target only** this phase; `PBSSnapshot` stays a `struct{}` stub (Phase B).
|
||||
- Bulk volumes: **report the gap only** (which `backup=0` mountpoints the guest vzdump omits).
|
||||
|
||||
## The crash-safe core (load-bearing)
|
||||
|
||||
The restore-test is journaled as a **single** entry whose `VMID` is the scratch id and
|
||||
`Kind = "scratch_restore_test"` with a new `Scratch bool` flag. The entry is **terminal only
|
||||
after teardown** — NOT when the restore sub-task's UPID completes. `Recover` must special-case
|
||||
Scratch entries **before** the generic UPID-recheck path (else it would mark the entry
|
||||
`succeeded` because the restore task ran OK, and drop it while the guest still exists → leak).
|
||||
|
||||
`RunRestoreTest` (pseudocode) — `defer` teardown so it runs on every path incl. failed verify:
|
||||
```
|
||||
vmid := pickScratchVMID(ListLXC, [min,max], exclude 9999) // refuse if band unset/invalid
|
||||
opID := "scratch-restore-"+vmid+"-"+seq
|
||||
append({OpID:opID, VMID:vmid, Kind:"scratch_restore_test", Scratch:true, State:OpStarted}) // BEFORE any mutation
|
||||
defer teardownScratch(opID, vmid) // gate.Authorize(IntentForScratchDestroy)=benign -> DestroyLXC -> WaitTask -> terminal
|
||||
upid := RestoreLXC{VMID:vmid, Archive, Storage:RestoreStorage}; waitOK(upid) // UPID = error-detection, NOT terminal
|
||||
for net := range GuestConfig(vmid).Nets(): SetConfig(vmid, {net: existing+",link_down=1"}) // benign
|
||||
Start(vmid); res.Reached = waitRunning(GuestStatus, vmid, BootTimeout)
|
||||
res = {Archive, ScratchVMID:vmid, Pass:res.Reached, Verified:"boot+running", Duration, Err}
|
||||
return res // defer tears down regardless
|
||||
```
|
||||
|
||||
`Recover` scratch branch (added before the existing UPID logic):
|
||||
```
|
||||
if entry.Scratch {
|
||||
guests, err := ListLXC; if err { Unresolved++; continue } // can't decide -> leave in-flight
|
||||
if !contains(guests, entry.VMID) { append(terminal(succeeded)); ScratchClean++; continue } // already gone
|
||||
dec := gate.Authorize(IntentForScratchDestroy(hostID, entry.VMID), nil)
|
||||
if !dec.Allowed { Unresolved++; continue } // fail-safe (should be benign)
|
||||
upid, err := DestroyLXC(entry.VMID); if err { Unresolved++; continue } // retry next Recover
|
||||
waitOK(upid); append(terminal(succeeded)); ScratchDestroyed++
|
||||
}
|
||||
```
|
||||
Idempotent: crash mid-destroy → next `Recover` finds the guest gone → `ScratchClean`. Both
|
||||
teardown paths (normal + recovery) go through `gate.Authorize` (benign `ClassGuestDestroy` +
|
||||
`Provenance{AgentTaggedScratch:true}`) for the audit trail.
|
||||
|
||||
## Files to create
|
||||
|
||||
- **`internal/reconcile/restoretest.go`** (+ `restoretest_test.go`): `RunRestoreTest(ctx, RestoreTestSpec{Archive, RestoreStorage, BootTimeout}) RestoreTestResult` (engine method — it needs the journal/gate/queue internals), `RestoreTestResult` (reconcile-local data — must NOT return `hub.RestoreTest`, to avoid a reconcile→hub edge), `IntentForScratchDestroy(hostID, vmid)`, `pickScratchVMID`. Runs the sequence on the scratch VMID's `Queue` lane.
|
||||
- **`internal/backup/{doc,runner,store,schedule}.go`** (+ tests):
|
||||
- `runner.go`: `BackupRunner.Backup(ctx, vmid) (hub.Backup, error)` = `Vzdump`+`WaitTask`+volid-resolve + bulk-gap from `GuestConfig.MountPoints()` `backup=0`; `LatestBackup(ctx, target) (volid, error)`; the `RestoreTestRunner` seam (satisfied by `*reconcile.Engine`); `RestoreTestResult → hub.RestoreTest` mapping.
|
||||
- `store.go`: mutex-guarded in-memory latest-`Backup`-per-target + latest-`RestoreTest`, implementing the hub `BackupReporter`/`RestoreTestReporter` seams.
|
||||
- `schedule.go`: cadence goroutine (default 24h; 0=disabled) → `LatestBackup` → `engine.RunRestoreTest` → write `Store`. No-ops cleanly when no backup exists yet.
|
||||
- Imports `reconcile`+`hub`+`proxmox` (acyclic; hub imports neither).
|
||||
- **`configs/`**: example agent config with the `backup` block.
|
||||
|
||||
## Files to modify
|
||||
|
||||
- **`internal/proxmox/mutate.go`**: `DestroyLXC(ctx, vmid)` → `DELETE /nodes/{node}/lxc/{vmid}` with `purge=1&destroy-unreferenced-disks=1` via `dataString` (async→UPID); add `Notes string` to `VzdumpOptions` → `notes-template` param (probe `notes-template` vs `notes` on the demo PVE 9.2.2 first, read-only).
|
||||
- **`internal/proxmox/query.go`**: `LatestBackupVolID(ctx, store, vmid) (string, error)` — `StorageContent` filtered `Content=="backup" && VMID==vmid`, max `CTime`.
|
||||
- **`internal/reconcile/state.go`**: add `RestoreLXC`, `DestroyLXC`, `GuestStatus` to the `GuestAPI` interface (`*proxmox.Client` already satisfies all three).
|
||||
- **`internal/reconcile/journal.go`**: add `Scratch bool \`json:"scratch,omitempty"\``; carry it (and `Kind`) through `terminal()`.
|
||||
- **`internal/reconcile/recover.go`**: the scratch branch above; add `ScratchClean`/`ScratchDestroyed` to `RecoverResult`.
|
||||
- **`internal/reconcile/recover_test.go`** + **`engine_test.go`**: `fakeAPI` gains `RestoreLXC`/`DestroyLXC`/`GuestStatus` recorders; new `TestRecover_LeakedScratchDestroyed` (in-flight Scratch + ListLXC returns the VMID → DestroyLXC called, gate benign, no longer in-flight), `…AlreadyGone` (ScratchClean), `…ListUnreadable` (Unresolved).
|
||||
- **`internal/hub/report.go`**: fill `Backup` + `RestoreTest` structs (see wire shapes); `PBSSnapshot` stays `struct{}`.
|
||||
- **`internal/hub/collect.go`**: add `BackupReporter { Backups(ctx) []Backup }` + `RestoreTestReporter { RestoreTests(ctx) []RestoreTest }` consumer seams (mirror `StorageObserver`); `collectBackups`/`collectRestoreTests` degrade nil/err → non-nil empty.
|
||||
- **`internal/hub/contract_test.go`** + **`internal/hub/testdata/host-report.golden.json`**: populated `backups[0]`/`restore_tests[0]`; assert their key sets (bidirectional, slice-5 pattern).
|
||||
- **`internal/config/config.go`**: `BackupConfig{RestoreTestCadenceSeconds, ScratchVMIDMin, ScratchVMIDMax, LocalBackupTarget, RestoreStorage}` + `RestoreTestCadence()` accessor (0→24h default) + env overlay; validate band (min>0, max>=min, 9999 excluded) only when cadence>0. `Config.Validate` stays proxmox-only.
|
||||
- **`cmd/felhom-agent/main.go`**: wire `backup.Store` into `NewCollector`; add the cadence goroutine alongside `engine.Run`/`loop.Run`/`watchdog.Run`; add `backup` + `restore-test` to `selftestFlag.Set` + the switch + a `-archive` flag; `runSelftestBackup`/`runSelftestRestoreTest` reuse the `runSelftestStorage` wiring (`NewGate(nil, hostID, SlogAudit{}, logger)` + journal + engine).
|
||||
- **felhom.eu/hub**: `hub/internal/api/handler.go` — add `hostBackup`/`hostRestoreTest` mirror structs to `hostReportPayload`, parse, persist via existing `report_json` (no new DB columns — slice-5 precedent), and **log a FAILED restore-test prominently** (`[WARN]`, the loudest DR signal). `hub/internal/api/testdata/host-report.golden.json` byte-identical with the agent golden; `host_test.go` adds `TestHostBackup_GoldenContract`/`TestHostRestoreTest_GoldenContract` (bidirectional key-set, slice-5 pattern).
|
||||
- **`CHANGELOG.md`** (prepend v0.6.0-rc1), **`REPORT.md`** (overwrite), `CLAUDE.md` current-state line.
|
||||
|
||||
## Proposed wire shapes (draft — must land byte-identical in both repos)
|
||||
|
||||
```go
|
||||
type Backup struct {
|
||||
TargetID string `json:"target_id"` // backup storage name
|
||||
VMID int `json:"vmid"`
|
||||
Archive string `json:"archive"` // produced volid
|
||||
Mode string `json:"mode"` // snapshot|stop
|
||||
CrashConsistent bool `json:"crash_consistent"` // always true this slice
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Success bool `json:"success"`
|
||||
Error string `json:"error,omitempty"`
|
||||
StartedAt string `json:"started_at"` // RFC3339
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
UncoveredVolumes []string `json:"uncovered_volumes"` // backup=0 mountpoints (bulk gap)
|
||||
}
|
||||
type RestoreTest struct {
|
||||
SourceArchive string `json:"source_archive"`
|
||||
SourceTier string `json:"source_tier"` // "local" (pbs = Phase B)
|
||||
ScratchVMID int `json:"scratch_vmid"`
|
||||
Pass bool `json:"pass"`
|
||||
Verified string `json:"verified"` // "boot+running"
|
||||
Error string `json:"error,omitempty"`
|
||||
TestedAt string `json:"tested_at"` // RFC3339
|
||||
DurationSeconds float64 `json:"duration_seconds"`
|
||||
}
|
||||
type PBSSnapshot struct{} // Phase B stub
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
- `go test ./...` (local, Windows) + `go test -race ./...` on the build server (192.168.0.180, cgo) — the cadence adds a goroutine; the Store is mutex-guarded.
|
||||
- New unit tests: vzdump async UPID→WaitTask + volid-resolve (fake API); restore-to-new benign `ClassCreate` passes the gate; restore-test end-to-end against fakes incl. **(a) teardown-on-failed-verify** and **(b) journal-recovery cleanup** (extend the slice-4 recover test: in-flight Scratch → `Recover` destroys the leaked guest, idempotent when already gone); cadence fires on interval / no-ops when disabled; cross-repo golden + hub-ingest key-set tests.
|
||||
- Build the linux binary on 192.168.0.180, relay to `felhom-pve` (see memory `demo-felhom-live-agent`: `MSYS_NO_PATHCONV=1`, config at `/root/.config/felhom-agent/agent.json`, mode=direct, served-cert pin `BA:7C:99…`).
|
||||
- **Live on the demo** (the checkpoint validation): `--selftest=backup -vmid <small stopped guest>` to a local target (e.g. `felhom-usb` or `local`, content=backup) → print the `Backup` record; then `--selftest=restore-test -archive <volid>` → restore into a 990000-band scratch guest, net-link-down, boot, verify `running`, teardown → print the `RestoreTest` record. Confirm no leaked scratch guest remains (`pct list`), and that a simulated mid-test crash + restart triggers `Recover` teardown.
|
||||
|
||||
## Push & checkpoint
|
||||
|
||||
Push **v0.6.0-rc1** to felhom-agent (and the hub changes to felhom.eu, deploy per the GitOps
|
||||
runbook if needed), update CHANGELOG/REPORT, then **stop and await validation** (restore-test
|
||||
teardown + recovery, the benign classifications, reporting) + the live demo restore-test.
|
||||
**Phase B (next): PBS** — datastore on the USB, zero-knowledge key custody, restore-from-PBS,
|
||||
PBS integrity-verify as the lighter frequent check.
|
||||
@@ -0,0 +1,351 @@
|
||||
# felhom-agent
|
||||
|
||||
The **host agent** for the Felhom platform — the operator-tier component that runs on each
|
||||
Proxmox host and owns *all* Proxmox interaction (provision/restore guests, host storage,
|
||||
backups, host+tunnel monitoring, hub control loop, per-guest local API). Design:
|
||||
[`felhom.eu/documentation/architecture/03-host-agent.md`](https://gitea.dooplex.hu/admin/felhom.eu/raw/branch/main/documentation/architecture/03-host-agent.md).
|
||||
|
||||
> **Status — slice 1 of N.** This repo currently contains the project scaffold and the
|
||||
> **`internal/proxmox`** interaction layer (the typed library every other module will call to
|
||||
> talk to Proxmox), plus a runnable read-only `--selftest`. **No** reconcile loop, hub client,
|
||||
> signing, or storage/backup orchestration yet — those are later slices.
|
||||
|
||||
Module: `gitea.dooplex.hu/admin/felhom-agent` · binary: `felhom-agent` · Go 1.24.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
cmd/felhom-agent/ # entry point + --selftest modes + the daemon (poll loop + reconcile + watchdog)
|
||||
internal/proxmox/ # the Proxmox interaction layer (API-first + fenced root-CLI)
|
||||
internal/config/ # JSON config + env overrides (secrets never logged)
|
||||
internal/log/ # slog setup
|
||||
internal/authz/ # operator signed-op verifier (SSHSIG); durable nonce store
|
||||
internal/hub/ # daemon: host-report collector + Bearer client + resilient poll loop
|
||||
internal/reconcile/ # reconcile engine + reversibility gate + op journal + crash recovery
|
||||
internal/storage/ # storage-target observer + durable_id + fast-poll watchdog (slice 5)
|
||||
internal/capability/ # privileged-capability self-probe (manifest + sudo -n -l check, v0.44.0)
|
||||
configs/agent.example.json
|
||||
```
|
||||
|
||||
## The `capability` package — privileged self-check (v0.44.0)
|
||||
|
||||
The non-root agent depends on a fixed set of `sudo -n` grants (`configs/felhom-agent.sudoers`). When
|
||||
the 2026-06-28 root→non-root cutover silently dropped some (lxc-info, make-private, …), features broke
|
||||
unnoticed until a user hit them. `internal/capability` makes that loud:
|
||||
|
||||
- **`Manifest()` is the source of truth** for the required `(binary, representative-arg)` vectors,
|
||||
seeded from the sudoers audit (the OK + CLOSED rows; the surfaced/deferred rows are excluded). When
|
||||
you add a privileged `runner.Run` call that needs a NEW grant, add it to BOTH the sudoers file AND
|
||||
the manifest — `manifest_test.go` asserts **manifest ⊆ sudoers** at build time (with a red-proof
|
||||
that dropping the lxc-info grant fails the gate), so a missing grant is caught in CI, not in prod.
|
||||
- **`Prober.Probe`** lists each vector with `sudo -n -l` (a policy LIST — never executes, safe for
|
||||
mkfs/pct) + an `os.Stat` existence check → an `ok`/`degraded` snapshot. It runs at startup (logged
|
||||
loud) and on every collect, riding the hub report as `HostReport.Capabilities`; the hub
|
||||
(`HostCapabilityChecker`) alerts the operator on a Critical capability going degraded. Serve-degraded
|
||||
— the probe never blocks startup. (Next self-health slice: the controller↔agent channel check.)
|
||||
|
||||
**Controller-swap under non-root (v0.45.0; the write since R-861 (a) A1).** The agent-owned controller image swap
|
||||
(`internal/localapi/controllerswap.go`) no longer shells out. The write goes on **stdin** to the ROOT verb
|
||||
`felhom-priv-apply controller-image <vmid>` (`GuestBinder.WriteControllerImage` → `Runner.RunStdin`, the same fenced
|
||||
`sudo -n` runner), which re-checks the ref against our registry + repository + an x.y.z tag and writes
|
||||
`/etc/felhom-controller-image` inside the guest itself; the agent has no in-guest `tee` grant any more (before, a
|
||||
compromised agent could feed any image — sudo cannot see stdin). Its grants live in the `FELHOM_CONTROLLERSWAP` sudoers
|
||||
alias (read-only or fixed-target) and in the capability manifest (Critical), so a dropped grant is a build failure + a
|
||||
live degraded signal. No general `pct exec` is granted.
|
||||
|
||||
## The `storage` package — observe + watchdog (slice 5)
|
||||
|
||||
Read-only this slice (no hub desired-state until slice 10):
|
||||
|
||||
- **Observer** builds the host-report's `storage_targets` from `ListStorage`/`NodeStorage`
|
||||
joined with non-privileged host reads (`/proc/mounts`, `/dev/disk/by-uuid`,
|
||||
`/sys/.../rotational`). It reports each target's `durable_id` (the DR-load-bearing
|
||||
re-attach key: fs-UUID for usb/local-dir, `server:export` for nfs/cifs,
|
||||
`repo+fingerprint` for pbs, `vg/pool` for lvmthin), state, usage, a rotational
|
||||
**class hint** (never authoritative — class is hub-owned), and the lvmthin **thin-pool
|
||||
data fill** (a full pool corrupts every guest on it). SMART is a Phase-B privileged read.
|
||||
- **Watchdog** is the third daemon goroutine: a fast poll (seconds) over the *known*
|
||||
target set that detects an `attached↔disconnected` transition and fires a **debounced,
|
||||
out-of-band** host-report so the hub learns of a USB drop in seconds rather than at the
|
||||
~15-minute cycle. It mutates nothing (the benign re-mount-by-UUID response lands in
|
||||
Phase B). The `HostReader` seam keeps it root-free and unit-testable with no real devices.
|
||||
|
||||
The reported `StorageTarget` shape is a cross-repo contract duplicated in `felhom.eu/hub`;
|
||||
`internal/hub/testdata/host-report.golden.json` is byte-identical with the hub's copy and a
|
||||
bidirectional key-set test guards drift.
|
||||
|
||||
### DR recipe — the storage/guest/PBS half (v0.38.0)
|
||||
|
||||
The host-report carries an additive `dr_recipe` section (`internal/hub/dr_recipe.go`) — the agent half
|
||||
of the secret-free reconstruction recipe (`SPIKE-dr-recipe-2026-06-16.md`). It is the non-secret
|
||||
**re-provision scaffolding** that complements escrow (keys) + PBS/restic (bytes): `guests[]` sizing,
|
||||
`drives[]` (user-data drives by `durable_id` → role → mount → intent), `pve_storage[]` (the
|
||||
`storage.cfg` defs), and `pbs` coordinates. Built by the pure `BuildDRRecipeHostHalf` from facts the
|
||||
report already collects — **no new reads**. **Boundary:** every field is an identifier/intent/size/
|
||||
coordinate — never a key/password/token/hash/`ENC:`; the PBS key + restic password stay in escrow.
|
||||
`recipe_version=1`, ignore-unknown on read, pinned in the cross-repo golden. The hub assembles it with
|
||||
the controller's app half into one customer recipe.
|
||||
|
||||
### The privileged `HostOps` surface (slice 5 Phase B)
|
||||
|
||||
The write side — the one place the agent steps outside its Proxmox API token into OS-root —
|
||||
is isolated behind the `HostOps` seam (`hostops.go`): production `SudoHostOps` shells out via
|
||||
a narrow **sudoers allowlist** (`configs/felhom-agent.sudoers`) with **fixed argument vectors
|
||||
and no shell**; tests use a fake (no real root in the suite).
|
||||
|
||||
- **Persistent mounts** are **systemd `.mount` units keyed by fs-UUID**
|
||||
(`What=/dev/disk/by-uuid/<UUID>`, enabled so they survive reboot) — not raw fstab or a
|
||||
transient `mount`. Benign re-mount is idempotent; **detach** (stop+disable) is destructive
|
||||
and routes through the gate.
|
||||
- **Host-reboot remount re-resolution (v0.37.0):** at startup (before binding drives into the
|
||||
guest) and on the periodic tick, `ReassertEnrolledMounts` re-asserts every enrolled `.mount`
|
||||
unit that isn't currently mounted: it re-resolves the drive's `uuid:<fs-uuid>` durable id to
|
||||
its **current** `/dev` node by re-scanning `/dev/disk/by-uuid` (`ResolveStorageDevice`, never a
|
||||
cached node) and re-runs the idempotent `enable --now`. This re-enables a unit a prior detach
|
||||
left `disabled` AND tolerates kernel re-enumeration moving a drive's letter (`/dev/sdb`→`sdc`)
|
||||
— the reshuffle is a no-op. Already-mounted drives and genuinely-absent UUIDs are skipped.
|
||||
- **Every argument is validated before any command is constructed** (`validate.go`): UUIDs
|
||||
against a strict hex regex, mount paths confined + traversal-checked, SMART devices
|
||||
whitelisted to raw disks, LVM names charset-checked. The adversarial matrix in
|
||||
`validate_test.go` proves a hostile UUID / path / device is refused with **zero** exec.
|
||||
- **SMART** (`smart.go`) fills `StorageTarget.smart` via `smartctl -a -j` — SATA *and* NVMe
|
||||
attribute sets, degrading to `UNKNOWN` for devices that expose no SMART (e.g. a USB bridge).
|
||||
**`lvs`** fills the lvmthin thin-pool **metadata** fill (metadata exhaustion corrupts a pool
|
||||
like data exhaustion).
|
||||
- The **watchdog** gains a benign **re-mount response**: when a known mount-backed target's
|
||||
device returns unmounted, it dispatches (off the poll path) a by-UUID re-mount, routed
|
||||
through the gate as benign. The **disk-grow executor** (`pct resize`, **grow-only**) lands
|
||||
in `internal/reconcile` as a benign action; **destructive storage ops** (detach/wipe/
|
||||
data-losing-resize) construct a `ClassStorageWipe`/`ClassDecommission` intent bound to the
|
||||
storage **target identity** and go through the slice-4 gate (built + tested, inert live).
|
||||
|
||||
### `--selftest=storage` (live storage harness)
|
||||
|
||||
Runs standalone on the Proxmox host (no hub needed):
|
||||
- bare: an **observe pass** printing the full `StorageTarget` table incl. the SMART summary
|
||||
and thin-pool data+metadata fill.
|
||||
- `-watch <dur>` (e.g. `--selftest=storage -watch 3m`): runs the watchdog verbose for the
|
||||
window with the **re-mount response live**, so an operator can physically cycle a drive and
|
||||
watch detect → report → re-mount in the logs.
|
||||
|
||||
## The `proxmox` package — model
|
||||
|
||||
Two backends, one fixed routing policy (the fence is structural — `Client` never shells out,
|
||||
`Privileged` never makes an HTTP call; asserted in `routing_test.go`):
|
||||
|
||||
| | Backend | Used for |
|
||||
|---|---|---|
|
||||
| **API (default)** | `proxmox.Client` | everything the scoped **FelhomAgent** token can do |
|
||||
| **root-CLI (fenced)** | `proxmox.Privileged` | the **three** proven OS-root exceptions only |
|
||||
|
||||
Grounded entirely in the spike findings (`felhom.eu/documentation/proxmox-platform.md`,
|
||||
`tests/phase{0,1-2,3}-findings.md`). Every mutating API op is **async**: it returns a UPID and
|
||||
the caller `WaitTask`s until the task stops, then asserts `exitstatus == "OK"` — authorization
|
||||
can surface at task execution, not the HTTP POST (phase1-2 §1.3).
|
||||
|
||||
### Public surface
|
||||
|
||||
`Client` (API):
|
||||
|
||||
- Read: `Version`, `Nodes`, `NodeStatus`, `ListLXC`, `GuestStatus`, `GuestConfig`,
|
||||
`ListStorage`, `NodeStorage`, `StorageContent`.
|
||||
- Async mutating (return UPID): `RestoreLXC` (primary create path), `Vzdump`, `Snapshot`,
|
||||
`Rollback`, `DeleteSnapshot`, `SetConfig`, `Start`, `Stop`.
|
||||
- Tasks: `WaitTask`, `TaskStatusOnce`, `TaskLogTail`.
|
||||
- Errors: `*APIError` (parses the offending privilege from a 403), `*TaskError` (parses it from
|
||||
a failed task `exitstatus`).
|
||||
|
||||
`Privileged` (fenced root-CLI) — each method documents *why it can't be the API*:
|
||||
|
||||
- `CreateGoldenLXC` — `pct create` with `keyctl=1` (root@pam-only; the only root-fenced create —
|
||||
the per-customer path provisions by **restore**, which preserves keyctl).
|
||||
- `MountUSBByUUID` — host mount-by-UUID (not a Proxmox API op).
|
||||
- `SMART`, `Sensors` — hardware reads (not API-exposed).
|
||||
|
||||
### API-vs-root routing table
|
||||
|
||||
See the table in [`internal/proxmox/doc.go`](internal/proxmox/doc.go). Summary: the entire guest
|
||||
lifecycle **including restore** is API-token-covered; OS-root is confined to golden-image
|
||||
`keyctl` create, host mounts, and SMART/sensors (phase3 §B3).
|
||||
|
||||
### Controller swap (agentic controller update, Phase 1 — v0.42.0)
|
||||
|
||||
The local API (`internal/localapi/`) owns the in-guest controller image **swap** — the new-architecture
|
||||
replacement for the controller's dead in-container `docker compose` self-update. The in-guest controller
|
||||
pre-pulls the target image then calls the agent:
|
||||
|
||||
- `POST /controller/swap {image}` (`withGuest`-scoped) → **202**, then async: record previous
|
||||
(`/var/lib/felhom-agent/controller-swap-<vmid>.json`) → confirm the target is present in the guest →
|
||||
write `/etc/felhom-controller-image` → `systemctl restart felhom-controller-bootstrap.service` → poll
|
||||
the new controller to healthy (`docker inspect`, ≤90s) → **roll back** to the previous image if not (the
|
||||
guest is never left without a controller). Strict image-ref gate; single-flight per guest (409).
|
||||
- **(v0.47.0) Verify hardening (F1):** the health poll reads `{{.RestartCount}}` (a 4th inspect
|
||||
field) — `running && RestartCount>0` is **not** healthy (a process that already crash-restarted
|
||||
isn't stably up), and a **no-healthcheck** image must report ok on 3 **consecutive** polls (a
|
||||
stability dwell) before acceptance; a real `healthy` result is trusted immediately. This closes the
|
||||
hole where a no-HEALTHCHECK image that crash-loops could land one "Running" instant and false-pass
|
||||
the point-in-time check → no rollback. Verify predicate only — the rollback orchestration + the
|
||||
`docker inspect -f *` grant (the `*` spans the extended template) are unchanged.
|
||||
- `GET /controller/swap/status` → `{state, current, previous, target, error}`.
|
||||
|
||||
The agent is **external** to the controller container, so it survives the controller being killed
|
||||
mid-swap (which the controller cannot do to itself). `GuestBinder.GuestExec` is the single `pct exec`
|
||||
seam. Exercise directly with `--selftest=controller-swap -vmid <id> -image <ref>`.
|
||||
|
||||
## Agent self-update (operator-signed, A/B slots, crash-loop auto-rollback — v0.70.0, TASK D1)
|
||||
|
||||
The agent updates ITSELF the same way it swaps the controller: **the thing that performs rollback is
|
||||
never the thing being updated.** For the agent that means systemd + an ~80-line root shell wrapper
|
||||
(`configs/felhom-selfupdate-guarded`) that changes almost never; the Go binary is what flips.
|
||||
|
||||
**Trust model.** An update is an **operator-signed `agent_update` op** delivered through the existing
|
||||
signed-jobs pipeline (same LOCKED authz gate as `storage_wipe`/`decommission`). The signed params pin
|
||||
the exact **version + sha256**, so the pinned sha is the *only* integrity root — **neither a
|
||||
compromised hub (dumb transport) nor a compromised Gitea (dumb storage) can substitute a binary.**
|
||||
The operator signs offline with `felhom-opsign -op agent_update -agent-version <v> -sha256 <hex>`.
|
||||
|
||||
**The flow** (`internal/selfupdate/` = the Go half; the wrapper = the root half):
|
||||
|
||||
1. The control loop sees a pending signed op → the gate verifies it (pinned-key SSHSIG → namespace →
|
||||
allow-list → crypto → host → time → **durable nonce-burn**) → the `agent_update` executor runs.
|
||||
2. Executor downloads the binary for the signed version from the config'd artifact host
|
||||
(`selfupdate.url_template`, `{version}` interpolated) to `/var/lib/felhom-agent/selfupdate/`,
|
||||
verifies its sha256 against the **signed** value (mismatch → refuse, remove, agent untouched),
|
||||
and hands it to `sudo -n felhom-selfupdate-guarded apply <staged> <sha>`. The job is completed on
|
||||
the hub **after verify+download, before apply** (the nonce is already burned — a queued job would
|
||||
only re-fetch and no-op on the spent nonce; a failed/rolled-back update is visible via the report).
|
||||
3. The wrapper (as root) **re-verifies** the sha, confines the staged path to the staging dir, asserts
|
||||
same-filesystem (the atomic-rename guarantee), snapshots the current binary to `.prev`, atomically
|
||||
`mv`s the new binary into place, writes a `pending.json` marker, `reset-failed`s, and schedules a
|
||||
**detached** restart (`systemd-run --on-active=2s … systemctl restart felhom-agent`, so the caller
|
||||
survives to log the handoff).
|
||||
4. The **new** binary boots; after it has run cleanly for a dwell (`selfupdate.dwell_seconds`, default
|
||||
60) *and* core init is done, `internal/selfupdate.Manager` calls the wrapper's `commit` (clears the
|
||||
marker; `.prev` retained as a manual net). A pending marker naming a *different* version than the
|
||||
running binary is **not** committed — loud WARN, marker left so the report shows why (a human
|
||||
decides).
|
||||
5. **Crash-loop auto-rollback (the safety property).** If the new binary crashes, systemd's
|
||||
`OnFailure=felhom-agent-rollback.service` (the `felhom-agent-limits.conf` drop-in) runs the
|
||||
wrapper's `rollback`: pending marker present → restore `.prev` byte-identical → clear marker →
|
||||
restart → the old binary is back **within seconds of the first crash**. On systemd 257 `OnFailure=`
|
||||
fires on *every* crash, so rollback triggers at the first one; the marker-guard makes every later
|
||||
fire (and any crash with no update in flight) a harmless no-op. The tuned start-limit
|
||||
(`[Unit] StartLimitIntervalSec=120 + StartLimitBurst=4`) is the terminal **backstop** (e.g. an
|
||||
environmental crash loop of the known-good binary → terminal `failed` ≈20s → the hub's
|
||||
`host_staleness` dead-man's-switch alerts the operator).
|
||||
|
||||
**Design provenance:** every systemd behaviour above is empirically validated in
|
||||
`felhom.eu/documentation/audits/SPIKE-agent-selfupdate-2026-07-05.md` (the SF-findings). The host
|
||||
report carries `selfupdate_pending` (+ version) so a runs-but-never-commits binary is visible even
|
||||
though it never crashes. v1 scope: no hub-floor auto-update, no auto-retry of a failed update, no
|
||||
pending-timeout auto-rollback (a stuck-but-alive binary is caught by `host_staleness`).
|
||||
|
||||
### TLS trust
|
||||
|
||||
The host serves a self-signed cert. Verification is **not** blanket-disabled. Pick one in
|
||||
config: `ca_file` (PEM, full verify), `fingerprint` (SHA-256 of the host leaf cert — pinned
|
||||
exact-cert match; the `/nodes` API returns each node's `ssl_fingerprint` to pin), or the
|
||||
explicitly-named `insecure_skip_verify` (off by default; selftest-against-127.0.0.1 only).
|
||||
|
||||
## Provisioning the token (out-of-band, operator side)
|
||||
|
||||
The agent only **consumes** a privilege-separated API token; role setup is a provisioning step.
|
||||
The role must be granted on **both the user AND the token** for the same path, or the
|
||||
intersection is empty and every call 403s (phase1-2 §1.2):
|
||||
|
||||
```bash
|
||||
pveum role add FelhomAgent -privs "VM.Allocate VM.Audit VM.Config.Disk VM.Config.CPU \
|
||||
VM.Config.Memory VM.Config.Network VM.Config.Options VM.PowerMgmt VM.Snapshot \
|
||||
VM.Snapshot.Rollback VM.Backup Datastore.Allocate Datastore.AllocateSpace \
|
||||
Datastore.Audit Sys.Audit SDN.Use" # 16 privileges, validated Phase 3 B3
|
||||
pveum user add felhom-agent@pve
|
||||
pveum user token add felhom-agent@pve agent --privsep 1 # capture the secret (shown once)
|
||||
pveum acl modify / -user 'felhom-agent@pve' -role FelhomAgent
|
||||
pveum acl modify / -token 'felhom-agent@pve!agent' -role FelhomAgent
|
||||
```
|
||||
|
||||
(`VM.Config.CPUMemory` is **not** a real privilege; `SDN.Use` **is** required for bridge use.)
|
||||
|
||||
## Run
|
||||
|
||||
```bash
|
||||
go build ./...
|
||||
# read-only health check against the host:
|
||||
./felhom-agent --config configs/agent.example.json --selftest
|
||||
# or via env (keeps the secret off disk):
|
||||
FELHOM_AGENT_PROXMOX_TOKEN='felhom-agent@pve!agent=SECRET' \
|
||||
FELHOM_AGENT_PROXMOX_NODE=demo-felhom \
|
||||
FELHOM_AGENT_PROXMOX_ENDPOINT=https://192.168.0.162:8006 \
|
||||
FELHOM_AGENT_PROXMOX_TLS_FINGERPRINT='BA:7C:...:CF' \
|
||||
./felhom-agent --selftest
|
||||
```
|
||||
|
||||
`--selftest` (read-only) loads config, builds the API client, and runs the read queries (version,
|
||||
nodes, node status, guests, storage), printing a short health report. It mutates nothing and says
|
||||
so cleanly if the token/endpoint isn't configured.
|
||||
|
||||
`--selftest=task --vmid N` (explicitly gated) exercises `WaitTask` on a **reversible** op
|
||||
(snapshot → rollback → delete-snapshot) against guest `N`. Default `--selftest` never mutates.
|
||||
|
||||
`--selftest=bring-up|provision` accept an optional operator **CPU/RAM cap**: `-cores N` and
|
||||
`-memory M` (MiB). Both default to `0` = keep the golden's baked sizes. When set, the cap is written
|
||||
into the SAME pre-start config PUT as the identity reset (via `BringUpSpec.Cores/MemoryMB`), so the
|
||||
guest never boots uncapped — useful when the appliance shares a host with other guests.
|
||||
|
||||
**Pool-scoped restore (v0.53.0):** every restore (provision bring-up, DR, and restore-test) allocates
|
||||
the guest INTO the `felhom` PVE pool (`reconcile.DefaultPool`; `RestoreLXCOptions.Pool` → `pct restore
|
||||
--pool`). This is what lets the agent token be scoped to `/pool/felhom` + `/storage/<targets>` instead
|
||||
of `/` (blast-radius containment on a shared host) — the restore is how a fresh vmid is allocated under
|
||||
that scoped token (`VM.Allocate`+`Pool.Allocate` at `/pool/felhom`). Layout + validation:
|
||||
`felhom.eu/documentation/audits/SPIKE-pool-scoped-acl-2026-07-01.md`.
|
||||
|
||||
**Drive discovery + tracking (v0.55.0, Impl-2a):** `GET /disks/candidates` lists host disks the Impl-1
|
||||
filter proves are free to enroll (split initialize/attach). The watchdog's known-drive set is sourced
|
||||
from the intent registry + Felhom `.mount` units (`RegistryKnownTargets`), NOT PVE storages — so a drive
|
||||
enrolled with no PVE dir-storage is still health-tracked (`Observe()` stays for real PVE storages). The
|
||||
controller wizard consuming this is Impl-2b.
|
||||
|
||||
**Format safety (v0.54.0, Impl-1; hardened v0.61.0, audit D1/D2/D3):** `Format` (mkfs) is gated by a
|
||||
mandatory **unclaimed-disk guard** (`internal/storage/claim.go`) — it refuses any device not provably
|
||||
free for Felhom (OS disk, LVM PV, ZFS/mdraid member, foreign mount, read-only; fail-safe on any read
|
||||
error, and on an empty/target-absent lsblk topology — D2), independent of `DataBearing`. Below the
|
||||
agent, mkfs runs ONLY through `configs/felhom-mkfs-guarded.sh` (the sole mkfs the sudoers permits),
|
||||
which re-checks the catastrophic cases as root: system disk, LVM PV (absolute-path pvs), foreign
|
||||
mount, read-only device, and any LVM/ZFS/mdraid/LUKS/swap member signature (D1 — validated by
|
||||
`scripts/mkfs-guarded-harness.sh`, a loop-device + recorder harness). The blank-format local-API path
|
||||
binds to the device's durable id and anti-retarget re-resolves before mkfs, same as the confirmed
|
||||
wipe (D3, AGENT-001's benign-branch twin). The pool-scoped token does NOT touch mkfs (sudo op) — the
|
||||
filter + wrapper are the guard. See `SPIKE-drive-enrollment-2026-07-01.md` and
|
||||
`felhom.eu/documentation/audits/AUDIT-blast-radius-hostroot-localapi-2026-07-02.md`.
|
||||
|
||||
## Process model
|
||||
|
||||
Native Go binary, systemd service, **non-root** `felhom-agent` service user holding the scoped token,
|
||||
with a **narrow sudoers allowlist** for the fenced host-root ops. `privileged.mode: "sudo"` matches
|
||||
this; `"direct"` is for dev/CI where the agent is already root.
|
||||
|
||||
The canonical artifacts (BUNDLE slice):
|
||||
- **`configs/felhom-agent.service`** — the canonical unit (`User=felhom-agent`,
|
||||
`ExecStart=/usr/local/bin/felhom-agent --config /etc/felhom-agent/agent.json`). It deliberately sets
|
||||
**no** `NoNewPrivileges` and **no** mount-namespacing hardening (`ProtectHome`/`PrivateTmp`/…): the
|
||||
first would block the `sudo` the agent needs, the second would put the agent in a private mount
|
||||
namespace so its `mount --bind` drive enrollments wouldn't propagate into the running guest. The
|
||||
security boundary is the sudoers allowlist, not systemd sandboxing.
|
||||
- **`configs/felhom-agent.sudoers`** → `/etc/sudoers.d/felhom-agent` (0440, `visudo -cf`-validated).
|
||||
- **`scripts/publish-agent.sh`** publishes the binary to Gitea as a generic package
|
||||
(`/api/packages/admin/generic/felhom-agent/<ver>/felhom-agent`), printing the sha256 the operator
|
||||
records in the hub artifact manifest.
|
||||
|
||||
**Install is automated.** The host-bootstrap script (`felhom.eu/scripts/felhom-host-install.sh`) fetches
|
||||
the binary from Gitea, verifies its sha256 against the hub-vouched manifest, then installs the user +
|
||||
binary + sudoers + unit + config — no manual agent install step.
|
||||
|
||||
## Test
|
||||
|
||||
```bash
|
||||
go vet ./... && go test ./...
|
||||
```
|
||||
|
||||
Unit tests use a mock HTTP transport + mock runner (no live host): UPID parse, `WaitTask`
|
||||
(running→OK / running→failed-403 / timeout / ctx-cancel), 403→privilege-named error, response
|
||||
decoding against the captured live shapes, and the API-vs-root routing fence.
|
||||
@@ -0,0 +1,90 @@
|
||||
# Findings — OS/Docker-data storage-split spike (Part B)
|
||||
|
||||
**Date:** 2026-06-13 · **Host:** demo-felhom (PVE 9.2.2) · **Throwaway:** unprivileged LXC 9300 (created, exercised, destroyed) · **Builds nothing** — this report gates the provisioning spec.
|
||||
|
||||
**Goal:** prove the mechanics of splitting the controller guest's **OS rootfs** from its **Docker/data** onto separate `local-lvm` volumes, and the live-migration path, before writing the provisioning spec. Target shape to validate: a small OS rootfs (~32 GB default) + a separate large Docker-data volume (~256 GB), sizes configurable at install.
|
||||
|
||||
**Method:** all risky steps were run on a throwaway unprivileged LXC (9300) replicating guest 9201's config (`unprivileged: 1`, `features nesting=1,keyctl=1`, rootfs on `local-lvm`), never on 9201's live data. Docker 29.5.3 — **identical to production 9201** (verified: both `overlayfs` driver + `io.containerd.snapshotter.v1`).
|
||||
|
||||
---
|
||||
|
||||
## Verdict summary
|
||||
|
||||
| Item | Verdict |
|
||||
|---|---|
|
||||
| B1 — second volume + Docker `data-root` on unprivileged LXC | ✅ **Works.** overlayfs on the ext4 `local-lvm` mountpoint; no idmap/ownership problem; survives reboot. |
|
||||
| B2 — live move-then-verify migration | ✅ **Works**, copy-not-move is the safety net. Caveat: rsync must exist + its exit code must be checked. |
|
||||
| B3 — vzdump/PBS coverage of the new mountpoint | ⚠️ **NOT by default — contradicts the spec's premise.** Extra CT mountpoints are **excluded** from vzdump unless `backup=1` is set **and the CT restarted**. |
|
||||
| B4 — live resize vs fresh-install carving | ✅ Live grow is online/non-disruptive (`pct resize`). Fresh-install carving = installer `hdsize`/`maxroot`/`maxvz` (host-level, documented below). |
|
||||
| B5 — provisioning seams | ✅ Mapped: golden rootfs size + `BringUpSpec.RootfsGrowGB`/`Mounts`; `bringup.go:313` is the exact spot that today omits `backup=1`. |
|
||||
|
||||
---
|
||||
|
||||
## B1 — Second volume + Docker `data-root` on an unprivileged LXC ✅
|
||||
|
||||
- Added a second `local-lvm` mountpoint: `pct set 9300 -mp0 local-lvm:16,mp=/mnt/dockerdata`. PVE created `vm-9300-disk-1`, formatted it **ext4**, mounted it; in-container it is `root:root 0755` — **no idmap/ownership fix needed** (PVE maps the unprivileged offset for a fresh volume automatically).
|
||||
- Pointed Docker at it via `/etc/docker/daemon.json` `{"data-root":"/mnt/dockerdata"}` → after `systemctl restart docker`, `docker info` reports `Docker Root Dir: /mnt/dockerdata`, `Storage Driver: overlayfs`.
|
||||
- **overlay works on the mountpoint's filesystem** (ext4): pulled an image + created a named volume + wrote to both the volume and the container's overlay layer. Image/volume bytes landed on `/mnt/dockerdata` (228K → 64M) while the rootfs `/var/lib/docker` stayed flat (232K).
|
||||
- **Survives reboot:** after `pct reboot`, `data-root` is still `/mnt/dockerdata`, the volume's marker file persists, the image is present.
|
||||
|
||||
**Gotcha (Docker 29 / containerd-snapshotter):** the daemon already auto-starts at boot, so `systemctl start docker` is a no-op and won't pick up a freshly-written `daemon.json` — use `systemctl restart docker`. Also note the driver is the new **`overlayfs`** (containerd-snapshotter), not legacy `overlay2` — relevant to B2.
|
||||
|
||||
## B2 — Live migration sequence (move-then-verify) ✅
|
||||
|
||||
Proven on the throwaway with seeded **running** apps (postgres with a known row `migrate-survive` in a named volume + nginx). The safe sequence:
|
||||
|
||||
1. `docker stop <apps>` 2. `systemctl stop docker docker.socket` 3. **copy, not move:** `rsync -aHAX --numeric-ids /var/lib/docker/ /mnt/dockerdata/` 4. write `data-root` → `/mnt/dockerdata` in `daemon.json` 5. `systemctl start docker` 6. **verify** images + `docker volume ls` + start apps + query the DB row 7. **only then** reclaim the old location (`rm -rf /var/lib/docker/*`).
|
||||
|
||||
Result: post-switch the `pgdata` volume was present, both apps booted, and the DB row was intact. Reclaiming the old path while apps ran on the new `data-root` did not disturb them — the new `data-root` is fully self-sufficient.
|
||||
|
||||
**Two hard lessons (both nearly caused a silent false-positive):**
|
||||
- **`rsync` was not installed** on the fresh guest; the first attempt's `rsync … | grep …` swallowed the `127` exit (pipe returns grep's status, `set -e` doesn't fire), so the migration "completed" having copied **nothing**. The migration script MUST install rsync and **check rsync's own exit code**, not a piped one.
|
||||
- With **containerd-snapshotter** the image content store sits under `data-root/containerd…`; switching `data-root` to an empty volume made images *appear* present (they resolve from the snapshotter's own addressing) while volumes/containers were empty — a misleading partial state. The reliable approach is to **rsync the entire `data-root` with docker stopped** (captures the containerd store + volumes + containers) and verify functional integrity (volume + DB row + boot), or treat images as re-pullable and migrate only the volumes (the irreplaceable data). The copy-not-move discipline meant the source was always intact and recoverable — I reverted `data-root` and the original apps + DB row came straight back.
|
||||
|
||||
## B3 — Backup coverage after the split ⚠️ (the load-bearing finding)
|
||||
|
||||
**The spec's premise is wrong.** It assumed the new mountpoint "should be [captured], as a CT mountpoint unless `backup=0`." The opposite is true on PVE 9.2.2:
|
||||
|
||||
```
|
||||
# mp0 with NO backup flag:
|
||||
INFO: excluding volume mount point mp0 ('/mnt/dockerdata') from backup (disabled)
|
||||
```
|
||||
|
||||
Additional LXC mountpoints are **excluded from vzdump by default**. They are included **only** when `backup=1` is set on the mountpoint. And setting it on a running CT is a **pending change** (`[pve:pending]`) — vzdump kept excluding mp0 until the **CT was restarted**, after which:
|
||||
|
||||
```
|
||||
# mp0 with backup=1, post-restart:
|
||||
INFO: including mount point mp0 ('/mnt/dockerdata') in backup
|
||||
```
|
||||
|
||||
Confirmed end-to-end: the resulting archive contains `./mnt/dockerdata/volumes/pgdata/_data/…` — the DB data is in PBS/vzdump only once `backup=1` is active.
|
||||
|
||||
**Implication:** if the OS/data split lands without `backup=1` on the docker-data mountpoint, **every named-volume database silently falls out of the PBS whole-guest snapshot** — exactly the class of silent-default trap the spike exists to catch (cf. the PBS `ignore-verified` default). This is the single must-fix for the provisioning spec.
|
||||
|
||||
Honest note (as the spec asked): because **both** volumes stay in PBS once `backup=1` is set, the backup **size does not shrink** — the win of the split is independent sizing/growth and a smaller, faster-to-restore OS rootfs, **not** a smaller backup. (Excluding images from PBS would shrink it but would drop DB coverage — out of scope; not done.)
|
||||
|
||||
## B4 — Resize paths: live vs fresh install ✅
|
||||
|
||||
- **Live (within the existing 349 GB `local-lvm`), non-disruptive:** `pct resize 9300 rootfs +2G` and `pct resize 9300 mp0 +4G` both grew the volume **and** the in-guest ext4 online (7.8→9.8 GB rootfs, 16→20 GB docker-data) with the CT running and the DB still queryable — no reboot, no downtime. Adding a brand-new mountpoint (`pct set -mpN`) is also non-disruptive to the rootfs, but the **mount itself only appears after a CT restart** on an unprivileged guest (same pending-activation behaviour seen with `backup=1` and with drive binds — consistent with the agent's existing "activate at next boot" note). Grow-only is safe; shrinking LVM-thin volumes is not supported online and was not attempted. The host root (`local`) was **not** repartitioned on the live system.
|
||||
- **Fresh install (host-level carving — this is where "configurable at install" lives):** the Proxmox ISO installer's *Advanced LVM options* expose `hdsize` (total of the disk PVE uses), `maxroot` (cap on the host `/` = `pve/root`), `maxvz` (cap on the `data` thin pool = `local-lvm`; set to 0 / leave headroom to keep space unallocated), `minfree` (reserved free space in the VG), and `swapsize`. These decide how the SSD is split between a small **host** root and a large **`local-lvm` thin pool**. Note the distinction the spec's target shape blurs: those installer knobs size the **host's** local-vs-thinpool; the **guest's** 32 GB-rootfs + 256 GB-docker-data split is sized at *provision* time via `pct` (rootfs size + an additive mountpoint), carved out of the thin pool — not by the installer. On the demo, the thin pool already has ~350 GB free, so per-guest carving needs no host repartition.
|
||||
|
||||
## B5 — Provisioning integration points ✅
|
||||
|
||||
Where the guest is sized today, and where the split + configurable sizes slot in:
|
||||
|
||||
- **Golden rootfs size — `configs/build-golden.sh:40`:** `--rootfs "${ROOTFS_STORAGE}:8"` — the OS rootfs is **hardcoded at 8 GB**. The golden also `docker pull`s the controller + app images into `/var/lib/docker` **on the rootfs** (lines ~71–92), so the baked-image set is bounded by this size. `ROOTFS_STORAGE` is the 3rd script arg (default `local-lvm`). The golden runs the same docker-ce as production (→ 29.5.3 overlayfs/containerd-snapshotter).
|
||||
- **Provision bring-up — `internal/reconcile/bringup.go`:** `BringUpSpec` already carries the right seams: `RestoreStorage` (rootfs target storage), `RootfsGrowGB` (grow-only rootfs resize, applied as its own `ResizeLXC` call at line ~205), and `Mounts []GuestMount{Storage,SizeGB,MountPoint}` (additive `mpN`, attached in `buildConfigParams` at line ~313). So a docker-data volume is a natural `GuestMount`, and the OS rootfs can be grown per-customer via `RootfsGrowGB`.
|
||||
- **🔴 Exact code spot for the B3 fix — `bringup.go:313`:** today it builds `fmt.Sprintf("%s:%d,mp=%s", m.Storage, m.SizeGB, m.MountPoint)` — **no `backup=` flag**, so any additive mount (including a future docker-data volume) is created `backup=0` = **excluded from PBS**. The docker-data mount MUST be attached with `,backup=1` (and the provision flow must account for the restart-to-activate behaviour). `GuestMount` should gain a `Backup bool` (or always-on for the data mount).
|
||||
- **Where the size config comes from:** currently caller-provided — the `--selftest=provision`/bring-up path and `cfg.Backup.RestoreStorage`. The in-code marker (`bringup.go:49-50`) states slice 10 wires the **hub storage manifest** into `Mounts`; that is the right home for per-customer sizes, consistent with bootstrap-v2 (the controller already pulls customer-scoped config from the hub). So "configurable at install" = a per-customer field in the hub desired-state → `RootfsGrowGB` + `GuestMount.SizeGB`.
|
||||
|
||||
---
|
||||
|
||||
## Open questions for the provisioning spec to resolve
|
||||
|
||||
1. **Where does `data-root` get redirected for a *fresh* guest?** The golden bakes images into `/var/lib/docker` on the rootfs. Mounting an empty data volume **over** `/var/lib/docker` hides the baked images. Options: (a) bake `daemon.json` `data-root=/mnt/dockerdata` into the golden and attach the (empty) data mount before first Docker start, re-pulling images on first boot; (b) bake the split into the golden itself (data volume present at golden-build time, images baked onto it); (c) first-boot migration step. Pick one; (b) keeps the baked-image fast-start, (a) is simplest but loses it.
|
||||
2. **`backup=1` is mandatory on the docker-data mount** (B3). Encode it at `bringup.go:313` and decide whether the OS rootfs alone (no app data) is worth a separate lighter backup cadence.
|
||||
3. **Restart-to-activate** (B3/B4): attaching a mount or flipping `backup=1` on a running unprivileged guest is pending until reboot. Provisioning attaches mounts *before* first start, so this is a non-issue at provision time, but any *post-hoc* migration of an existing customer guest must schedule a reboot.
|
||||
4. **Default sizes & source:** confirm 32 GB rootfs / 256 GB docker-data defaults, and wire them through the hub storage manifest (per-customer), sized to the actual SSD via the installer carving facts in B4.
|
||||
5. **Backup size does not shrink** (B3): set expectations — the split is for independent sizing/growth and a smaller OS-rootfs restore, not a smaller PBS footprint.
|
||||
|
||||
**Nothing was shipped.** Throwaway LXC 9300 and its test archives were destroyed after the spike.
|
||||
@@ -0,0 +1,30 @@
|
||||
# REPORT — two agent fixes on main, NOT released (night 2026-10-28 → 29, unattended)
|
||||
|
||||
**Plain words first.** Two changes are on main and tested. Neither is released: an agent release needs the operator's
|
||||
vouch, and the night brief allowed no release. Both go on the morning list.
|
||||
|
||||
## What changed
|
||||
|
||||
- **R-528 — the OOM counter (`6f933d6`, CI run id 2473 / #284 green).** New local-API route `GET /guest/oom-kills`. It
|
||||
reads the kernel's `oom_kill` count of each Docker container of the token's own guest, on the host, from
|
||||
`/sys/fs/cgroup/lxc/<vmid>/ns/system.slice/docker-<id>.scope/memory.events` (mode 0444, readable by the non-root
|
||||
agent user). Docker inside an LXC guest does not see these kills (`OOMKilled` stays false). The controller reads the
|
||||
route every 30 s (controller `eed23e7`). Tests `TestR528_*`, red first by mutation (another guest's container served;
|
||||
`oom` parsed for `oom_kill`).
|
||||
- **R-1254 — the escrow ceremony half (`5ddd124`, CI run id 2471 / #283 green).** After an escrow upload that fails, the
|
||||
agent asks the hub which package it holds (the identity blob compared): the new one → success; another one → 502 and
|
||||
„the previous code still works"; no answer → 504, which promises nothing. Tests `TestR1254_*`, red first.
|
||||
|
||||
## Live proof
|
||||
|
||||
- R-528, 2026-10-28 23:27 UTC on Tester 1 with a test build of this agent (put back after: 0.167.0, sha256 `28bdd282…`).
|
||||
A Radicale capped at 128 MB, a memory hog inside it killed: the host counter 0 → 1, one `app_oom` „seen by
|
||||
docker,cgroup", one mail to the operator with `"source":"docker,cgroup"`. Evidence: felhom.eu
|
||||
`documentation/audits/night-2026-10-28/r528-oom/`.
|
||||
- R-1254: no live proof. It needs a hub answer lost on purpose; that is a hub change, and no hub deploy runs at night.
|
||||
|
||||
## Next
|
||||
|
||||
The agent release (the operator's vouch), then `pinned.day0_agent` in `scripts/retention-policy.json` (D50).
|
||||
The controller needs no MinAgent change for R-528: an old agent answers 404, and the controller then waits 10 minutes
|
||||
before it asks again.
|
||||
@@ -0,0 +1,227 @@
|
||||
# REUSE.md — felhom-agent
|
||||
|
||||
> Before writing new code, check here. Canonical helpers, patterns to copy, traps to avoid.
|
||||
> Maintenance: update in the SAME commit that adds/changes/deprecates a shared helper.
|
||||
> Entries cite file + symbol. Line numbers are landmarks only — reconfirm before editing.
|
||||
|
||||
## 1. Canonical helpers (MUST reuse — do not reinvent)
|
||||
|
||||
### Allowlisted exec / privileged surface (sudoers)
|
||||
|
||||
| Symbol | File | Short signature | Use for | Gotchas |
|
||||
|---|---|---|---|---|
|
||||
| `Runner` / `ExecRunner.Run`, `RunStdin` | internal/proxmox/privileged.go | `Run(ctx, name, args...) (stdout, stderr []byte, err)` | ALL host command exec (direct or `sudo -n` prefix) | Arg vectors only, never a shell string; `capBuf` caps output at 1 MiB |
|
||||
| `Privileged` (CreateGoldenLXC/MountUSBByUUID/SMART/Sensors) | internal/proxmox/privileged.go | methods on `*Privileged` | the 3 fenced root-CLI exceptions ONLY | Do NOT add methods — fence is structural (`routing_test.go` asserts it) |
|
||||
| `SudoHostOps.run` | internal/storage/hostops.go | `run(ctx, name, args...) error` | allowlisted exec with stderr-wrapped error | Every arg pre-validated via validate.go before this is called |
|
||||
| `Prober.Probe` | internal/capability/probe.go | `Probe(ctx) []Status` | live sudo-policy capability check (`sudo -n -l --`) | Needs a DIRECT runner (never the sudo-prefixing one — double-sudo); never executes probed cmds. v0.86.0: config-gated caps (`Capability.GatedBy` + `Prober.GateActive`) report `inactive`/"disabled by configuration" ONLY when healthy — broken plumbing stays degraded; the pbsdr-* gate answers from `pbsdr.Manager.DRConfigured` (marker-backed across restarts) |
|
||||
| ~~`stageTemp`~~ (REMOVED v0.146.0, R-861) | — | — | — | Nothing the agent writes is `install`ed where root reads it any more: use `felhom-priv-apply` (below) or ship a fixed file in the bundle |
|
||||
| `felhom-priv-apply` (v0.146.0, R-861) | configs/felhom-priv-apply | `felhom-priv-apply unit <name> \| unit-forget <name> (v0.159.0, R-1124) \| dnsmasq <tmp> <name> \| wg \| sshd-config \| sshd-key \| controller-image <vmid>` (the last reads the ref on stdin, R-861 (a) A1) | ANY agent-rendered file a root program reads (systemd unit, dnsmasq drop-in, wg-quick conf, OOB sshd) — fixed source + destination, CONTENT checked against the agent's own renderers | A new renderer needs a verb + a contract test (`internal/privapplytest.Check`) feeding its REAL output; never a new `install` sudoers line |
|
||||
| `privapplytest.Check` | internal/privapplytest/check.go | `Check(t, verb, name, content) string` | the Go↔root-checker contract: a renderer's real output must read `OK` | Skips without python3; one call per rendered shape + one refused control |
|
||||
| `BUNDLE_FILES` + `Bundle` (mode `bundle`, `--install-bundle`; mode `agent_update` v0.146.0, R-861) | configs/felhom-os-apply | the ONE table of root-owned paths + the installer of them | ANY new root-owned file the installer writes (sudoers line, wrapper, unit) — add it to the table, never a new installer fetch (R-840) | The builder (`scripts/build-config-bundle.py`) and the installer read the same table; `test_every_root_file_the_installer_writes_is_in_the_bundle` fails on a path the bundle lacks. Trust files (`/etc/felhom/os-trust.json`, `operator-signers`) are NEVER bundle paths (R17) |
|
||||
| `osupdate.ConfigUpdateExecutor` | internal/osupdate/bundle.go | signed op `agent_config_update` {agent_version, bundle_sha256} | delivering the bundle to an installed box | a courier only: the root wrapper re-verifies signature, host, nonce and sha itself |
|
||||
| `osupdate.Leg.SendUnsent` / `lockPass` (v0.144.0, R-868) | internal/osupdate/unsent.go | `(ctx) int` | an OS-pass report the agent never sent (killed mid-pass): the wrapper keeps `report-<run>-<layer>-apply.json` beside the plan; the agent deletes it once the hub has it | any new caller that runs an apply pass must hold `lockPass` (flock, across processes) — the sender must never take a running pass's copy |
|
||||
| `osupdate.LoadSavedBlock` (v0.144.0, R-866) | internal/osupdate/leg.go | `(planDir) (block, savedAt, ok)` | the hub's newest os_update block as the daemon last received it (`os-update-block.json`) | the debug pass uses it ONLY when the hub cannot be reached, and says so in its header; no saved block → no pass |
|
||||
| `dpkg_state()` / `DPKG_STATE_SCRIPT` (v0.145.0, R-876) | configs/felhom-os-apply | `audit, journal = self.dpkg_state()` | dpkg's state in ONE call: `--audit` AND the update journal | never gate a repair on `--audit` alone — a crash leaves only the journal (measured); keep it one call (R-845) |
|
||||
| `thincheck.Checker.Check` + `Describe` (v0.159.0, R-1012) | internal/thincheck/thincheck.go | `Check(ctx, vmid) []Finding` | name an ORPHAN thin device (pool holds it, LVM does not) in a whole-guest backup failure — `BackupRunner.SetThinCheck` | READ-ONLY by design: the root helper `felhom-thin-check <vg> <pool>` sends only reserve/release_metadata_snap; an orphan needs two reads; fails open (never stops a backup). The repair stays human |
|
||||
| `guesthook.SnippetReady` / `Register` (v0.146.0, R-861) | internal/guesthook/install.go | `SnippetReady(path) error` | is the pre-start hook (a FIXED file from the bundle) in place — register only then | The agent never installs the hook (Proxmox runs it as root); a missing hookscript stops a guest start, so never `Register` without `SnippetReady` |
|
||||
|
||||
### Disk / format safety (role gates, durable IDs, format guards)
|
||||
|
||||
| Symbol | File | Short signature | Use for | Gotchas |
|
||||
|---|---|---|---|---|
|
||||
| `SudoHostOps.Format` | internal/storage/hostops.go | `Format(ctx, device, fstype) error` | THE only mkfs path | Guards, in order: `ValidateBlockDevice` + `ValidateFSType` → mandatory `deviceUnclaimed` (claim.go) → exec `felhom-mkfs-guarded` (sudoers allowlists ONLY the wrapper, not raw mkfs) |
|
||||
| `SudoHostOps.InspectDevice` + `DeviceProbe.DataBearing` | internal/storage/hostops.go | `InspectDevice(ctx, device) (DeviceProbe, error)` | data-bearing verdict from the AGENT's own read | Fail-safe: `Probed=false` ⇒ DataBearing=true; blkid output is evidence, lsblk is read-success authority |
|
||||
| `classifyClaim` / `SudoHostOps.deviceUnclaimed` | internal/storage/claim.go | `classifyClaim(claimFacts) (unclaimed bool, reason string)` | "is this disk provably free to format" | Pure function of `gatherClaimFacts`; ANY read error/ambiguity/empty-lsblk ⇒ CLAIMED (audit D2); Felhom's own `/mnt/felhom-drives` mounts are not a foreign claim |
|
||||
| `SudoHostOps.ListCandidateDisks` | internal/storage/candidates.go | `ListCandidateDisks(ctx) ([]CandidateDisk, error)` | enroll-candidate discovery | Fail-safe: omits anything not provably unclaimed |
|
||||
| `antiRetargetResolveExpect` (+ `antiRetargetResolve`, `antiRetargetResolveBlank`) | internal/localapi/wipe_reresolve.go | `(durableID, expectDataBearing, resolve, derive, inspect) (device, err)` | pre-mkfs anti-retarget: resolve durable id → re-derive+match → re-inspect | AGENT-001 + audit D3; refuses path-only bindings; wired via `Server.reresolveWipe`/`reresolveBlank` (test-injectable) |
|
||||
| `signedjobs.WipeExecutor.Execute` | internal/signedjobs/wipe.go | `Execute(ctx, op, params) error` | operator-signed data-bearing wipe | Durable-id bound; nonce burned by gate BEFORE execute; refuses no-longer-data-bearing targets |
|
||||
| `selfupdate.Executor` / `selfupdate.Manager` | internal/selfupdate/{executor,commit}.go | `NewExecutor(Config)` / `NewManager(ManagerConfig)` | operator-signed agent self-update (D1): download+verify-vs-signed-sha → wrapper `apply`; startup dwell → `commit` | sha is the ONLY integrity root; wrapper (`felhom-selfupdate-guarded`) re-verifies as root + does the A/B flip; NEVER rolls back (systemd + wrapper do). `WrapperRunner` seam. Report seam `SelfUpdatePending()` |
|
||||
| `Gate.AuthorizeStorageWipe` | internal/reconcile/gate.go | `AuthorizeStorageWipe(StorageWipeAuthz, *SignedOp) Decision` | tiered wipe authz | user-data ⇒ customer confirm bound to agent's DeviceDurableID; system/backup ⇒ operator signature only, `Confirmed` IGNORED by role |
|
||||
| `Gate.Authorize` | internal/reconcile/gate.go | `Authorize(Intent, *SignedOp) Decision` | every destructive intent | role-scoping (`roleAuthorizes`) + op-to-action binding; benign passes unsigned; audits every decision |
|
||||
| `storage.DeviceDurableID` / `ResolveDurableDevice` | internal/storage/durable_device.go | `DeviceDurableID(device) (string, error)` | WIPE-binding ids (`byid:`/`byuuid:`) | Single seam for /disks list AND gate (F20-BUG2); `ResolveDurableDevice` refuses bare paths |
|
||||
| `storage.ResolveStorageDevice` | internal/storage/durable_device.go | `ResolveStorageDevice("uuid:<fs-uuid>") (dev, err)` | re-resolve enrolled STORAGE drives (remount) | `uuid:` scheme ONLY — distinct from the wipe schemes; never trust a remembered /dev node |
|
||||
| `deriveDurableID` | internal/storage/durableid.go | `deriveDurableID(typ, s, backingDevice, uuid) string` | storage-target durable id (DR re-attach key) | Deterministic per type; `uuid:` for usb/local-dir; PBS id includes `#<fingerprint>` |
|
||||
| `SystemDisks` / `isSystemBacked` / `RoleForStorage` / `RoleForRawDevice` | internal/storage/role.go | `RoleForRawDevice(device, sysDisks, sysKnown) DeviceRole` | protection-tier classification | Fails safe to `system` (most protected) on any ambiguity; role is AGENT-derived, never caller-supplied |
|
||||
| `storage.IdentityResolver` / `Relation` / `MountFor` | internal/storage/identity.go | `HostPath(path) PlaceIdentity`, `Device(dev) PlaceIdentity`, `Relation(a, b) (different, proven, reason)` | physical-disk identity of a storage place (felhom.eu `16-storage-physical-identity.md` §3, v0.157.0) | Reads /sys + /run/udev/data + /proc/mounts only (no block I/O). Judge a drive ONLY on its own mount (`localapi.ownMountIdentity`) — never the containing fs, or an absent drive reads as the root disk. A shared disk key is "same, proven" even when unknown. Pinned by identity_test.go (fixture trees) + samedisk_test.go |
|
||||
| `localapi.Server.EnsureInternalPlace` / `InternalPlaceRecord` | internal/localapi/internalplace.go | `EnsureInternalPlace(ctx)` (startup + 20 s tick); `ReadInternalPlace` / `WriteInternalPlace` | the one-disk box's internal data place (felhom.eu `16` §5, v0.158.0): mounts it at /mnt/belso by fs-UUID like an enrolled drive, records it `enrolled`; the controller binds + registers it | Never takes /mnt/belso from a household drive (a foreign mount, or a unit naming another UUID, waits). Never formats/ejects (dm on the system disk → role system). The ONLY maker is the root helper `configs/felhom-internal-place` (installer + signed `add_internal_place`). Pinned by internalplace_test.go |
|
||||
| `configs/felhom-internal-place` | configs/ (bundle → /usr/local/sbin) | `plan\|create <planned GiB>` | the ruled size rule (reserve max(16, 10 %) ≤ 48 GiB; ≥ 20 GiB or nothing) + the only lvcreate/mkfs of a system-disk volume | Formats only a felhom-belso with NO signature; refuses a foreign one; never resizes/removes. Sudoers: `^(plan\|create) 0$` only. Pinned by configs/test_felhom_internal_place.py |
|
||||
| `storage.candidatePartitions` / `CandidatePartition` | internal/storage/partitions.go | `CandidateDisk.Partitions` | partitions as places (`16` §6): empty / filesystem / in_use per part of a free disk; `Formattable` only while no node of the disk is mounted | A part with children or a member type is never listed; a raw /mnt mount counts as Felhom's only when corroborated (felhomOwnedMounts) — for LISTING only: the root mkfs wrapper trusts no such claim (v0.158.1). Pinned by partitions_test.go |
|
||||
| partition wipe id `byid:<…-partN>+fsuuid:<uuid>` | internal/storage/durable_device.go | `DeviceDurableID` / `ResolveDurableDevice` | a data-bearing PART binds disk id + part number + filesystem (`16` §6) | Both halves must resolve to the SAME device or the wipe refuses. Whole disks keep the plain id. Pinned by partition_durable_test.go |
|
||||
| `ValidateUUID/MountPath/BlockDevice/FSType/SMARTDevice/LVMName`, `UnitNameForMount` | internal/storage/validate.go | `Validate*(v) error` | EVERY arg that reaches a root shell-out | The security boundary; strict whitelists (no by-* symlinks, no dm, no traversal); `systemdEscapePath` computed in-process |
|
||||
| `ValidateNetworkMountSpec` | internal/storage/netmount.go | `ValidateNetworkMountSpec(spec) error` | NAS mount input boundary | Same discipline as validate.go; SMB requires a creds ref; mountpoint confined under `NetworkMountRoot` |
|
||||
|
||||
### Mount lifecycle (host + guest binds)
|
||||
|
||||
| Symbol | File | Short signature | Use for | Gotchas |
|
||||
|---|---|---|---|---|
|
||||
| `SudoHostOps.EnsureMount` | internal/storage/hostops.go | `EnsureMount(ctx, MountSpec) error` | persistent by-UUID systemd .mount | Validate→render→stage→`install`→`enable --now`; idempotent |
|
||||
| `SudoHostOps.Unmount` | internal/storage/hostops.go | `Unmount(ctx, where) error` | detach a mount unit | DESTRUCTIVE — caller MUST have gated it; does not self-authorize |
|
||||
| `SudoHostOps.ReassertEnrolledMounts` | internal/storage/hostops.go | `ReassertEnrolledMounts(ctx)` | reboot remount (re-resolve by UUID) | Re-asserts unless mounted AND enabled (`shouldReassertMount`); skips absent UUIDs |
|
||||
| `SudoHostOps.ForgetRetiredDrives` (v0.159.0, R-1124) | internal/storage/forget_retired.go | `ForgetRetiredDrives(ctx, IntentReader) int` | drop the host mount unit of a DECOMMISSIONED drive that is GONE (start-up + the 20 s tick) | Only decommissioned + absent + unmounted; a present decommissioned drive keeps its unit (H3 re-enroll). The file goes only through `felhom-priv-apply unit-forget`, which re-checks all three as root |
|
||||
| `GuestBinder.AttachDrive` / `DetachDrive` | internal/localapi/intermediary.go | `AttachDrive(ctx, vmid, where) (guestPath, err)` | live drive hot-swap under `/mnt/felhom-drives` | Normalizes to EXACTLY ONE bind via `countHostMounts` (converges double-binds); force re-bind when guest can't see it |
|
||||
| `GuestBinder.EnsureSharedParent` | internal/localapi/intermediary.go | `EnsureSharedParent(ctx) error` | shared-parent bind + boot unit | make-private+make-shared ONLY on first bind — re-running orphans the guest's slave; F2-a: compares script AND unit for staleness |
|
||||
| `StablePathForRaw` / `DriveNameFromRaw` | internal/localapi/intermediary.go | `StablePathForRaw("/mnt/<n>") string` | raw host mount → stable guest path | Single source of truth both repos derive from |
|
||||
| `GuestBinder.GuestSeesMount` / `GuestBootID` | internal/localapi/intermediary.go | `GuestSeesMount(ctx, vmid, path) bool` | guest-visible (usable) signal; reboot detection | Host bind present ≠ guest sees it (non-recursive parent bind) |
|
||||
| `SudoHostOps.EnsureNetworkMount` / `RemoveNetworkMount` / `ListNetworkMounts` | internal/storage/netmount.go | `EnsureNetworkMount(ctx, spec) error` | NAS automount pair | rm glob confined to `mnt-felhom*` units; NAS ≠ drive (no durable-id/SMART/wipe); RemoveNetworkMount doubles as the verify-fail rollback (idempotent) |
|
||||
| `NetworkMountedAt` / `NetworkEndpointReachable` | internal/storage/netmount.go | `NetworkMountedAt(where) bool` | verify mount-truth + the 2 s add pre-probe | /proc/mounts is the ONLY mount-success judge (autofs trigger ≠ mounted; readability ≠ mounted — SPIKE-nas-verify §8) |
|
||||
| `SudoHostOps.ReassertNetworkAutomounts` + `Server.ReassertNetworkMounts` + `guesthook.PostStartNetworkReassert` | internal/storage/netreassert.go, internal/localapi/netreassert.go, internal/guesthook/netreassert.go | `ReassertNetworkAutomounts(ctx) []NetReassertResult` | NAS guest-reboot heal (RCA fix 1): re-arm idle automount triggers (stop + enable --now) so the fresh mount event propagates into running guests | NEVER call from periodic health paths (an idle trigger is HEALTHY); active real mounts are never touched; hook leg runs as root (direct systemctl), daemon leg via sudo |
|
||||
| `ClassifyNetVerifyFailure` | internal/storage/netverify.go | `ClassifyNetVerifyFailure(journalTail, tcpReachable) (code, hint)` | NAS verify failure categories | String-based BY DESIGN (every mount failure is rc=32); substrings verbatim from SPIKE-nas-verify Q4; `nfs_export` merges not-found/not-permitted (NFSv4 identical) |
|
||||
|
||||
### Durable stores (atomic state)
|
||||
|
||||
| Symbol | File | Short signature | Use for | Gotchas |
|
||||
|---|---|---|---|---|
|
||||
| `IntentStore` (`Get/SetEnrolled/SetEjected/SetDecommissioned/OnAbsent`) | internal/storage/intent.go | `OpenIntentStore(path)` | drive intent (4-state self-heal) | Keyed by durable-id only; `OnAbsent` is the ONLY ejected→enrolled path; refuses empty ids |
|
||||
| `GuestBindStore` (`Record/Remove/Guests`) | internal/localapi/guestbindstore.go | `OpenGuestBindStore(path)` | per-guest enrolled binds (F9 re-assert) | Same tmp+rename 0600 pattern as IntentStore |
|
||||
| `FormatJobStore` + `startFormatDetached` + `RecoverFormatJob` | internal/localapi/formatjob.go | `startFormatDetached(device, durableID, fstype, blank) (*formatJob, <-chan error)` | detached, restart-surviving mkfs (F20-BUG3) | Runs off `s.baseCtx` (60-min bound) so a request deadline can't SIGKILL mkfs; recovery re-resolves by durable id; blank jobs re-check STILL-blank; on success `job.FSUUID` = the new filesystem's UUID, read back only when the durable id still resolves to the formatted device (R-25) — read it only after `done` delivers |
|
||||
| `TokenStore.Mint` / `Lookup` | internal/localapi/tokenstore.go | `Mint(vmid) (plaintext, error)` | per-guest local-API tokens | Only the SHA-256 hash persists (fsync'd append log); constant-time compare on lookup; plaintext returned exactly once. Lookup stats the file on EVERY call and reloads BEFORE answering when the append-only log grew (R-269; was reload-on-miss only, v0.63.0 B3, which let a token rotated out by another process keep authorizing as a map hit): the one-shot provisioner mints into the same file the daemon indexes — cross-process coherence both ways without a restart; unchanged size = no re-read. Pinned by `TestTokenStore_RotatedOutTokenRejectedFirst` |
|
||||
| `FileNonceStore.SeenOrRecord` | internal/authz/noncestore.go | `SeenOrRecord(nonce, exp) bool` | durable anti-replay | fsync'd before returning false; prune only after exp |
|
||||
| `Journal` (`Append/Latest/InFlight/AlreadyApplied`) | internal/reconcile/journal.go | `OpenJournal(path)` | op journal + idempotency + crash recovery | `Recover` consumes `InFlight()`; scratch entries special-cased |
|
||||
|
||||
### Local-API plumbing
|
||||
|
||||
| Symbol | File | Short signature | Use for | Gotchas |
|
||||
|---|---|---|---|---|
|
||||
| `Server.withGuest` | internal/localapi/server.go | `withGuest(fn(w, r, vmid)) http.HandlerFunc` | bearer auth + self-scoping for EVERY route | Token→vmid is the authority; explicit `?vmid=` only a consistency check (403 on mismatch) |
|
||||
| `Server.scopedFromBody` | internal/localapi/server.go | `scopedFromBody(w, bodyVMID, tokenVMID, path) bool` | POST-body vmid self-scope check | Call right after decode; false = already 403'd |
|
||||
| `decodeBody` / `writeOK` / `writeErr` / `writeStatus` | internal/localapi/server.go | `writeStatus(w, code, ok, data, errMsg)` | ALL local-API JSON I/O | Do not hand-roll response envelopes |
|
||||
| `EnsureLeaf` | internal/localapi/cert.go | `EnsureLeaf(certPath, keyPath, host) (cert, fingerprint, generated, err)` | pinned self-signed leaf | `generated=true` invalidates every issued bootstrap pin — log LOUD (B.1) |
|
||||
| `Server.RecoverStaleLockedGuests` | internal/localapi/stalelock.go | `RecoverStaleLockedGuests(ctx)` | startup stale vzdump-lock heal (F2-b) | Clears ONLY `backup`/`snapshot-delete`, only when no vzdump in-flight; A1 RESOLVED (v0.62.0): scan is pool-intersected (`ListLXC` ∩ `Client.Pool`), fail-safe skip on pool-read failure |
|
||||
| `ControllerSwapper.Swap` + `ValidControllerImage` | internal/localapi/controllerswap.go | `Swap(ctx, vmid, target) *ControllerSwapState` | agent-owned controller image swap + rollback | Strict image regex (repo + 3-part semver); state file written BEFORE swap; no-healthcheck images need `verifyDwell` |
|
||||
| `Server.ControllerSupervisorTick` + `ControllerParkedMarker` | internal/localapi/controllersupervisor.go | `ControllerSupervisorTick(ctx)` | R-523: restart a provisioned guest's not-running controller via its bootstrap unit | Two-sweep confirm; honours swapInFlight, the host-side park marker, guest lock + vzdump; 3 restarts/15 min → 30 min pause; record rides the report as `controller_supervisor` (the hub mints the events — the agent has no event channel) |
|
||||
| `MemoryOps` + `Server.readMemoryBounds` | internal/localapi/guestmemory.go | `readMemoryBounds(ctx, vmid) (memoryBounds, err)` | guest RAM resize (v0.90.0, R-24): GET/POST /guest/memory | NEW narrow seam (never extend `GuestAPI` — it breaks every fake); the AGENT is the boundary — bounds recomputed FRESH per request (min 2048 / max host_total−2048 / shrink floor max(2048, usage+512)); §8 UNITS TRAP (config `memory`=MB, status/node=bytes); verify maxmem==target after `SetConfig` before claiming success; SetConfig NEVER called on a refusal path |
|
||||
|
||||
### Proxmox client / hub / PBS / provisioning
|
||||
|
||||
| Symbol | File | Short signature | Use for | Gotchas |
|
||||
|---|---|---|---|---|
|
||||
| `pvegate.Write` / `pvegate.Step` | internal/pvegate/pvegate.go | `Write(ctx) (release, waited, err)` / `Step(ctx) (end, err)` | R-812 option A: keep the agent's own /etc/pve writes out of a Proxmox package step (pmxcfs restarts) | Already wired at the two chokepoints — `Client.doBody` (every non-GET) and `ExecRunner.RunStdin` (`WritesEtcPVE`: pct config verbs, pvesm, pveum, felhom-pbs-apply create/reconcile). A new root CLI that writes /etc/pve goes into `WritesEtcPVE`, never its own lock. Never take `Step` around anything but the wrapper call (`Leg.runPVE`) — a `Write` inside a `Step` deadlocks until its context ends. |
|
||||
| `osupdate.KernelVerdict` / `Leg.KernelAfterBoot` | internal/osupdate/kernel.go | `KernelVerdict(before, after, tunnel, hubReached) (ok, why)` | R-836: THE one-shot-boot rule — the host health rule (`HostHealthVerdict`) AND the box reached the hub since the boot | The only judge of a new kernel. Never reboot the host from Go: every reboot is the wrapper's (`kernel-reboot`, `kernel-revert`), and only for a staged step. A new kernel-lane state lives in the wrapper's `/var/lib/felhom-kernel/state.json`, never in the agent's own files (the agent can write those). |
|
||||
| `Client.WaitTask` | internal/proxmox/task.go | `WaitTask(ctx, upid, opts) (TaskStatus, error)` | asserting EVERY mutating op | POST 200 ≠ success; authz can fail at task exec; `AllowWarnings` opt-in |
|
||||
| `Client.Pool` | internal/proxmox/query.go | `Pool(ctx, name) (PoolInfo, error)` | felhom-pool membership (the ownership registry, A1) | Needs `Pool.Audit` at `/pool/<name>` (host-install v1.9.0+); `Pool.Allocate` does NOT satisfy the read; members can be storages (type `storage`, vmid 0) — filter them |
|
||||
| `Client` mutate wrappers (`RestoreLXC/Vzdump/DestroyLXC/Snapshot/Rollback/SetConfig/ResizeLXC/Start/Stop`) | internal/proxmox/mutate.go | return `(upid, error)` | all API mutations | Async → always pair with WaitTask; route via gate/queue, not ad-hoc |
|
||||
| `Client.PoolAddVMID` | internal/proxmox/mutate.go | `PoolAddVMID(ctx, pool, vmid) error` | re-assert pool membership after a restore-over-existing (campaign-2 R2) | SYNC (no UPID, don't WaitTask); PVE `PUT /pools` is additive (merge, not replace) — `delete=1` removes; idempotent (already-member swallowed); needs `Pool.Allocate` at `/pool/<pool>`. `pct restore --pool` sets membership only at CREATE — a restore over an existing vmid drops it, so bring-up re-asserts post-restore |
|
||||
| `reconcile.PreflightRestoreSpace` + `restorespace.Provider` (v0.133.0) | internal/reconcile/restoretest_space.go, internal/restorespace/restorespace.go | `PreflightRestoreSpace(ctx, space, policy, archive, rawCfg, configured) SpaceVerdict` | ANY step that restores or copies a guest onto a storage — size it first | The restored size is UNCOMPRESSED (vzdump log "Total bytes written" / PBS snapshot size) — never the archive FILE (6.9 GB file → 22.6 GB restore, R-672); an unknown refuses; eligibility needs Datastore.AllocateSpace on `/storage/<id>` specifically (the `/` grant answers every path) |
|
||||
| `Engine.RetryScratchTeardown` + the daemon janitor (v0.133.0) | internal/reconcile/restoretest_retry.go, cmd/felhom-agent/janitor.go | `RetryScratchTeardown(ctx) ScratchRetryResult` | retrying a leftover on a TIMER instead of only at start | Never `Recover` on a timer — it also resolves generic in-flight ops; a periodic sweep that unlocks guests holds the one-heavy-op gate (`InFlight.TryAcquire`) |
|
||||
| `TLSConfig.build` / `normalizeFingerprint` | internal/proxmox/tls.go | `build() (*tls.Config, error)` | PVE leaf-cert SHA-256 pinning | No insecure default |
|
||||
| `pinnedTLS` | internal/pbs/pin.go | `pinnedTLS(fingerprint) (*tls.Config, error)` | PBS leaf pinning | Same model as PVE; 64-hex fingerprint normalized |
|
||||
| `httpx.NewTransport` | internal/httpx/transport.go | `NewTransport(tlsCfg, idleConnTimeout) *http.Transport` | **EVERY** hand-rolled `http.Transport` in this repo — pbs, hub and proxmox all pin TLS, so none can use `http.DefaultTransport` | **R-344: never inline `&http.Transport{TLSClientConfig: ...}` again.** A composite literal takes `IdleConnTimeout` **zero, which means retain idle connections FOREVER** — `http.DefaultTransport` sets 90s and a literal does not inherit it. Combined with a client rebuilt per cycle and dropped (`pbsTargetsFromPVE`), that stranded **388 sockets on ep0 in 46 h**, held open on BOTH sides. `idleConnTimeout <= 0` means **use the default**, never "no timeout". Returns a **FRESH** transport every call — a shared one would pool connections across differently pinned endpoints. Pinned by `internal/pbs/client_leak_test.go` (server-side connection counting) + `internal/httpx/transport_test.go` |
|
||||
| `hub.Client.Report` | internal/hub/client.go | `Report(ctx, *HostReport) (*ControlEnvelope, error)` | the heartbeat | Typed `TransportError`/`HTTPError`, never contain the bearer token |
|
||||
| `hub.Loop` + `MultiObserver` | internal/hub/loop.go | `NewLoop(...)`; `MultiObserver(obs...)` | resilient report loop + envelope fan-out | Errors logged, loop continues; interval clamped 60–3600 s |
|
||||
| `provision.BackHalf.Provision` | internal/provision/backhalf.go | `Provision(ctx, Input) (Result, error)` | guest bootstrap back-half | mint→render→0600 write→chown 100000:100000→`pct set` ro bind→onboot; token NEVER logged/returned. Bootstrap `local_api.endpoint` = the caller's `cfg.LocalAPI.ListenAddr` (main.go) — moving the agent bind to the island moves the guest dial for free (R-50, no template) |
|
||||
| `buildBringUpConfig` island NIC | internal/reconcile/bringup.go | (pure) `BringUpSpec{IslandBridge,IslandGuestAddr}` → `params["net1"]` | R-50 island control plane | When BOTH island fields are set (from `cfg.LocalAPI`), attaches a static `net1=name=eth1,bridge=<vmbr9>,ip=<.2/30>` (no hwaddr → fresh MAC), so the controller reaches the agent over a fixed private address immune to LAN/DHCP/site moves. Empty = pre-R-50, no net1. All-or-nothing + CIDR enforced in `LocalAPIConfig.Validate`. The guestnet healer is eth0-only (`parseMode` is dev-scoped) so it never touches the static island NIC |
|
||||
| `reconcile.Queue.Submit` | internal/reconcile/queue.go | `Submit(vmid, fn) <-chan error` | per-guest serialization of ALL mutations | Same vmid strictly FIFO; lanes parallel across guests |
|
||||
| `Engine.RunSignedJob` | internal/reconcile/job.go | `RunSignedJob(ctx, intent, signed, exec) JobResult` | executing a gated destructive job | Idempotency by nonce; journaled |
|
||||
| `escrow.Create` | internal/escrow/escrow.go | `Create(ctx, CreateOptions) (CreateResult, R, error)` | PBS-key escrow (zero-knowledge) | Recovery code returned SEPARATELY from the result (anti-log); self-verifies recoverability |
|
||||
| `escrow.GenerateRecoveryCode` / `joinSafe` / `RecoveryCodeSep` | internal/escrow/wordlist.go | `GenerateRecoveryCode() (string, error)` | minting the customer recovery code R | Draws from the EFF large list **filtered of every word containing `RecoveryCodeSep`** (4 entries: drop-down, felt-tip, t-shirt, yo-yo) so a code always segments back into exactly 10 words — a hyphenated word made codes ambiguous to transcribe AND flaked the test ~1/5 (v0.93.0). Generation-only: **already-issued codes stay valid**, R is verified as a whole passphrase and never re-split. Never count words by splitting the joined string — count what the generator drew |
|
||||
| `escrow.CeremonyBinary` / `CeremonyArgs()` / `CeremonyOutput` | internal/escrow/ceremony.go | the ONE fixed sudo self-invocation argv + the `--output=json` wire object (v2, R-978: json mode returns the package in `PendingUpload` instead of uploading) | controller-driven ceremony (v0.88.0) | SINGLE SOURCE shared by the localapi exec, the capability manifest entry, and (byte-identically) the FELHOM_ESCROW sudoers line — `TestEscrowCeremonyArgvPinned` + `TestManifestCoveredBySudoers` lock all three. Never flag-helpers, never `--`→`-` (spike §2.2) |
|
||||
| localapi escrow ceremony job | internal/localapi/escrow_ceremony.go | `POST /escrow/ceremony` + status + ONE-SHOT claim + preflight | the wizard's agent half | R lives ONLY in `Server.escrowR` (NEVER the job struct — snapshots must be structurally R-free); zeroed on claim/supersede/10-min TTL (`unclaimed_void`); in-memory BY DESIGN (restart loses R safely; re-run supersedes); **R-978: the new package reaches the hub only at the claim, before R is handed out (`EscrowCeremonyConfig.Upload`); TTL and a new start wait while that upload is in flight — an abandoned ceremony leaves the previous package current**; **R-1151: a claim with `{"defer_upload":true}` hands out R + two word positions and keeps the package until `POST /escrow/ceremony/confirm` brings those two words back (30-min window, only the two words kept, never R); a claim without it keeps the R-978 order (old controllers)**; subprocess stdout is SECRET-BEARING → parsed then zeroed, never logged |
|
||||
| `poke.Listener` + `poke.Port` | internal/poke/poke.go | `NewListener(resolve, trigger, port, logger)`; `poke.Port = 51822` | agent-plane immediate-sync (Direction-2a, v0.89.0) | Binds a contentless UDP socket EXCLUSIVELY to the box's WG /32 (`wgtunnel.LoadAssignedAddr`), fires the hub-loop out-of-band trigger. **Port 51822 is a SHARED cross-repo contract** — the hub poke sender + the ep0 `felhom-poke` forced-command target the SAME number; change one → change all three. Contentless (payload ignored), leading-edge debounced (`DebounceWindow`), WG-confined (kernel EKEYREJECTED refuses non-peer /32s). Wired only when `wg_tunnel.enabled` |
|
||||
| `wgtunnel.LoadAssignedAddr` | internal/wgtunnel/manager.go | `LoadAssignedAddr(stateDir) (netip.Addr, bool)` | the box's own WG /32 without a Manager | Reads `registered.json`; ok=false until registered; strips the /32 → bare addr (the poke bind target) |
|
||||
| `fasttick.Loop` + `fasttick.SourceFunc` | internal/fasttick/fasttick.go | `New(out chan<- struct{}, interval, logger, sources...)`; `Source.Unconverged() (bool, reason)` | agent-plane immediacy SECONDARY (v0.90.0, R-28): pulse the SAME out-of-band trigger every 30 s while ANY source is unconverged, self-disarm on convergence | STATE-BASED (no timer, nothing to journal). Every source MUST be a CACHED read (no exec/network per tick) — `desiredProvider.Generation()`, `reconcile.Engine.LastResult()` (Planned−Pending>0), `pbsdrLoop.PBSDRStatus().State=="waiting_secret"` ONLY, `wgtunnel.Manager.TunnelConvergence()`. The LOUD pbsdr states + destructive `pending_signature` are DELIBERATELY EXCLUDED (a stuck-loud box must not hammer). Pulses the cap-1 channel non-blocking (coalesces with poke/watchdog); NEVER touch `MinPollSeconds`/`clampInterval`/the ticker |
|
||||
| `wgtunnel.Manager.TunnelConvergence` / `reconcile.Engine.LastResult` | internal/wgtunnel/manager.go, internal/reconcile/engine.go | `TunnelConvergence() (desired, operational bool)`; `LastResult() (Result, bool)` | cached convergence snapshots for the fast-tick | TunnelConvergence is refreshed at the END of every `Apply` (its own cadence) so the fast-tick never execs `wg`/`systemctl`; LastResult is mutex-recorded per `reconcileOnce`, ok=false until the first pass |
|
||||
|
||||
## 2. Canonical patterns (copy structure from THE named file)
|
||||
|
||||
| Pattern | Canonical file | Key traits |
|
||||
|---|---|---|
|
||||
| Validate-then-exec privileged op | internal/storage/hostops.go (`EnsureMount`) | validate EVERY arg (validate.go) → render → stage in agent dir → root `install` → `systemctl`; refuse before any command exists |
|
||||
| Fail-safe pure classifier over gathered facts | internal/storage/claim.go (`classifyClaim` over `claimFacts`) | pure function ⇒ fixture-testable; any error/ambiguity refuses; gather separated from verdict |
|
||||
| Anti-retarget durable-id binding | internal/localapi/wipe_reresolve.go | resolve id → re-derive + exact match → re-inspect expected state → act on RE-RESOLVED device only |
|
||||
| Atomic single-file JSON store | internal/storage/intent.go | `Open*` loads (missing=empty, corrupt=fail-loud), mutex, tmp+rename 0600, idempotent set |
|
||||
| Durable append-only log + index | internal/authz/noncestore.go (`FileNonceStore`) | fsync before returning "new"; replay into index on open; expiry-only compaction |
|
||||
| Injectable seam funcs on Server | internal/localapi/server.go (`reresolveWipe`, `deviceDurableID`, `boundCheck`, `deviceCheck`, `livenessCheck`, net-verify: `netTrigger`/`netMounted`/`netJournal`/`netReachable`; R-856 `crashGuardStatePath` — GET /host/crash-guard's state file, internal/localapi/crashguard.go) | prod default wired in `NewServer`; tests override — no real /dev, /proc/mounts, journalctl or TCP in tests. **For mount-table predicates prefer the DATA seams `procSelfMountinfo` / `procGuestMountinfo` (internal/localapi/intermediary.go) over `boundCheck`/`livenessCheck`**: pointing them at a captured fixture runs the real parser, the real predicate and the real handler, so the test cannot go hollow the way R-116's did |
|
||||
| `Server.devicePresent` (R-113, v0.114.0) | internal/localapi/disks.go | `devicePresent(rawMountPath) bool`; seam `deviceCheck`, default `isHostMountpoint` | the agent's DEVICE-presence signal — asks whether the drive's RAW mount is still mounted | **Use this, never the bind, to answer "is the drive there".** The raw mount is a device-bound systemd unit and dies with its device; the agent's own bind under the shared parent is NOT device-bound and outlives it as a stale shell. `BoundUnderParent` is now `boundUnderParent(...) && devicePresent(...)` at BOTH /disks construction sites — dropping either half is a regression with its own red-proof. Empty path ⇒ **true** (unknown is never absent: absent stops a customer's apps) |
|
||||
| `bindLiveness` + `BindLiveness` (R-117, v0.117.0) | internal/localapi/intermediary.go | `bindLiveness(stable, raw) BindLiveness`; seam `livenessCheck`; read verdicts ONLY via `.Usable()` | the agent's bind-LIVENESS signal — the third term of `BoundUnderParent` | **`devicePresent` and `boundUnderParent` are both PATH-PRESENCE tests and neither is liveness.** They compare only mountinfo field 5, so both stay true over a bind that names the drive that went away while the raw mount healed onto the returning one (measured: raw 8:32 /dev/sdc, bind 8:16 /dev/sdb `shutdown`, EIO both ways, payload healthy). Two dead states, and a fix needs BOTH checks: devno mismatch (the detach/return case) AND the ext4 abort tokens `shutdown`/`emergency_ro` (the steady-state case, where the devnos AGREE because the device never left). **THREE states, never a bool** — `BindUnknown` must exist and `Usable()` treats it as PRESENT (absent stops a customer's apps). **Order matters:** compare devices first and read the abort flag off the RAW mount in the stale case — abort-first classifies the real return state as aborted and refuses the re-bind that repairs it. **NO BLOCK I/O, ever** (CLAUDE.md rule; a probe on a wedged device survives SIGKILL). 6 red-proofs |
|
||||
| `AttachDrive` repair ruling (R-117, v0.117.0) | internal/localapi/intermediary.go | the `switch bindLiveness(...)` inside the `n == 1 && GuestSeesMount` arm | decides whether the existing self-heal runs | `BindStaleDevice` ⇒ **re-bind** (the raw mount is a healthy new superblock; repairs live, no guest restart). `BindAborted` ⇒ **quiet no-op** — a re-bind lands on the SAME dead superblock and this runs every 20 s, so re-binding is an infinite silent retry that also masks the state; it must surface via `BoundUnderParent=false`. `BindLive`/`BindUnknown` ⇒ no-op, unchanged. **Do not return an error for the aborted case** — the reconcile loop would log a failure every 20 s |
|
||||
| Detached IN-MEMORY verify job (single slot, deliberately unpersisted) | internal/localapi/netverifyjob.go | claim slot sync (single-flight 409) → detached pipeline off baseCtx → auto-rollback on fail; restart ⇒ slot empty ⇒ the CALLER rolls back (Scenario F) — contrast formatjob (persisted+recovered) |
|
||||
| Optional dependency degradation | internal/localapi/server.go (`Options`) | nil dep ⇒ endpoint answers "not configured" (503), never a crash |
|
||||
| Version channel (v0.82.0) | internal/localapi/server.go (`Options.AgentVersion`; `Handler()` mux wrap) | sets `X-Felhom-Agent-Version` on EVERY response (all routes/statuses, incl. auth-fail/404) — the controller's capability-comparison source; empty version ⇒ header omitted |
|
||||
| Root-file install via random temp | internal/localapi/intermediary.go (`installSharedParentUnit`) | `stageTemp` (os.CreateTemp) → sudoers-globbed `install -m` → pinned destination |
|
||||
| Detached destructive job + restart recovery | internal/localapi/formatjob.go | persist `running` → run off baseCtx → record outcome; recovery re-resolves durable id, never a path |
|
||||
| Signed-op verify pipeline | internal/authz/verifier.go (`Verify`) | armor→namespace→key-material allowlist→crypto over RAW bytes→blob→target→window→nonce LAST |
|
||||
| Resilient daemon loop | internal/hub/loop.go (`Loop.Run`) | ctx-cancel returns nil; errors logged and loop continues |
|
||||
| Guarded-below-the-agent wrapper | configs/felhom-mkfs-guarded.sh | root re-checks catastrophic cases (system disk/LVM PV/foreign mount/RO/member FS) even against an agent bug |
|
||||
|
||||
## 3. Dangerous lookalikes — do NOT reuse
|
||||
|
||||
| Trap | Why it bites | Use instead |
|
||||
|---|---|---|
|
||||
| Acting on the caller's `req.Device` (or any remembered /dev path) after inspection | classify→mkfs TOCTOU (AGENT-001): /dev re-enumeration retargets the node to a different physical disk | `Server.reresolveWipe`/`reresolveBlank` → format the RE-RESOLVED device |
|
||||
| Exec'ing raw `mkfs.*` (incl. `Binaries.MkfsExt4/MkfsXfs`) | sudoers no longer allowlists raw mkfs; bypasses the claim filter + wrapper re-checks | `SudoHostOps.Format` (→ `felhom-mkfs-guarded`) |
|
||||
| `DiskInfo.DurableID` (`uuid:`) as a wipe-confirmation id | wipe gate resolves `byid:`/`byuuid:` — a `uuid:` id is a `binding_mismatch` (F20-BUG2) | `DiskInfo.WipeDurableID` / `storage.DeviceDurableID` |
|
||||
| `ResolveDurableDevice` for enrolled-storage remounts (or vice versa) | schemes differ: wipe = `byid:`/`byuuid:`, storage = `uuid:` — each refuses the other | `ResolveStorageDevice` for mounts; `ResolveDurableDevice` for wipes |
|
||||
| `authz.MemoryNonceStore` on a real host | replay protection dies on restart | `authz.FileNonceStore` |
|
||||
| Adding methods to `proxmox.Privileged` | breaks the 3-exception root-CLI fence (`routing_test.go`) | `proxmox.Runner` + a new sudoers Cmnd_Alias + validate.go-style checks |
|
||||
| Calling `Client.DestroyLXC`/`Vzdump`/`SetConfig` outside the gate/queue/journal | skips classification, signature, per-guest serialization, crash recovery | `reconcile.Engine` paths / `RunSignedJob`; queue via `Queue.Submit` |
|
||||
| `GuestBinder.AttachBind`/`DetachBind` (per-drive `pct set -mpN`) | legacy model; a missing bind source can brick guest boot (C1) | `AttachDrive`/`DetachDrive` (intermediary model) |
|
||||
| `isHostMountpoint` to reconcile bind state | boolean can't converge stacked double-binds (the /mnt doubling bug) | `countHostMounts` normalization inside `AttachDrive` |
|
||||
| Acting on a raw `ListLXC` list as if it were "guests the agent owns" | audit A1 (pre-v0.62.0 the stale-lock reaper did exactly this — contained only by the pool-scoped token) | ownership must be PROVEN: intersect with `Client.Pool` membership like `staleLockController.Guests()` (v0.62.0), fail-safe on read failure |
|
||||
|
||||
## 4. Seams & interfaces (testing + cross-repo)
|
||||
|
||||
| Interface | Defined in | Implemented by | Fakes/tests at |
|
||||
|---|---|---|---|
|
||||
| `proxmox.Runner` | internal/proxmox/privileged.go | `*ExecRunner` (direct/sudo) | `mockRunner` internal/proxmox/mock_test.go; runner fakes in storage tests |
|
||||
| `storage.HostOps` | internal/storage/hostops.go | `*SudoHostOps` (prod), `NoopHostOps` (degraded) | fakes in internal/storage/observe_test.go, watchdog_test.go |
|
||||
| `storage.HostReader` | internal/storage/hostread.go | `*ProcHostReader` | `fakeHostReader` internal/localapi/disks_test.go; internal/storage/role_test.go. v0.87.0: `BlockSlaves(name)` lists `/sys/block/<name>/slaves` (root-free) — backs the `SystemDisks` dm/md walk (`physicalDisksOf`/`walkSlaves`, role.go); per-branch conservatism: an unresolvable slave fails the WHOLE walk → all-system fail-safe. NEVER weaken the signature test `TestSystemDisks_WalkTopologies` (root-backing disk always in the system set). |
|
||||
| `storage.IdentityHost` | internal/storage/identity.go | `*SysfsIdentityHost` | fixture /sys + udev trees in identity_test.go (`newFx`); `fakeIdentHost` in internal/localapi/samedisk_test.go. A SEPARATE interface from HostReader on purpose — adding methods to HostReader would break every existing fake. |
|
||||
| `localapi.DiskOps` / `StorageGate` / `GuestAttacher` / `GuestLister` | internal/localapi/disks.go | `*storage.SudoHostOps`; `storageGateAdapter` (cmd/felhom-agent/main.go); `*GuestBinder`; `*proxmox.Client` | `fakeDiskOps`/`fakeGate`/`fakeGuestAttacher`/`fakeGuestList` internal/localapi/disks_test.go |
|
||||
| `lanresolver.hostRoot` + `dnsmasqUnitPaths` (data seam, R-317) | internal/lanresolver/lanresolver.go | prod `hostRoot = "/"`; probe = the `dnsmasq` package's systemd UNIT, never `/usr/sbin/dnsmasq` (owned by `dnsmasq-base`) | internal/lanresolver/ensure_dnsmasq_test.go — fixture root tree + recording `proxmox.Runner`; the REAL `os.Stat` probe and `EnsureDnsmasq` run. `TestEnsureDnsmasq_ProductionProbeIsTheUnit` pins the production wiring |
|
||||
| `localapi.GuestAPI` / `BackupService` / `BackupStore` / `TokenAuthority` | internal/localapi/server.go | `*proxmox.Client`, `*backup.BackupRunner`, `*backup.Store`, `*TokenStore` | `fakeGuests`/`fakeBackups`/`fakeStore` internal/localapi/server_test.go |
|
||||
| `backup.InFlight` | internal/backup/inflight.go | `TryAcquire(what) (release, busy, ok)` / `Busy()` | THE host-wide "one heavy guest operation at a time" gate — shared by the local-API backup path and the restore-test scheduler (R-85) | A **LINK** guard, not a lock one: the scratch VMID never touches the live guest's vzdump lock, but an offsite restore PULLS multi-GB over the tunnel a backup PUSHES one. Callers **DEFER, never cancel** — a deferred restore-test costs coverage, a cancelled backup costs the backup. A nil gate is ungated (pre-R-85 callers). |
|
||||
| `fstrim.Trimmer` (R-444) | internal/fstrim/fstrim.go | `New(runner, guests, gate, statePath, logger)` / `Pass(ctx)` / `GuestDiskTrimStatus(ctx)` / `ParseTrimmed(out)` | the weekly `pct fstrim <vmid>` of owned running guests (Wednesday from 10:00 local, starts 10:00-20:59 only), under `backup.InFlight`; last result per guest persisted and reported as `guest_disk_trim` | A busy gate DEFERS to the next hourly tick, never waits; a failed trim retries at most `MaxAttemptsPerWeek`; the report reads the persisted record, it never runs pct |
|
||||
| `linetest.Tester` + `linetest.Client` (R-1247) | internal/linetest/linetest.go · internal/linetest/iperf.go | `New(runner, target, gate, uplink, hostID, statePath, logger)` / `Tick(ctx)` / `Value()` / `LineTestStatus(ctx)`; `Client{Streams, Time}.Run(ctx, addr)` | the weekly upload line test: a 10-second iperf3 upload (the agent speaks the iperf3 protocol — nothing installed on the host) to ep0's tunnel-only server (`wgtunnel.Manager.LineTestTarget`), Tuesday 10:00-15:59 local from a per-host slot, after 10 quiet minutes of the PHYSICAL uplink (`SysUplinkTx`: the bridge's ports with a `device` link — the bridge counter misses bridged guest traffic), under `backup.InFlight`; best of 3; reported as `line_test`, served as GET /host/line-test | **The value is the SERVER's received bytes**, never the client's written ones (the send buffer). `Run` refuses a context without a deadline. iperf3 numbers streams 1, 3, 4, 5 — the server matches results by those ids. „server busy" (ACCESS_DENIED) stops the measurement and is retried at the next tick, at most `MaxTriesPerDay` |
|
||||
| `capability` store-grant probe (`storeGrantStatuses` / `storeGrantVerdict` / `Client.Permissions`) | cmd/felhom-agent/main.go, internal/proxmox/query.go | *"may the agent READ this backup tier?"*, one `capability.Status` per configured tier | R-185. **Never infer permission from an empty content listing** — `{"data":[]}` is what a FORBIDDEN tier and a NEWBORN tier both return, and that ambiguity hid an unreadable host tier on both demo boxes. Ask `/access/permissions` **as the agent's own token** (root always says yes). **The ungranted answer is not empty and not a 403** — it carries the privileges inherited from the box-wide `/` grant, so test for **`Datastore.AllocateSpace`** specifically; path-presence or `Datastore.Audit` reports a blinded storage healthy. Probed set comes from `BackupTiers()`, never a fixed list. Critical except the `local` fallback. Composes AROUND the sudo prober (the `poolReadStatus` precedent); `Status`'s wire shape is untouched so the hub alert is free. Unreachable PVE ⇒ degraded, never ok. |
|
||||
| `backup.RestoreTestState` | internal/backup/restoretest_state.go | `RecordSuccess(target,archive,tier,verified,t)` / `ProvenArchive(target)` / `ProvenRestoreTests(ctx)` / `LastSuccess(target)` / `OldestFirst(targets)` | Per-tier restore-test PROOF state, persisted (atomic tmp+rename) — **which archive** was proven, and when (R-86) | **Credit ONLY on success** — a permanently failing tier must keep sorting first, or it looks freshly proven and stops being retried. Ties break on target id: without it, two tiers proven in the same second rotate by Go's randomised map order. **This one NEEDS persistence unlike R-84** — R-84 had ground truth to consult (the archive is still on the storage); a restore-test destroys its scratch and leaves no artifact. **R-86: the ARCHIVE is the state, the time is metadata** — a time alone cannot answer "have we proven THIS archive", which is the due-check's whole question. A pre-R-86 file (bare RFC3339 per target) keeps its time and yields NO proven archive, so each tier is due once after the upgrade; reading a legacy time as proof of the current archive would invent a guarantee. **R-189: it is also the REPORTABLE half of the restore-test signal.** The in-memory `backup.Store` holds only this process's latest run, and under per-archive due-ness the agent will not re-test a proven archive — so a proof lost to a restart is not repeated for a whole archive generation (observed live: a passing 14.5 GB offsite restore reached no host-report). `ProvenRestoreTests` renders the stored proofs as `hub.RestoreTest` entries and the collector merges them; a record missing the archive or the tier is NOT emitted, because an unproven tier reading as proven is worse than the defect. **Only successes are stored, deliberately:** a success suppresses future work, a failure causes it. |
|
||||
| `backup.BackupSuccessState` | internal/backup/backup_state.go | `RecordBackupSuccess(target, b)` / `LastKnownSuccess(target, vmid)` | Newest SUCCESSFUL backup per tier+guest, persisted (atomic tmp+rename) — the due-check's fallback when the tier's storage cannot be read after a restart (R-894) | **Read ONLY when the storage cannot be read** — a storage that answers is the ground truth (R-84), and an archive absent there must make the tier due even when this file remembers one. A saved copy older than the cadence still reads due. Only successes are written. |
|
||||
| localapi backup cadence cap (R-973) | internal/localapi/backup_cap.go | `backupCadenceCap(cadence)`, `Server.capRefuses`, `Server.BackupCapRefusal()` | POST /backup refuses (429) a run the `07` §6 schedule would never ask for: ≤24 h tier → 3 successes per rolling 24 h; longer tier → 1 per max(cadence−24 h, cadence/2) | Counts SUCCESSES only, from this process AND `BackupSuccessState.RecentBackupSuccesses` (so a restart does not reset it); refusals saved (`RecordBackupRefusal`) and raised for 24 h as the CRITICAL host-report status `localapi:backup-cadence-cap` (`backupCapStatus` in main) → the hub's existing `agent_capability_degraded`. Tests `TestBackupCap_*`. |
|
||||
| `hub.ProvenRestoreTestReporter` + `Collector.SetProvenRestoreTests` | internal/hub/collect.go | the DURABLE restore-test source, merged with the in-memory one | R-189. Merge rule: **one entry per tier, newest by `TestedAt` wins** — a fresh failure beats a stored success (the failure is the news, and it lives nowhere else), a stored success beats a stale in-memory entry after a restart, and a tier never appears twice (the hub would read two tests). An unparseable timestamp counts as OLDER, so a malformed entry cannot displace a good one. **The wiring is pinned by an AST test** — the method this replaced (`RestoreTestState.Snapshot`) carried a doc comment naming a host-report gauge and had no caller for weeks. |
|
||||
| `backup.SpecBuilder` / `backup.TierPicker` / `(*BackupRunner).PickSettledRestoreCandidateOn` | internal/backup/schedule.go, runner.go | `func(ctx,archive) RestoreTestSpec`; `func(ctx,target,notAfter) (archive,landed,error)` | The per-run restore-test spec + per-tier **settled** candidate lookup (R-85, widened by R-86) | The spec is built **PER RUN**, never frozen at construction — the pre-R-85 immediately-invoked value made the offsite tier unschedulable AND went stale on any config change. `SourceTier` comes from **the archive**, never the configured target (the v0.100.0 rule). A tier with no archive returns `("", zero, nil)` — **`""` is NOT an error**, or every fresh box looks broken for its first week. **R-86: `notAfter` is the settle cutoff** (zero = no cutoff, which is what keeps `PickRestoreCandidateOn` a one-line call into it), and the picker now skips entries failing `archivePlausiblyComplete` — under per-archive due-ness an incomplete phantom would be picked forever, fail forever, never earn proof, and make the tier due at EVERY evaluation. |
|
||||
| `localapi.BackupTier` + `normalizeBackupTiers` / `config.BackupConfig.BackupTiers` | internal/localapi/backup_tiers.go, internal/config/config.go | `normalizeBackupTiers(tiers, legacy, cadence) []BackupTier`; `BackupTiers() ([]BackupTier, []string)` | THE R-82 multi-tier resolution — one runner per tier, primary first | **The untargeted local-API contract is FROZEN**: no `?target=` ⇒ primary tier ⇒ pre-R-82 response BYTES (Target is `omitempty` and stays empty). Never default a missing cadence — reject it and log the warning at ERROR. Never share one retention knob between tiers. Jobs are keyed by (vmid,target). |
|
||||
| `localapi.StaleLockController` | internal/localapi/stalelock.go | `*staleLockController` (Client + Runner + pool) | `fakeStaleLock` (Server-level) stalelock_test.go; `fakeStaleLockAPI` (controller-level, tests the A1 pool intersect) stalelock_pool_test.go |
|
||||
| `localapi.GuestExecutor` | internal/localapi/controllerswap.go | `*GuestBinder` (pct exec; the image write via `felhom-priv-apply controller-image`) | `fakeGuestExec` internal/localapi/controllerswap_test.go |
|
||||
| `guestnet.Runner` / `guestnet.GuestSource` (R-54, v0.92.0) | internal/guestnet/{probe,watchdog}.go | `*proxmox.ExecRunner`; the POOL-VERIFIED `localapi.StaleLockController.Guests` (ListLXC ∩ felhom pool, audit A1) | `scriptedRunner` + `fakeGuests` internal/guestnet/watchdog_test.go. **Never wire a bare `ListLXC` here** — under a broad token that would run dhclient inside a co-tenant's container. Every assertion is an exec COUNT, and the load-bearing ones are the negatives: a static guest, an unprobeable guest, a boot-race guest and an unproven guest list must record **zero** heal calls |
|
||||
| `guestnet.Watchdog.SetDampers` / `now` (clock seam) | internal/guestnet/watchdog.go | config `guest_net.*`; `now` defaults to `time.Now` | tests advance a manual clock (the storage-watchdog pattern) and assert the heal ceilings EXACTLY — ≥10 min apart, ≤3/hour, and ≤30 over a scripted 10 hours of permanent failure. A damper with no test is a comment |
|
||||
| `hub.GuestNetReporter` (R-54) | internal/hub/collect.go | `*guestnet.Watchdog` (`GuestNetStatus`) | internal/hub/collect_guestnet_test.go asserts the stanza through the PRODUCTION `Collect` path AND that the `guest_net` key is ABSENT from the wire when no reporter is wired — an always-present empty stanza would make "not wired" and "found nothing" the same signal, which is the shape v0.91.0 hid behind |
|
||||
| `hub.AddressEnumerator` (v0.119.0) | internal/hub/hostaddr.go | **defaults to the REAL `systemInterfaces`** when `Collector.addrEnum` is nil — deliberately inverting the nil-reporter-means-off convention, because this stanza has no config gate and a forgotten wiring call would otherwise ship silently empty (the inert-seam shape, four instances on record) | internal/hub/hostaddr_test.go drives fixtures TRANSCRIBED from `ip -o addr show` on demo-felhom AND demo-hp, including the address-less veth/NIC rows — the "no denylist needed" claim rests on those rows really being empty, so omitting them would prove the claim by assuming it. `filterHostAddresses` keeps GLOBAL UNICAST only: one predicate that drops loopback, `fe80::/10`, and `169.254/16` — the last being the R-50 island literal, identical on every box and actively misleading if surfaced |
|
||||
| `reconcile.OpVerifier` | internal/reconcile/gate.go | `*authz.Verifier` | fake verifier in internal/reconcile gate tests |
|
||||
| `signedjobs.WipeOps` / `Executor` (`ExecutorChain`) | internal/signedjobs/wipe.go + runner.go | `*storage.SudoHostOps`; `WipeExecutor`+`DecommissionExecutor` | internal/signedjobs wipe/runner/decommission tests |
|
||||
| `hub.reporter` / `collectorIface` / `EnvelopeObserver` | internal/hub/loop.go | `*hub.Client`, `*hub.Collector`; `desired.Syncer` + `signedjobs.Runner` | `fakeReporter`/`fakeCollector` internal/hub/loop_test.go; `recordingReporter` loop_logtail_test.go |
|
||||
| `applog.Ring` (always-DEBUG capture ring) + fan-out `applog.New → (logger, ring)` | internal/log/log.go | wired in cmd/felhom-agent/main.go → `localapi.Options.LogRing` + `Loop.SetLogTailSource(ring.Lines)` | internal/log/log_test.go; localapi/debuglogs_test.go — v0.83.0; the byte-capped `Lines` is the heartbeat tail source |
|
||||
| `pbsdr.StorageReader` / `SecretConsumer` / `Manager.probeFP` (func seam) | internal/pbsdr/manager.go | `*proxmox.Client`; `*hub.Client`; `pbs.ProbeFingerprint` | `fakeStorage`/`fakeConsumer`/`fakeRunner` internal/pbsdr/manager_test.go (argv+stdin recorder) |
|
||||
| `capability.Runner` | internal/capability/probe.go | `*proxmox.ExecRunner` (RunnerDirect) | `fakeRunner` internal/capability/probe_test.go |
|
||||
| Cross-repo: local API ↔ controller | internal/localapi/server.go routes; contract seeded by internal/provision/doc.go (`bootstrap.json`: endpoint + leaf fingerprint + token) | felhom-controller's agentapi client | pin = served leaf cert (memory gotcha) |
|
||||
| Cross-repo: agent ↔ hub | internal/hub/report.go (`HostReport`), `ControlEnvelope`; POST `/api/v1/host-report` | hub mirrors structs field-for-field | new event/report fields need hub-side ingest changes |
|
||||
| Cross-repo: shipped host artifacts | configs/felhom-agent.sudoers, configs/felhom-mkfs-guarded.sh, configs/felhom-pbs-apply, shared-parent script/unit (inline in internal/localapi/intermediary.go) | deployed WITH the binary | sudoers globs must match `stageTemp` patterns + staging dirs exactly |
|
||||
| Operator signing | internal/authz (OpBlob, SSHSIG) | cmd/felhom-opsign (offline CLI) | blob/verify tests in internal/authz |
|
||||
|
||||
## 5. Extension points (where new features plug in)
|
||||
|
||||
- **Local-API route**: add to `Server.Handler` (internal/localapi/server.go) wrapped in `s.withGuest`; new deps go into `Options` as OPTIONAL fields degrading to "not configured".
|
||||
- **New signed-job verb**: implement `signedjobs.Executor` (return `ErrNoExecutor` for foreign ops) and append to the `signedjobs.ExecutorChain` in cmd/felhom-agent/main.go; add the op class + role scoping in internal/reconcile (classify.go, gate.go `roleAuthorizes`).
|
||||
- **New privileged host op**: validate args (internal/storage/validate.go style) → exec via `Runner` → add a `Cmnd_Alias` to configs/felhom-agent.sudoers → add a probe vector to internal/capability/manifest.go (so degradation is visible) → ship sudoers with the binary.
|
||||
- **New reconcile action**: `ActionKind` + `classOfAction` (internal/reconcile/classify.go), plan emission in internal/reconcile/plan.go; destructive ⇒ gate handles it automatically.
|
||||
- **Hub-report field**: extend `hub.HostReport` (internal/hub/report.go) + `Collector` — hub side must mirror + allowlist it (cross-repo).
|
||||
- **DR-recipe section (host-half)**: add the field to `DRRecipeHostHalf` (internal/hub/dr_recipe.go) **AND** to
|
||||
the hub's `hostHalfShape` + `AssembledRecipe` (felhom.eu `hub/internal/store/dr_recipe.go`). Those two
|
||||
hub structs are **ALLOW-LISTS**: a section only the agent knows about is stored intact and silently
|
||||
dropped before any operator sees it — that is R-122, which cost `offsite_restic` its entire existence.
|
||||
Then update BOTH copies of `testdata/host-report.golden.json` (byte-identical, cross-repo) and extend
|
||||
`TestAssembleDRRecipe_CarriesEveryEmittedSection`. **A recipe field that cannot be resolved records an
|
||||
explicit unknown with a reason — never a default, an empty string, or a placeholder** (`DRState*` /
|
||||
`DRReason*`); a recipe read during a rebuild must not present a guess as a fact.
|
||||
- **Envelope-driven behavior**: implement `hub.EnvelopeObserver`, add to the `MultiObserver` in cmd/felhom-agent/main.go.
|
||||
- **Selftest mode**: `selftestFlag` + `runSelftest*` in cmd/felhom-agent/main.go.
|
||||
- **Config**: internal/config/config.go (`Load` + `applyEnv` `FELHOM_AGENT_*` overlay; keep secrets out of `Redacted()` output).
|
||||
|
||||
## 6. Known duplication (observed — NOT fixed)
|
||||
|
||||
- Two lsblk `-J` parsers with near-identical structs: `parseLsblkDevice`/`lsblkDevice` (internal/storage/hostops.go) vs `parseLsblkNodes`/`lsblkDev` (internal/storage/claim.go).
|
||||
- Two smartctl `-a -j` paths: `SudoHostOps.SMART` (internal/storage/hostops.go, parsed `hub.SmartSummary`) vs `Privileged.SMART` (internal/proxmox/privileged.go, raw map).
|
||||
- **SMART device resolution (v0.95.0):** `smartDeviceFor` (internal/storage/observe.go) resolves partition→disk AND dm/LVM→disk (`dmWholeDisk` in internal/storage/smartdev.go, via `/sys/block/<dm>/slaves`, `sysBlockRoot` test seam). `storage.SmartReader.SMARTForBacking` is the shared read the localapi `/disks` union path uses (Fix B) — do NOT re-implement smartctl parsing. The builtin-`local` SMART device comes from `containingMountDevice` (SMART-only; never feeds backing/durable_id).
|
||||
- Atomic tmp+rename JSON store implemented 3×: `IntentStore.saveLocked` (internal/storage/intent.go), `FormatJobStore.save` (internal/localapi/formatjob.go), `GuestBindStore.saveLocked` (internal/localapi/guestbindstore.go) — comments say "mirrors", no shared helper.
|
||||
- `run(ctx, name, args...) error` stderr-wrapping helper duplicated 4×: `SudoHostOps.run`, `Privileged.run`, `BackHalf.run` (internal/provision/backhalf.go), `GuestBinder.run` (internal/localapi/guestbind.go).
|
||||
- Several independent /proc mount-table readers: `SudoHostOps.mountedSet` (internal/storage/hostops.go), `ProcHostReader.Mounts` (internal/storage/hostread.go). **In localapi they were unified in v0.117.0**: `isHostMountpoint` and `countHostMounts` are now one-liners over `hostMountEntries`, the single parser that also yields devno/fstype/super-options for `bindLiveness`.
|
||||
- Deliberate mirror: `antiRetargetResolveExpect` (internal/localapi/wipe_reresolve.go) duplicates `WipeExecutor.Execute` steps 1–3 (internal/signedjobs/wipe.go) across packages.
|
||||
- `stableParentDir` literal duplicated in internal/provision/backhalf.go to avoid a provision→localapi import edge (commented as intentional); `trim` (internal/storage/hostops.go) vs `trimBody` (internal/proxmox/errors.go) output-truncation twins.
|
||||
@@ -0,0 +1,129 @@
|
||||
# RUNBOOK — provisioning the OS / Docker-data / user-data storage split
|
||||
|
||||
How the controller guest's **OS rootfs**, **Docker data**, and **SSD user-data** are carved onto three
|
||||
separate `local-lvm` volumes, and how to size them per node/customer. Grounded by the storage-split spike
|
||||
(`REPORT-storage-split-spike.md`). Defaults target a ~512 GB SSD: **32 GB OS rootfs + 200 GB Docker-data
|
||||
+ 50 GB user-data**, rest of the thin pool free for other guests.
|
||||
|
||||
## The design (locked)
|
||||
|
||||
Three thin volumes are baked into the golden and grown at provision:
|
||||
|
||||
| mp slot | Mount | Holds | Golden default | Provision grow knob |
|
||||
|---------|-------|-------|----------------|---------------------|
|
||||
| rootfs | `/` | OS only | `OS_SIZE_GB=32` | `-rootfs-grow` |
|
||||
| `mp0` | `/var/lib/docker` | ALL Docker images + overlay + named volumes | `GOLDEN_DOCKER_GB=16` | `-datavol-grow` |
|
||||
| `mp1` | `/mnt/sys_drive` | SSD user-data (the controller's `system_data_path`; `<sys_drive>/felhom-data` = SSD-only app data + DBs) | `GOLDEN_SYSDATA_GB=8` | `-sysdata-grow` |
|
||||
|
||||
- **Split for resilience, not backup size.** An isolated OS rootfs stays bootable + agent-recoverable
|
||||
if the Docker volume fills. All three volumes ride into PBS regardless (so the backup size is
|
||||
unchanged) — the win is independent sizing/growth and a smaller, faster-to-restore OS rootfs.
|
||||
- **One Docker daemon, one data-root, one volume** at `/var/lib/docker` for ALL images + overlay
|
||||
(controller, traefik, cloudflared, filebrowser, and customer apps). Infra is NOT relocated to the OS
|
||||
partition — infra/user separation under one daemon isn't cleanly possible and isn't needed. Infra is
|
||||
protected by the controller's **prevention layer** (reserved buffer + deploy gate + disk alerts +
|
||||
baked log rotation), not by placement.
|
||||
- **`/mnt/sys_drive` is a static CT volume, NOT an enrolled drive.** It never enrolls, ejects, or
|
||||
decommissions — it is part of the golden's baked layout. The controller auto-discovers
|
||||
`<sys_drive>/felhom-data` and, via `system.IsMountPoint`, **warns when `/mnt/sys_drive` is not its own
|
||||
mount** (`selftest.go`). Making `mp1` a real mount clears that warning with **zero controller change**;
|
||||
the controller is untouched by this slice. The `mp` mounted under the guest's `/mnt` propagates into
|
||||
the controller container through the existing `-v /mnt:/mnt:rslave` bind (the same path enrolled drives
|
||||
use).
|
||||
- **`backup=1` on the Docker-data AND user-data volumes is non-negotiable.** Extra LXC mountpoints
|
||||
default to `backup=0` = EXCLUDED from vzdump/PBS (spike B3). Without it, named-volume DBs / SSD app
|
||||
data fall out of the snapshot AND (for mp0) the baked images are excluded from the golden archive.
|
||||
|
||||
## Golden build (per controller-image baseline)
|
||||
|
||||
`configs/build-golden.sh` builds the template guest with the split baked in:
|
||||
|
||||
```
|
||||
# defaults shown; override via env
|
||||
OS_SIZE_GB=32 GOLDEN_DOCKER_GB=16 GOLDEN_SYSDATA_GB=8 \
|
||||
./build-golden.sh <VMID> <TEMPLATE> <ROOTFS_STORAGE> <ARCHIVE_STORAGE> <BRIDGE> <CONTROLLER_IMAGE>
|
||||
```
|
||||
|
||||
- `--rootfs ${ROOTFS_STORAGE}:${OS_SIZE_GB}`
|
||||
+ `--mp0 ${ROOTFS_STORAGE}:${GOLDEN_DOCKER_GB},mp=/var/lib/docker,backup=1`
|
||||
+ `--mp1 ${ROOTFS_STORAGE}:${GOLDEN_SYSDATA_GB},mp=/mnt/sys_drive,backup=1`.
|
||||
- Keep `GOLDEN_DOCKER_GB` / `GOLDEN_SYSDATA_GB` small (the golden ships them near-empty) — provision
|
||||
grows both. The controller creates `<sys_drive>/felhom-data` itself once `/mnt/sys_drive` is a real
|
||||
mount; the golden does NOT pre-create it.
|
||||
- The script bakes `/etc/docker/daemon.json` with **`features.containerd-snapshotter: false`** (the
|
||||
classic **overlay2** driver) + log rotation (`max-size 10m`, `max-file 3`), verifies `/var/lib/docker`
|
||||
AND `/mnt/sys_drive` are separate mounts + the driver is overlay2, and **aborts if vzdump excludes mp0
|
||||
or mp1** (the B3 trap).
|
||||
- **Why overlay2 (not the Docker-29 default containerd-snapshotter):** containerd-snapshotter keeps the
|
||||
image content store under `/var/lib/containerd`, which is NOT `/var/lib/docker` — so it would stay on
|
||||
the OS rootfs and the split would only move named volumes, leaving images (the bulk) on the rootfs
|
||||
(validated live: 1.2 GB of images landed on the rootfs). overlay2 stores **everything** (images +
|
||||
overlay + volumes) under data-root = the data volume, which is what the split + the controller's
|
||||
`statfs("/")` prevention guard both require.
|
||||
|
||||
## Provision (per customer guest)
|
||||
|
||||
The bring-up restores the golden (carrying rootfs + the small mp0 Docker-data + mp1 user-data volumes +
|
||||
baked images), then sizes per customer. Each volume is **grown** (not re-attached) so the baked images
|
||||
are never shadowed:
|
||||
|
||||
```
|
||||
felhom-agent --selftest=provision \
|
||||
-archive <golden-volid> -vmid <VMID> \
|
||||
-customer-id <id> -hub-password <retrieval-passphrase> \
|
||||
-rootfs-grow <N> # GiB to add to the OS rootfs beyond the golden's 32 (0 = keep 32)
|
||||
-datavol-grow <N> # GiB to add to the Docker-data volume beyond the golden's 16 (e.g. 184 → 200 total)
|
||||
-datavol-mount mp0 # the golden's Docker-data slot (default mp0)
|
||||
-sysdata-grow <N> # GiB to add to the user-data volume beyond the golden's 8 (e.g. 42 → 50 total)
|
||||
-sysdata-mount mp1 # the golden's user-data slot (default mp1)
|
||||
```
|
||||
|
||||
In code: `reconcile.BringUpSpec.{RootfsGrowGB, DataVolGrowGB, DataVolMount, SysDataGrowGB, SysDataMount}`
|
||||
— all grows are online (`pct resize`, spike B4), each its own task. **Separateness, not size, clears the
|
||||
controller warning** — `-sysdata-grow 0` still yields a separate, warning-free mount at the golden size;
|
||||
the grow is an orthogonal knob. `GuestMount.Backup` (→ `,backup=1`) is set on any data-bearing additive
|
||||
mount (`bringup.go` `buildBringUpConfig`). The bootstrap mount is `mp9`, the shared parent bind is `mp8`;
|
||||
USB/data drive enroll picks the lowest free slot (`freeMountSlot`), which auto-skips the baked mp0/mp1.
|
||||
|
||||
**Auto-start on host reboot (`onboot:1`).** The provision back-half sets the customer guest to
|
||||
**`onboot:1`** (`provision/backhalf.go`, v0.41.0) so the whole home-server (controller + apps) comes back
|
||||
on its own after a host reboot/power-cut. The **golden template stays `onboot:0`** (`build-golden.sh`) —
|
||||
a template must not auto-start. No `startup`/boot-order is used: the v0.75 mountpoint-gate covers the
|
||||
drive-bind race at boot, so the controller won't write app data onto the rootfs while drives re-bind.
|
||||
(Guests provisioned before v0.41.0 need a one-time `pct set <vmid> -onboot 1`.)
|
||||
|
||||
**Per-customer sizing source:** flags today; the **slice-10 hub storage manifest** is the intended
|
||||
home (marked at `bringup.go` `GuestMount` / the `slice 10 wires the hub storage manifest` comment) —
|
||||
the controller already pulls customer-scoped config from the hub, so OS/data sizes belong there.
|
||||
|
||||
### Default sizing guidance
|
||||
- ~512 GB SSD: 32 GB rootfs + 200 GB Docker-data + 50 GB user-data
|
||||
(`-rootfs-grow 0 -datavol-grow 184 -sysdata-grow 42`), leaving the rest of the thin pool free for
|
||||
other guests.
|
||||
- Other SSDs: size the Docker-data + user-data volumes to "most of the free thin pool minus a reserve
|
||||
for other guests." `local-lvm` is thin-provisioned, so an over-allocation only consumes space as it's
|
||||
written.
|
||||
|
||||
## Fresh PVE host install — thin-pool carving (host-level, do this at install)
|
||||
|
||||
This sizes the **host's** `local` root vs the `local-lvm` thin pool (the pool the guests' volumes are
|
||||
carved from). It is NOT the guest split — do it on the Proxmox ISO installer's **Advanced LVM options**:
|
||||
|
||||
- `hdsize` — total disk PVE uses (leave headroom on multi-disk nodes).
|
||||
- `maxroot` — cap on the host `/` (`pve/root`). Keep small (e.g. 20–40 GB) — the host OS is thin.
|
||||
- `maxvz` — cap on the `data` thin pool (`local-lvm`). Set to 0 / leave default to let the pool take the
|
||||
rest, OR cap it to reserve unallocated VG space.
|
||||
- `minfree` — reserved free space in the VG (thin-pool safety).
|
||||
- `swapsize` — host swap.
|
||||
|
||||
**Never repartition a live host root.** Per-guest carving (rootfs + Docker-data) is done at provision
|
||||
time via `pct` against the existing thin pool — no host repartition needed (the demo's pool already has
|
||||
~350 GB free).
|
||||
|
||||
## In-place migration of an EXISTING guest (NOT used by this slice)
|
||||
|
||||
This slice rebuilds clean (fresh re-provision), so the move-then-verify in-place migration is **not
|
||||
used here**. It remains documented in `REPORT-storage-split-spike.md` §B2 for a future in-place node
|
||||
upgrade: stop apps → stop docker → `rsync -aHAX --numeric-ids /var/lib/docker/ <new-vol>/` (check
|
||||
rsync's OWN exit code) → switch data-root → verify volumes+DBs+boot → only then reclaim. `backup=1` on
|
||||
the new mount and a CT restart to activate it are required (spike B3/B4).
|
||||
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// R-82 live regression (2026-07-26): the restore-test derived its tier from the CONFIGURED default
|
||||
// target instead of the archive's own storage. Restoring a `felhom-pbs:` archive on a box whose
|
||||
// primary target is "local" was classified "local" → the 10-minute local wait instead of the
|
||||
// generous PBS one → the wait expired mid-restore at 600s against a 14.46 GB WAN restore, teardown
|
||||
// fired at a still-restoring guest, and the scratch leaked.
|
||||
func TestArchiveStorageID(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"felhom-pbs:backup/ct/9201/2026-07-26T12:21:48Z", "felhom-pbs"},
|
||||
{"local:backup/vzdump-lxc-9201-2026_07_26-09_03_19.tar.zst", "local"},
|
||||
{"", ""},
|
||||
{"no-prefix", ""},
|
||||
{":leading-colon", ""}, // i>0 guard: a leading colon is not a storage id
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := archiveStorageID(c.in); got != c.want {
|
||||
t.Fatalf("archiveStorageID(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
|
||||
)
|
||||
|
||||
// R-973: a cadence-cap refusal becomes a CRITICAL degraded status — what the hub's existing
|
||||
// agent_capability_degraded event (operator-only) keys on — and clears when the refusal ages out.
|
||||
func TestBackupCapStatus_RefusalIsCriticalDegraded(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
s := backupCapStatus(func() (string, int, time.Time, bool) { return "felhom-pbs", 9201, at, true })
|
||||
if !s.Critical || s.Status != capability.StatusDegraded || !strings.Contains(s.Reason, "9201") || !strings.Contains(s.Reason, "felhom-pbs") {
|
||||
t.Fatalf("a refusal must raise a critical degraded status naming the guest and the tier: %+v", s)
|
||||
}
|
||||
if s := backupCapStatus(func() (string, int, time.Time, bool) { return "", 0, time.Time{}, false }); s.Status != capability.StatusOK || !s.Critical {
|
||||
t.Fatalf("no refusal → ok (and still critical, so a later refusal alerts): %+v", s)
|
||||
}
|
||||
if s := backupCapStatus(nil); s.Status != capability.StatusOK {
|
||||
t.Fatalf("no local API → ok: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// Wiring (the built-but-never-wired class): runDaemon's capability probe appends backupCapStatus, and the
|
||||
// late-bound source is set from the local API server.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-10-18): delete `backupCapRefusal = localSrv.BackupCapRefusal` →
|
||||
// "runDaemon never binds backupCapRefusal to the local API server". Restored.
|
||||
func TestR973_CapRefusalIsWiredIntoTheReport(t *testing.T) {
|
||||
_, f := parseMain(t)
|
||||
var appended, bound bool
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != "runDaemon" || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
switch x := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
if id, ok := x.Fun.(*ast.Ident); ok && id.Name == "backupCapStatus" && len(x.Args) == 1 {
|
||||
if a, ok := x.Args[0].(*ast.Ident); ok && a.Name == "backupCapRefusal" {
|
||||
appended = true
|
||||
}
|
||||
}
|
||||
case *ast.AssignStmt:
|
||||
if len(x.Lhs) == 1 && len(x.Rhs) == 1 {
|
||||
if l, ok := x.Lhs[0].(*ast.Ident); ok && l.Name == "backupCapRefusal" {
|
||||
if sel, ok := x.Rhs[0].(*ast.SelectorExpr); ok && sel.Sel.Name == "BackupCapRefusal" {
|
||||
bound = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !appended {
|
||||
t.Fatal("runDaemon's capability probe does not call backupCapStatus(backupCapRefusal)")
|
||||
}
|
||||
if !bound {
|
||||
t.Fatal("runDaemon never binds backupCapRefusal to the local API server")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestEscrowUploadContract pins the agent→hub escrow wire shape. It MUST match the hub's ingest
|
||||
// struct (felhom-hub api.escrowUploadRequest). Cross-repo, no shared module — this is the agent
|
||||
// half of the contract guard; the hub has the mirror in its own test.
|
||||
func TestEscrowUploadContract(t *testing.T) {
|
||||
b, _ := json.Marshal(escrowUploadRequest{BlobB64: "x", KeyFingerprint: "y", Posture: "z", CreatedAt: "t", ResticPwSHA256: "h"})
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
got = append(got, k)
|
||||
}
|
||||
sort.Strings(got)
|
||||
want := []string{"blob_b64", "created_at", "key_fingerprint", "posture", "restic_pw_sha256"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("escrow wire contract drift: got %v want %v (must match the hub ingest struct)", got, want)
|
||||
}
|
||||
// SLICE 3: no staged password folded in → the hash field is OMITTED on the wire (the hub stores NULL →
|
||||
// the controller never matches → stays pending; correct — the blob doesn't cover the key).
|
||||
b2, _ := json.Marshal(escrowUploadRequest{BlobB64: "x", KeyFingerprint: "y", Posture: "z", CreatedAt: "t"})
|
||||
var m2 map[string]any
|
||||
_ = json.Unmarshal(b2, &m2)
|
||||
if _, present := m2["restic_pw_sha256"]; present {
|
||||
t.Fatal("restic_pw_sha256 must be omitted when no staged password was sealed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
// R-1254 H2, the production question: does the hub's current package equal the pending upload body? Compared by the
|
||||
// identity blob the hub serves back. A package without an identity, or a hub that cannot answer, is never „held".
|
||||
// RED-PROOF: make hubHoldsEscrowPackage return true when the hub has any package → the „another package" case
|
||||
// passes as held → FAILS.
|
||||
func TestR1254_HubHoldsComparesTheIdentityBlob(t *testing.T) {
|
||||
body := func(id string) []byte {
|
||||
b, _ := json.Marshal(escrowUploadRequest{BlobB64: base64.StdEncoding.EncodeToString([]byte("k")),
|
||||
IdentityBlobB64: base64.StdEncoding.EncodeToString([]byte(id))})
|
||||
return b
|
||||
}
|
||||
hubWith := func(id string, present bool, err error) func(context.Context) (*hub.IdentityEscrowResponse, error) {
|
||||
return func(context.Context) (*hub.IdentityEscrowResponse, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &hub.IdentityEscrowResponse{Present: present, IdentityEscrowB64: base64.StdEncoding.EncodeToString([]byte(id))}, nil
|
||||
}
|
||||
}
|
||||
ctx := context.Background()
|
||||
if held, err := hubHoldsEscrowPackage(ctx, hubWith("NEW", true, nil), body("NEW")); err != nil || !held {
|
||||
t.Fatalf("the hub holds the new package: held=%v err=%v", held, err)
|
||||
}
|
||||
if held, err := hubHoldsEscrowPackage(ctx, hubWith("OLD", true, nil), body("NEW")); err != nil || held {
|
||||
t.Fatalf("the hub holds another package: held=%v err=%v", held, err)
|
||||
}
|
||||
if held, err := hubHoldsEscrowPackage(ctx, hubWith("", false, nil), body("NEW")); err != nil || held {
|
||||
t.Fatalf("the hub holds nothing: held=%v err=%v", held, err)
|
||||
}
|
||||
if _, err := hubHoldsEscrowPackage(ctx, hubWith("", false, errors.New("unreachable")), body("NEW")); err == nil {
|
||||
t.Fatal("a hub that cannot answer must be an error (unknown), never „not held\"")
|
||||
}
|
||||
noID, _ := json.Marshal(escrowUploadRequest{BlobB64: "aw=="})
|
||||
if _, err := hubHoldsEscrowPackage(ctx, hubWith("NEW", true, nil), noID); err == nil {
|
||||
t.Fatal("a package without an identity blob cannot be compared — an error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"go/ast"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/config"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
|
||||
)
|
||||
|
||||
// R-978, the root half: the controller-driven ceremony (json mode, the FELHOM_ESCROW argv with
|
||||
// `--upload`) must hand its package back instead of sending it — the daemon sends it at the claim.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-10-18): set `deferUpload: false` in escrowOptsFromCLI → this fails
|
||||
// with "json mode must defer the upload (R-978): got deferUpload=false", and the delivery test's json leg
|
||||
// then sends the package at create time ("put called 1 times"). Restored.
|
||||
func TestEscrowOptsFromCLI_JSONModeDefersUpload(t *testing.T) {
|
||||
// The FELHOM_ESCROW argv: --selftest=escrow-create --upload --output=json.
|
||||
if o := escrowOptsFromCLI("", false, false, true, "", true); !o.deferUpload || !o.upload {
|
||||
t.Fatalf("json mode must defer the upload (R-978): got deferUpload=%v upload=%v", o.deferUpload, o.upload)
|
||||
}
|
||||
// Text mode (an operator at a terminal) keeps the immediate upload; no --upload means no upload at all.
|
||||
if o := escrowOptsFromCLI("", false, false, true, "", false); o.deferUpload {
|
||||
t.Fatal("text mode must upload at once")
|
||||
}
|
||||
if o := escrowOptsFromCLI("", false, false, false, "", true); o.deferUpload {
|
||||
t.Fatal("without --upload there is nothing to defer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliverEscrowPackage_JSONModeSendsNothing(t *testing.T) {
|
||||
cfg := config.Config{}
|
||||
cfg.Hub.URL, cfg.Hub.HostID, cfg.Hub.APIKey = "https://hub.invalid", "host-1", "k"
|
||||
res := escrow.CreateResult{Blob: []byte("ciphertext"), KeyFingerprint: "f2:87", Posture: escrow.PostureZeroKnowledge, IdentityBlob: []byte("id")}
|
||||
|
||||
for _, jsonMode := range []bool{true, false} {
|
||||
calls := 0
|
||||
var sent []byte
|
||||
put := func(_ context.Context, _ config.Config, body []byte) error { calls++; sent = body; return nil }
|
||||
o := escrowOptsFromCLI("", false, false, true, "", jsonMode)
|
||||
pending, uploaded, err := deliverEscrowPackage(context.Background(), cfg, res, "abc", o.deferUpload, put)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jsonMode {
|
||||
if calls != 0 || uploaded {
|
||||
t.Fatalf("json mode: put called %d times (uploaded=%v), want 0 — the package must wait for the claim", calls, uploaded)
|
||||
}
|
||||
var m map[string]any
|
||||
if json.Unmarshal(pending, &m) != nil || m["blob_b64"] == "" || m["identity_blob_b64"] == "" || m["restic_pw_sha256"] != "abc" {
|
||||
t.Fatalf("json mode: the pending package is not the hub upload body: %s", pending)
|
||||
}
|
||||
} else if calls != 1 || !uploaded || pending != nil || len(sent) == 0 {
|
||||
t.Fatalf("text mode: put called %d times, uploaded=%v, pending=%q — want one immediate upload", calls, uploaded, pending)
|
||||
}
|
||||
}
|
||||
|
||||
// A hub that is not configured fails in both modes (as the pre-R-978 upload did).
|
||||
if _, _, err := deliverEscrowPackage(context.Background(), config.Config{}, res, "", true, nil); err == nil {
|
||||
t.Fatal("an unconfigured hub must fail the ceremony even when the upload is deferred")
|
||||
}
|
||||
}
|
||||
|
||||
// R-978 wiring (the built-but-never-wired class): runDaemon's localapi.EscrowCeremonyConfig carries an
|
||||
// Upload that calls putEscrowBody. Without it every claim of a new package answers 503 and no customer
|
||||
// can finish a ceremony.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-10-18): delete the `Upload:` line in runDaemon → this fails with
|
||||
// "runDaemon's EscrowCeremonyConfig has no Upload calling putEscrowBody". Restored.
|
||||
func TestR978_CeremonyUploadIsWiredIntoTheDaemon(t *testing.T) {
|
||||
_, f := parseMain(t)
|
||||
if !callsWithin(f, "main")["runDaemon"] {
|
||||
t.Fatal("main → runDaemon is broken — the path this test asserts is not the live one")
|
||||
}
|
||||
wired := false
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != "runDaemon" || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
cl, ok := n.(*ast.CompositeLit)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := cl.Type.(*ast.SelectorExpr)
|
||||
if !ok || sel.Sel.Name != "EscrowCeremonyConfig" {
|
||||
return true
|
||||
}
|
||||
for _, el := range cl.Elts {
|
||||
if kv, ok := el.(*ast.KeyValueExpr); ok {
|
||||
if k, ok := kv.Key.(*ast.Ident); ok && k.Name == "Upload" {
|
||||
ast.Inspect(kv.Value, func(n ast.Node) bool {
|
||||
if ce, ok := n.(*ast.CallExpr); ok {
|
||||
if id, ok := ce.Fun.(*ast.Ident); ok && id.Name == "putEscrowBody" {
|
||||
wired = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !wired {
|
||||
t.Fatal("runDaemon's EscrowCeremonyConfig has no Upload calling putEscrowBody")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Scenario H — THE SEAM IS WIRED IN THE PRODUCTION PATH, proven by walking the AST rather than by
|
||||
// grepping for a string.
|
||||
//
|
||||
// WHY THIS TEST EXISTS AND WHY IT IS AN AST WALK. This project's built-but-never-wired count is six,
|
||||
// and links 6 and 7 of the recovery chain were TWO of them: `UnwrapIdentityBundle` sat in the tree
|
||||
// for two months with no caller but a `--selftest`, and the hub's blob-serving endpoints have no
|
||||
// client to this day. The fix must not become the seventh. `strings.Contains` on the file would pass
|
||||
// against a commented-out line, a line inside a test helper, or a line in dead code behind a flag
|
||||
// nobody sets — so this resolves the call graph instead: `Options{EscrowRecovery: …}` must be
|
||||
// constructed inside a function that `runDaemon` reaches, and `runDaemon` must be reached by `main`.
|
||||
|
||||
func parseMain(t *testing.T) (*token.FileSet, *ast.File) {
|
||||
t.Helper()
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, parser.ParseComments)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing main.go: %v", err)
|
||||
}
|
||||
return fset, f
|
||||
}
|
||||
|
||||
// callsWithin returns the set of function names called (directly, by identifier or selector) inside
|
||||
// the named top-level function.
|
||||
func callsWithin(f *ast.File, fnName string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != fnName || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
ce, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
switch fn := ce.Fun.(type) {
|
||||
case *ast.Ident:
|
||||
out[fn.Name] = true
|
||||
case *ast.SelectorExpr:
|
||||
if x, ok := fn.X.(*ast.Ident); ok {
|
||||
out[x.Name+"."+fn.Sel.Name] = true
|
||||
}
|
||||
out[fn.Sel.Name] = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestEscrowRecoveryIsWiredIntoTheDaemon asserts the whole chain from func main() to the field.
|
||||
func TestEscrowRecoveryIsWiredIntoTheDaemon(t *testing.T) {
|
||||
_, f := parseMain(t)
|
||||
|
||||
// 1. main() reaches runDaemon.
|
||||
if !callsWithin(f, "main")["runDaemon"] {
|
||||
t.Fatal("func main() does not call runDaemon — the daemon path this test asserts is not the live one")
|
||||
}
|
||||
// 2. runDaemon reaches buildLocalAPIServer.
|
||||
if !callsWithin(f, "runDaemon")["buildLocalAPIServer"] {
|
||||
t.Fatal("runDaemon does not call buildLocalAPIServer — the local API is not built on the daemon path")
|
||||
}
|
||||
|
||||
// 3. Inside buildLocalAPIServer, a localapi.Options composite literal carries EscrowRecovery, and
|
||||
// an escrow.OffsiteKeyRecoverer is constructed there.
|
||||
var optionsHasField, recovererConstructed bool
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != "buildLocalAPIServer" || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
cl, ok := n.(*ast.CompositeLit)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := cl.Type.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
pkg, _ := sel.X.(*ast.Ident)
|
||||
if pkg == nil {
|
||||
return true
|
||||
}
|
||||
switch pkg.Name + "." + sel.Sel.Name {
|
||||
case "localapi.Options":
|
||||
for _, el := range cl.Elts {
|
||||
kv, ok := el.(*ast.KeyValueExpr)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if k, ok := kv.Key.(*ast.Ident); ok && k.Name == "EscrowRecovery" {
|
||||
optionsHasField = true
|
||||
}
|
||||
}
|
||||
case "escrow.OffsiteKeyRecoverer":
|
||||
recovererConstructed = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !recovererConstructed {
|
||||
t.Error("no escrow.OffsiteKeyRecoverer is constructed in buildLocalAPIServer — links 6→8 have no " +
|
||||
"production assembly point (the built-but-never-wired shape, seventh instance)")
|
||||
}
|
||||
if !optionsHasField {
|
||||
t.Error("localapi.Options in buildLocalAPIServer carries no EscrowRecovery field — the recoverer " +
|
||||
"exists and the route would answer 503 forever")
|
||||
}
|
||||
}
|
||||
|
||||
// The hub fetch must be the DAEMON's own hub client, not a freshly constructed one with different
|
||||
// credentials — the self-scoping that makes cross-host retrieval impossible is a property of WHICH
|
||||
// key is used.
|
||||
func TestEscrowRecoveryUsesTheDaemonHubClient(t *testing.T) {
|
||||
fset, f := parseMain(t)
|
||||
var fetchUsesHubClient bool
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != "buildLocalAPIServer" || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
ce, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := ce.Fun.(*ast.SelectorExpr)
|
||||
if !ok || sel.Sel.Name != "FetchIdentityEscrow" {
|
||||
return true
|
||||
}
|
||||
if x, ok := sel.X.(*ast.Ident); ok && x.Name == "hubClient" {
|
||||
fetchUsesHubClient = true
|
||||
} else {
|
||||
t.Errorf("FetchIdentityEscrow at %s is called on something other than the injected hub client",
|
||||
fset.Position(ce.Pos()))
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !fetchUsesHubClient {
|
||||
t.Fatal("the recoverer's fetcher does not call hubClient.FetchIdentityEscrow — either the fetch is " +
|
||||
"not wired, or it uses a client whose credentials are not this host's")
|
||||
}
|
||||
}
|
||||
|
||||
// The route itself must be registered on the local API. A handler with no route is the same defect
|
||||
// one layer down, and it has shipped here before.
|
||||
func TestRecoverRouteIsRegistered(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "../../internal/localapi/server.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parsing localapi/server.go: %v", err)
|
||||
}
|
||||
var registered bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
ce, ok := n.(*ast.CallExpr)
|
||||
if !ok || len(ce.Args) < 2 {
|
||||
return true
|
||||
}
|
||||
sel, ok := ce.Fun.(*ast.SelectorExpr)
|
||||
if !ok || sel.Sel.Name != "HandleFunc" {
|
||||
return true
|
||||
}
|
||||
lit, ok := ce.Args[0].(*ast.BasicLit)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(lit.Value, "/escrow/recover-offsite-password") {
|
||||
registered = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !registered {
|
||||
t.Fatal("POST /escrow/recover-offsite-password is not registered on the local API mux — the handler " +
|
||||
"exists and nothing can reach it")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-444: the weekly trim has the guestnet shape (component + reporter seam + goroutine), so its wiring is asserted
|
||||
// from the AST like TestMainWiresGuestNetWatchdog — a unit-green trim job that main.go never starts is the inert-seam
|
||||
// defect. It must also share the ONE heavy-op gate (heavyOps), or it could run beside a backup.
|
||||
func TestMainWiresGuestDiskTrim(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
var constructedWithGate, reporterWired, started bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch node := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
if fn, ok := node.Fun.(*ast.SelectorExpr); ok {
|
||||
switch fn.Sel.Name {
|
||||
case "New":
|
||||
if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "fstrim" && len(node.Args) >= 3 {
|
||||
if id, ok := node.Args[2].(*ast.Ident); ok && id.Name == "heavyOps" {
|
||||
constructedWithGate = true
|
||||
}
|
||||
}
|
||||
case "SetGuestDiskTrimReporter":
|
||||
reporterWired = true
|
||||
}
|
||||
}
|
||||
case *ast.GoStmt:
|
||||
if sel, ok := node.Call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Run" {
|
||||
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "diskTrim" {
|
||||
started = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !constructedWithGate {
|
||||
t.Error("main.go never calls fstrim.New(..., heavyOps, ...) — no trim job, or one outside the heavy-op gate")
|
||||
}
|
||||
if !reporterWired {
|
||||
t.Error("main.go never calls collector.SetGuestDiskTrimReporter — the guest_disk_trim stanza never reaches the hub")
|
||||
}
|
||||
if !started {
|
||||
t.Error("main.go never starts the trim job with `go diskTrim.Run(ctx)`")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// F10/rc255 (CAMPAIGN-3): a guest-hook phase body that PANICS must never crash the process — the hook
|
||||
// must return cleanly so the guest start proceeds (a nonzero exit blocks the start). runHookPhase
|
||||
// recovers the panic and returns.
|
||||
func TestRunHookPhase_PanicRecovered(t *testing.T) {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runHookPhase("9201", "pre-start", 5*time.Second, func(context.Context) {
|
||||
panic("simulated heal panic (e.g. a future Heal bug)")
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
// returned cleanly — the guest start would proceed
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("runHookPhase did not return after a panicking body (would have crashed the hook)")
|
||||
}
|
||||
}
|
||||
|
||||
// A phase body that overruns the timeout must be abandoned — the hook returns rather than hanging the
|
||||
// PVE start task. (The body's context is cancelled; the hook does not wait for the body to notice.)
|
||||
func TestRunHookPhase_TimeoutReturns(t *testing.T) {
|
||||
bodyCtxCancelled := make(chan struct{}, 1)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
runHookPhase("9201", "post-start", 20*time.Millisecond, func(ctx context.Context) {
|
||||
<-ctx.Done() // simulate a body that respects cancellation eventually
|
||||
bodyCtxCancelled <- struct{}{}
|
||||
})
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("runHookPhase did not return after the timeout (would hang the guest start)")
|
||||
}
|
||||
select {
|
||||
case <-bodyCtxCancelled:
|
||||
// the body's context was cancelled at the deadline — the intended signal
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("the phase body's context was not cancelled at the timeout")
|
||||
}
|
||||
}
|
||||
|
||||
// A body that errors (returns normally, no panic) is fine — the hook returns cleanly.
|
||||
func TestRunHookPhase_NormalBodyReturns(t *testing.T) {
|
||||
ran := false
|
||||
runHookPhase("9201", "pre-start", time.Second, func(context.Context) { ran = true })
|
||||
if !ran {
|
||||
t.Fatal("the phase body must run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The hook WIRING red-proof target: `guest-hook <vmid> post-start` must invoke the network
|
||||
// reassert with the vmid; pre-start and unknown phases must NOT. (Companion red-proof: remove the
|
||||
// PhasePostStart case from runGuestHook → the invoked assertion fails.)
|
||||
func TestRunGuestHook_PostStartInvokesNetworkReassert(t *testing.T) {
|
||||
orig := postStartNetworkReassertFn
|
||||
t.Cleanup(func() { postStartNetworkReassertFn = orig })
|
||||
|
||||
var gotVMIDs []string
|
||||
postStartNetworkReassertFn = func(_ context.Context, vmid string) {
|
||||
gotVMIDs = append(gotVMIDs, vmid)
|
||||
}
|
||||
|
||||
runGuestHook([]string{"9201", "post-start"})
|
||||
if len(gotVMIDs) != 1 || gotVMIDs[0] != "9201" {
|
||||
t.Fatalf("post-start must invoke the network reassert with vmid 9201, got %v", gotVMIDs)
|
||||
}
|
||||
|
||||
// pre-start must not touch the network reassert (it is the placeholder-heal phase; the heal
|
||||
// no-ops on a nonexistent config path and never blocks).
|
||||
runGuestHook([]string{"9201", "pre-start"})
|
||||
// unknown phases are ignored entirely.
|
||||
runGuestHook([]string{"9201", "pre-stop"})
|
||||
if len(gotVMIDs) != 1 {
|
||||
t.Fatalf("only post-start may invoke the network reassert, got %v", gotVMIDs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-54 §9 rule 6 — the seam-discipline test, and the one this repo has the most reason to write:
|
||||
// v0.91.0 shipped the PBS auth-probe seam with `main.go` never calling `SetAuthSink`, every unit
|
||||
// test green because they all injected the seam directly. The guestnet watchdog has the identical
|
||||
// shape (a component + a reporter seam + a goroutine), so its wiring is asserted here rather than
|
||||
// trusted.
|
||||
//
|
||||
// This walks the AST rather than grepping the source: a commented-out call still satisfies a
|
||||
// substring match (found while red-proofing the controller's twin of this test), and a comment is
|
||||
// not a caller.
|
||||
func TestMainWiresGuestNetWatchdog(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
|
||||
var constructed, reporterWired, started bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch node := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
switch fn := node.Fun.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
switch fn.Sel.Name {
|
||||
case "New":
|
||||
// guestnet.New(...)
|
||||
if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "guestnet" {
|
||||
constructed = true
|
||||
}
|
||||
case "SetGuestNetReporter":
|
||||
reporterWired = true
|
||||
}
|
||||
}
|
||||
case *ast.GoStmt:
|
||||
if sel, ok := node.Call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Watch" {
|
||||
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "gnWatchdog" {
|
||||
started = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if !constructed {
|
||||
t.Error("main.go never calls guestnet.New — the watchdog does not exist at runtime")
|
||||
}
|
||||
if !reporterWired {
|
||||
t.Error("main.go never calls collector.SetGuestNetReporter — the guest_net stanza would " +
|
||||
"never reach the hub (the exact v0.91.0 inert-seam defect)")
|
||||
}
|
||||
if !started {
|
||||
t.Error("main.go never starts the watchdog with `go gnWatchdog.Watch(ctx)` — it would be " +
|
||||
"constructed, reported on, and never probe anything")
|
||||
}
|
||||
}
|
||||
|
||||
// The watchdog must NOT join the errc fan-out: a guest-network watchdog that can terminate the
|
||||
// agent turns a customer's DHCP problem into an operator-plane outage. If it is ever changed to
|
||||
// `errc <- ...`, the drain bound at the bottom of main() also has to change — this catches the
|
||||
// first half of that mistake.
|
||||
func TestGuestNetWatchdogIsNotInTheErrcFanout(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
bad := false
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
send, ok := n.(*ast.SendStmt)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if ch, ok := send.Chan.(*ast.Ident); !ok || ch.Name != "errc" {
|
||||
return true
|
||||
}
|
||||
call, ok := send.Value.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Watch" {
|
||||
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "gnWatchdog" {
|
||||
bad = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if bad {
|
||||
t.Fatal("the guestnet watchdog was added to the errc fan-out — a watchdog over customer " +
|
||||
"guests must never be able to bring the agent down (and the drain bound in main() " +
|
||||
"would now be off by one, hanging shutdown)")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
)
|
||||
|
||||
// janitorInterval is how often the leftovers of an interrupted restore-test or backup are retried
|
||||
// (R-672 rule 3, R-673). Both used to be resolved ONLY at agent start: on 2026-09-24 a failed scratch
|
||||
// teardown kept a full thin pool full for 2.5 h, and a stale `snapshot-delete` lock blocked 9201's
|
||||
// whole-box backups for five hours — each cleared within a minute of an agent restart.
|
||||
const janitorInterval = 10 * time.Minute
|
||||
|
||||
// janitorDeps are the janitor's seams (tests drive one pass with fakes).
|
||||
type janitorDeps struct {
|
||||
retryScratch func(ctx context.Context) reconcile.ScratchRetryResult
|
||||
staleLocks func(ctx context.Context) // localapi Server.RecoverStaleLockedGuests; nil when the local API is off
|
||||
heavy *backup.InFlight
|
||||
record func(hub.RestoreTest)
|
||||
now func() time.Time
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// janitorPass is one pass. The stale-lock sweep runs only while holding the one-heavy-operation gate, so
|
||||
// no agent backup can START between its "no vzdump is running" check and its unlock (at start-up the
|
||||
// sweep ran before the backup loop existed; on a timer that ordering must be made, not assumed). A busy
|
||||
// gate skips the sweep this pass — the next pass retries.
|
||||
func janitorPass(ctx context.Context, d janitorDeps) {
|
||||
if d.retryScratch != nil {
|
||||
r := d.retryScratch(ctx)
|
||||
if r.Examined > 0 {
|
||||
d.logger.Info("janitor: restore-test scratch retry pass", "examined", r.Examined,
|
||||
"destroyed", r.Destroyed, "already_gone", r.Clean, "failed", r.Failed)
|
||||
}
|
||||
for _, vmid := range r.GaveUp {
|
||||
// The operator is told through the existing restore-test failure path: the hub raises
|
||||
// restore_test_failed (operator) once per distinct archive — this record's archive names
|
||||
// the stuck scratch guest.
|
||||
if d.record != nil {
|
||||
d.record(hub.RestoreTest{
|
||||
SourceArchive: fmt.Sprintf("scratch-teardown:%d", vmid),
|
||||
ScratchVMID: vmid,
|
||||
Pass: false,
|
||||
Error: fmt.Sprintf("restore-test scratch guest %d could not be torn down after %d retries — it holds its disks; remove it by hand (pct destroy %d) after checking what keeps it busy",
|
||||
vmid, reconcile.MaxTeardownTries, vmid),
|
||||
TestedAt: d.now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
if d.staleLocks != nil {
|
||||
release, busy, ok := d.heavy.TryAcquire("stale-lock-sweep")
|
||||
if !ok {
|
||||
d.logger.Info("janitor: stale-lock sweep deferred — a heavy operation is in flight", "busy", busy)
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
d.staleLocks(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// runJanitor runs janitorPass every janitorInterval until ctx ends.
|
||||
func runJanitor(ctx context.Context, d janitorDeps) {
|
||||
d.logger.Info("janitor: starting (restore-test scratch retry + stale-lock sweep)", "interval", janitorInterval)
|
||||
t := time.NewTicker(janitorInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
janitorPass(ctx, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
)
|
||||
|
||||
// R-672 / R-673 (v0.133.0): one janitor pass, driven with fakes.
|
||||
|
||||
func quiet() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) }
|
||||
|
||||
// A scratch the engine gave up on reaches the hub as a failed restore-test record naming it — the
|
||||
// existing operator path (restore_test_failed). Never a pass.
|
||||
func TestJanitor_GaveUpIsReportedAsAFailure(t *testing.T) {
|
||||
var got []hub.RestoreTest
|
||||
janitorPass(context.Background(), janitorDeps{
|
||||
retryScratch: func(context.Context) reconcile.ScratchRetryResult {
|
||||
return reconcile.ScratchRetryResult{Examined: 1, Failed: 1, GaveUp: []int{990000}}
|
||||
},
|
||||
heavy: &backup.InFlight{}, record: func(r hub.RestoreTest) { got = append(got, r) },
|
||||
now: time.Now, logger: quiet(),
|
||||
})
|
||||
if len(got) != 1 || got[0].Pass || got[0].ScratchVMID != 990000 || !strings.Contains(got[0].Error, "990000") {
|
||||
t.Fatalf("records = %+v — want one FAILED record naming scratch 990000", got)
|
||||
}
|
||||
}
|
||||
|
||||
// R-673: the stale-lock sweep runs only while holding the one-heavy-operation gate, so no agent backup can
|
||||
// start between its "no vzdump running" check and its unlock.
|
||||
//
|
||||
// COMPANION RED-PROOF (REPORT): drop the TryAcquire → "the sweep ran while a backup held the gate".
|
||||
func TestJanitor_StaleLockSweepWaitsForTheHeavyGate(t *testing.T) {
|
||||
heavy := &backup.InFlight{}
|
||||
swept := 0
|
||||
d := janitorDeps{staleLocks: func(context.Context) { swept++ }, heavy: heavy, now: time.Now, logger: quiet()}
|
||||
release, _, ok := heavy.TryAcquire("backup")
|
||||
if !ok {
|
||||
t.Fatal("setup")
|
||||
}
|
||||
janitorPass(context.Background(), d)
|
||||
if swept != 0 {
|
||||
t.Fatal("the sweep ran while a backup held the gate")
|
||||
}
|
||||
release()
|
||||
janitorPass(context.Background(), d)
|
||||
if swept != 1 {
|
||||
t.Fatalf("swept %d times with the gate free — want 1", swept)
|
||||
}
|
||||
if _, _, ok := heavy.TryAcquire("after"); !ok {
|
||||
t.Fatal("the sweep did not release the gate")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/linetest"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
||||
)
|
||||
|
||||
// R-1250: SetLineTest takes a LineTestReader and finds the request half by a type assertion, so a Tester that lost
|
||||
// WaitingForQuiet or RunNow would compile and silently answer 503 to every seeding box. This line makes it a build
|
||||
// failure instead.
|
||||
var _ localapi.LineTestRequester = (*linetest.Tester)(nil)
|
||||
|
||||
// R-1247: the weekly line test has the fstrim shape (component + reporter seam + goroutine) plus a local-API reader,
|
||||
// so its wiring is asserted from the AST. It must share the ONE heavy-op gate (heavyOps) — or it could run beside a
|
||||
// backup — and take its server from the tunnel manager, never a literal address.
|
||||
func TestMainWiresTheLineTest(t *testing.T) {
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
var withGate, withTunnelTarget, reporterWired, localAPIWired, started bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch node := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
if fn, ok := node.Fun.(*ast.SelectorExpr); ok {
|
||||
switch fn.Sel.Name {
|
||||
case "New":
|
||||
if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "linetest" && len(node.Args) >= 3 {
|
||||
if id, ok := node.Args[2].(*ast.Ident); ok && id.Name == "heavyOps" {
|
||||
withGate = true
|
||||
}
|
||||
if sel, ok := node.Args[1].(*ast.SelectorExpr); ok && sel.Sel.Name == "LineTestTarget" {
|
||||
withTunnelTarget = true
|
||||
}
|
||||
}
|
||||
case "SetLineTestReporter":
|
||||
reporterWired = true
|
||||
case "SetLineTest":
|
||||
localAPIWired = true
|
||||
}
|
||||
}
|
||||
case *ast.GoStmt:
|
||||
if sel, ok := node.Call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "Run" {
|
||||
if id, ok := sel.X.(*ast.Ident); ok && id.Name == "lineTest" {
|
||||
started = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !withGate {
|
||||
t.Error("main.go never calls linetest.New(..., heavyOps, ...) — no line test, or one outside the heavy-op gate")
|
||||
}
|
||||
if !withTunnelTarget {
|
||||
t.Error("linetest.New does not take its server from wgMgr.LineTestTarget")
|
||||
}
|
||||
if !reporterWired {
|
||||
t.Error("main.go never calls collector.SetLineTestReporter — the line_test stanza never reaches the hub")
|
||||
}
|
||||
if !localAPIWired {
|
||||
t.Error("main.go never calls localSrv.SetLineTest — the controller's GET /host/line-test always reads present:false")
|
||||
}
|
||||
if !started {
|
||||
t.Error("main.go never starts the line test with `go lineTest.Run(ctx)`")
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,91 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/osupdate"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
|
||||
)
|
||||
|
||||
// R-1285, in the REAL order (measured 2026-10-26 on demo-hp): the box gets agent_update and agent_config_update in one
|
||||
// pass; the OLD process installs the bundle (no policy pass in it), then restarts into the new agent, which dwells and
|
||||
// commits. The consequence asserted: the NEW agent runs the bundle policy pass exactly once, AFTER its commit — so a
|
||||
// BYO host loses our drop-in in the same update. No pass without a commit (no marker, a mismatched marker).
|
||||
// RED-PROOF: drop the wireBundlePolicyAfterCommit call in main → no bundle-policy plan after the commit.
|
||||
|
||||
// orderRunner records, in order, the self-update wrapper's verbs and the os-apply plans' modes.
|
||||
type orderRunner struct {
|
||||
mu sync.Mutex
|
||||
order []string
|
||||
}
|
||||
|
||||
func (r *orderRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if name == osupdate.WrapperPath && len(args) == 2 && args[0] == "--plan" {
|
||||
var p map[string]any
|
||||
b, _ := os.ReadFile(args[1])
|
||||
_ = json.Unmarshal(b, &p)
|
||||
mode, _ := p["mode"].(string)
|
||||
r.order = append(r.order, "os-apply:"+mode)
|
||||
return []byte(`OSAPPLY-REPORT {"mode":"bundle-policy","bundle_policy":{"install_mode":"byo","removed":["/etc/ssh/sshd_config.d/10-felhom-root-key-only.conf"]}}` + "\n"), nil, nil
|
||||
}
|
||||
if len(args) >= 1 {
|
||||
r.order = append(r.order, filepath.Base(name)+":"+args[0])
|
||||
}
|
||||
return []byte("ok"), nil, nil
|
||||
}
|
||||
|
||||
func (r *orderRunner) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
|
||||
return r.Run(ctx, name, args...)
|
||||
}
|
||||
|
||||
func r1285Start(t *testing.T, marker *selfupdate.PendingMarker, running string) []string {
|
||||
t.Helper()
|
||||
stateDir := t.TempDir()
|
||||
if marker != nil {
|
||||
dir := filepath.Join(stateDir, "selfupdate")
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := json.Marshal(marker)
|
||||
if err := os.WriteFile(filepath.Join(dir, "pending.json"), b, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
r := &orderRunner{}
|
||||
lg := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
mgr := selfupdate.NewManager(selfupdate.ManagerConfig{StateDir: stateDir, RunningVersion: running, Dwell: time.Millisecond, Runner: r, Logger: lg})
|
||||
leg := &osupdate.Leg{Runner: r, Logger: lg, PlanDir: t.TempDir(), StatePath: filepath.Join(t.TempDir(), "last")}
|
||||
exec := osupdate.ConfigUpdateExecutor{Leg: leg}
|
||||
wireBundlePolicyAfterCommit(mgr, exec, lg, running)
|
||||
mgr.MaybeCommit(context.Background())
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]string(nil), r.order...)
|
||||
}
|
||||
|
||||
func TestR1285_TheNewAgentRunsThePolicyPassAfterItsCommit(t *testing.T) {
|
||||
got := r1285Start(t, &selfupdate.PendingMarker{OldVersion: "0.166.0", NewVersion: "0.167.0", SHA256: "ab", AppliedAt: "t"}, "0.167.0")
|
||||
if len(got) != 2 || got[0] != "felhom-selfupdate-guarded:commit" || got[1] != "os-apply:bundle-policy" {
|
||||
t.Fatalf("the new agent must commit, then run ONE bundle policy pass: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR1285_NoCommitNoPolicyPass(t *testing.T) {
|
||||
if got := r1285Start(t, nil, "0.167.0"); len(got) != 0 {
|
||||
t.Fatalf("an ordinary start (no pending update) must run nothing: %v", got)
|
||||
}
|
||||
got := r1285Start(t, &selfupdate.PendingMarker{OldVersion: "0.166.0", NewVersion: "0.168.0", SHA256: "ab", AppliedAt: "t"}, "0.167.0")
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("a mismatched marker is not committed, so no pass: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/config"
|
||||
)
|
||||
|
||||
// R-861 (agent v0.146.0): the root escrow ceremony builds a file path from the storage id; an id that is a path is
|
||||
// refused before anything is read. RED-PROOF: drop the pbsStorageIDRe check → the "../" ids reach the key stat and
|
||||
// come back as a "setup" error instead of "usage".
|
||||
func TestEscrowCeremony_StorageIDIsNeverAPath(t *testing.T) {
|
||||
lg := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
for _, id := range []string{"../../../etc/shadow", "a/b", "/etc/pve/priv/x", ".hidden", ""} {
|
||||
cfg := config.Default()
|
||||
cfg.Escrow.PBSStorageID = "" // the flag decides here
|
||||
_, e := escrowCeremony(context.Background(), cfg, lg, escrowCeremonyOpts{storage: id})
|
||||
if e == nil || e.kind != "usage" {
|
||||
t.Errorf("storage id %q was not refused as usage (got %+v)", id, e)
|
||||
}
|
||||
}
|
||||
cfg := config.Default()
|
||||
cfg.Backup.PBSSecretDir = t.TempDir()
|
||||
_, e := escrowCeremony(context.Background(), cfg, lg, escrowCeremonyOpts{storage: "felhom-pbs"})
|
||||
if e == nil || e.kind != "setup" {
|
||||
t.Fatalf("control: a plain id must pass the check and fail later on the missing key (setup), got %+v", e)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/osupdate"
|
||||
)
|
||||
|
||||
type r866Fetcher struct{ err error }
|
||||
|
||||
func (f r866Fetcher) FetchDesiredState(context.Context) (*hub.DesiredStateResponse, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
r := &hub.DesiredStateResponse{}
|
||||
r.DesiredState.OSUpdate = &hub.WireOSUpdate{Ring: 1, Enabled: true}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// R-866 (v0.144.0). THE NIGHT'S SHAPE (A3, Tester 1 box, hub blocked): `selftest=os-update: desired state: hub:
|
||||
// transport error … connect: invalid argument` — the debug pass could not run at all. Now it runs from the block the
|
||||
// daemon saved, and its header says so.
|
||||
// COMPANION RED-PROOF: return at once on a fetch error in selftestOSBlock → "the pass did not run from the saved block".
|
||||
func TestR866_DebugPassUsesTheSavedBlockWhenTheHubIsAway(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
leg := &osupdate.Leg{PlanDir: dir}
|
||||
r := &hub.DesiredStateResponse{}
|
||||
r.DesiredState.OSUpdate = &hub.WireOSUpdate{Ring: 0, Enabled: true}
|
||||
leg.OnDesiredState(context.Background(), r) // the daemon received a block and saved it
|
||||
away := r866Fetcher{err: errors.New("hub: transport error: connect: invalid argument")}
|
||||
b, src, ok := selftestOSBlock(context.Background(), away, dir)
|
||||
if !ok || b == nil || b.Ring != 0 {
|
||||
t.Fatalf("the pass did not run from the saved block: ok=%v block=%+v src=%q", ok, b, src)
|
||||
}
|
||||
if !strings.HasPrefix(src, "SAVED(") || !strings.Contains(src, "hub unreachable") {
|
||||
t.Fatalf("the header must say the block is the saved one: %q", src)
|
||||
}
|
||||
// the hub reachable: its block wins, and the header says "hub"
|
||||
b, src, ok = selftestOSBlock(context.Background(), r866Fetcher{}, dir)
|
||||
if !ok || b.Ring != 1 || src != "hub" {
|
||||
t.Fatalf("hub block not used: %+v %q", b, src)
|
||||
}
|
||||
}
|
||||
|
||||
// No hub and nothing saved: the pass does not run on a guessed block.
|
||||
func TestR866_NoHubNoSavedBlockDoesNotRun(t *testing.T) {
|
||||
_, why, ok := selftestOSBlock(context.Background(), r866Fetcher{err: errors.New("down")}, t.TempDir())
|
||||
if ok || !strings.Contains(why, "no saved block") {
|
||||
t.Fatalf("ok=%v why=%q", ok, why)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-894 — the on-disk backup record is WIRED on the daemon path (the built-but-never-wired class).
|
||||
// main → runDaemon → buildLocalAPIServer, and inside it the localapi.Options literal carries
|
||||
// LastKnownBackups built by backup.NewBackupSuccessState. An AST walk, not a string match, for the
|
||||
// reasons in escrow_recover_wiring_test.go.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): delete the `LastKnownBackups:` line from buildLocalAPIServer → this
|
||||
// fails with "localapi.Options in buildLocalAPIServer has no LastKnownBackups field". Restored.
|
||||
//
|
||||
// 2026-10-09: the SAME instance also feeds the host report (collector.SetKnownBackupReporter), so a
|
||||
// restart right after a backup no longer erases the hub's evidence of it. The field may name a local
|
||||
// variable; the variable must be built by backup.NewBackupSuccessState and be the one handed to the
|
||||
// collector. RED-PROOF (observed): drop the SetKnownBackupReporter call → "not handed to the collector".
|
||||
func TestR894_LastKnownBackupsIsWiredIntoTheDaemon(t *testing.T) {
|
||||
_, f := parseMain(t)
|
||||
if !callsWithin(f, "main")["runDaemon"] || !callsWithin(f, "runDaemon")["buildLocalAPIServer"] {
|
||||
t.Fatal("main → runDaemon → buildLocalAPIServer is broken — the path this test asserts is not the live one")
|
||||
}
|
||||
var field, built bool
|
||||
var fieldVar, handed string
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != "buildLocalAPIServer" || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
cl, ok := n.(*ast.CompositeLit)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := cl.Type.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if pkg, _ := sel.X.(*ast.Ident); pkg == nil || pkg.Name+"."+sel.Sel.Name != "localapi.Options" {
|
||||
return true
|
||||
}
|
||||
for _, el := range cl.Elts {
|
||||
kv, ok := el.(*ast.KeyValueExpr)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if k, ok := kv.Key.(*ast.Ident); ok && k.Name == "LastKnownBackups" {
|
||||
field = true
|
||||
if callsIn(kv.Value)["backup.NewBackupSuccessState"] {
|
||||
built = true
|
||||
}
|
||||
if id, ok := kv.Value.(*ast.Ident); ok {
|
||||
fieldVar = id.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
// a local variable: built by NewBackupSuccessState, and handed to the collector
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
switch x := n.(type) {
|
||||
case *ast.AssignStmt:
|
||||
for i, l := range x.Lhs {
|
||||
if id, ok := l.(*ast.Ident); ok && fieldVar != "" && id.Name == fieldVar && i < len(x.Rhs) &&
|
||||
callsIn(x.Rhs[i])["backup.NewBackupSuccessState"] {
|
||||
built = true
|
||||
}
|
||||
}
|
||||
case *ast.CallExpr:
|
||||
if fn, ok := x.Fun.(*ast.SelectorExpr); ok && fn.Sel.Name == "SetKnownBackupReporter" && len(x.Args) == 1 {
|
||||
if id, ok := x.Args[0].(*ast.Ident); ok {
|
||||
handed = id.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !field {
|
||||
t.Fatal("localapi.Options in buildLocalAPIServer has no LastKnownBackups field")
|
||||
}
|
||||
if !built {
|
||||
t.Fatal("LastKnownBackups is not built by backup.NewBackupSuccessState")
|
||||
}
|
||||
if handed == "" || handed != fieldVar {
|
||||
t.Fatalf("the saved backups (%q) are not handed to the collector (SetKnownBackupReporter got %q)", fieldVar, handed)
|
||||
}
|
||||
}
|
||||
|
||||
func callsIn(n ast.Node) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
ast.Inspect(n, func(n ast.Node) bool {
|
||||
if ce, ok := n.(*ast.CallExpr); ok {
|
||||
if fn, ok := ce.Fun.(*ast.SelectorExpr); ok {
|
||||
if x, ok := fn.X.(*ast.Ident); ok {
|
||||
out[x.Name+"."+fn.Sel.Name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-86 Scenario I — the seam-discipline test for the due-check.
|
||||
//
|
||||
// A due-check is worth nothing if the daemon still wires the OLD picker: every unit test in
|
||||
// internal/backup would stay green (they inject the seam directly), the scheduler would ask for the
|
||||
// newest archive with no settle cutoff, and the per-archive rule would run against a candidate that
|
||||
// changes every time a backup lands. That is the same shape as the v0.91.0 inert seam — built,
|
||||
// tested, never called — and this repo has shipped it four times.
|
||||
//
|
||||
// It walks main.go's AST rather than grepping: a commented-out call still satisfies a substring
|
||||
// match, and a comment is not a caller.
|
||||
func TestMainWiresTheSettleAwareTierPicker(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var settlePicker, oldPicker, settleWired, evalInterval bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch node := n.(type) {
|
||||
case *ast.SelectorExpr:
|
||||
// runner.PickSettledRestoreCandidateOn passed as a value (not called).
|
||||
switch node.Sel.Name {
|
||||
case "PickSettledRestoreCandidateOn":
|
||||
settlePicker = true
|
||||
case "PickRestoreCandidateOn":
|
||||
oldPicker = true
|
||||
}
|
||||
case *ast.KeyValueExpr:
|
||||
key, ok := node.Key.(*ast.Ident)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if key.Name == "Settle" {
|
||||
settleWired = true
|
||||
}
|
||||
case *ast.CallExpr:
|
||||
if sel, ok := node.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "RestoreTestEvalInterval" {
|
||||
evalInterval = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if !settlePicker {
|
||||
t.Error("main.go never passes runner.PickSettledRestoreCandidateOn as the scheduler's TierPick — " +
|
||||
"the due-check would run without a settle cutoff, i.e. against an archive that may still be being written")
|
||||
}
|
||||
if oldPicker {
|
||||
t.Error("main.go still wires the pre-R-86 PickRestoreCandidateOn as a tier picker — " +
|
||||
"two pickers means the one under test is not the one running")
|
||||
}
|
||||
if !settleWired {
|
||||
t.Error("main.go never sets SchedulerOptions.Settle — the settle lag would default to 0 in the daemon " +
|
||||
"and every freshly-landed archive would be an immediate candidate")
|
||||
}
|
||||
if !evalInterval {
|
||||
t.Error("main.go never calls cfg.Backup.RestoreTestEvalInterval() — the scheduler would be driven by " +
|
||||
"the retired cadence knob")
|
||||
}
|
||||
}
|
||||
|
||||
// The two R-85 guarantees the due-check must not have quietly dropped: the spec is still built PER
|
||||
// RUN, and the shared heavy-operation gate is still handed to the scheduler.
|
||||
func TestMainStillWiresTheHeavyOperationGateAndPerRunSpec(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var inFlightWired, specIsAFunc bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
kv, ok := n.(*ast.KeyValueExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
key, ok := kv.Key.(*ast.Ident)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
switch key.Name {
|
||||
case "InFlight":
|
||||
inFlightWired = true
|
||||
case "Spec":
|
||||
// A FuncLit means it is evaluated per run; anything else is a frozen value.
|
||||
if _, isFunc := kv.Value.(*ast.FuncLit); isFunc {
|
||||
specIsAFunc = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if !inFlightWired {
|
||||
t.Error("main.go no longer hands the scheduler the shared InFlight gate — a restore-test could pull a " +
|
||||
"multi-GB archive over the same tunnel an offsite backup is pushing one over (Scenario F)")
|
||||
}
|
||||
if !specIsAFunc {
|
||||
t.Error("SchedulerOptions.Spec is no longer a function literal — a frozen spec is the R-85 defect " +
|
||||
"(the tier and its timeout evaluated once at daemon start, forever)")
|
||||
}
|
||||
}
|
||||
|
||||
func parseMainForWiring(t *testing.T) *ast.File {
|
||||
t.Helper()
|
||||
fset := token.NewFileSet()
|
||||
f, err := parser.ParseFile(fset, "main.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse main.go: %v", err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// R-189 Scenario I — the DURABLE proof source must actually be wired into the collector.
|
||||
//
|
||||
// This test exists because the method it feeds is the project's own cautionary tale:
|
||||
// `RestoreTestState.Snapshot` carried the doc comment "for the host-report gauge" from the day it
|
||||
// was written and **had no caller at all** — a seam built, documented and never connected, found
|
||||
// only when a live restore-test's PASS reached no host-report. The fix must not become the next
|
||||
// instance, so the wiring is asserted rather than trusted.
|
||||
//
|
||||
// AST, not grep: a commented-out call still contains the string (proven yesterday, when commenting
|
||||
// out the tier-picker line failed this test while a `strings.Contains` check would have passed).
|
||||
func TestMainWiresTheDurableRestoreTestProof(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var wired, feedsState bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok || sel.Sel.Name != "SetProvenRestoreTests" {
|
||||
return true
|
||||
}
|
||||
wired = true
|
||||
// ...and it must be fed the PERSISTED state, not the in-memory store.
|
||||
if len(call.Args) == 1 {
|
||||
if id, ok := call.Args[0].(*ast.Ident); ok && id.Name == "rtState" {
|
||||
feedsState = true
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if !wired {
|
||||
t.Error("main.go never calls collector.SetProvenRestoreTests — the persisted proof would never " +
|
||||
"reach the hub, which is the R-189 defect exactly: a passing restore-test that vanishes on restart")
|
||||
}
|
||||
if wired && !feedsState {
|
||||
t.Error("collector.SetProvenRestoreTests is not fed rtState — the in-memory store is the thing " +
|
||||
"that does NOT survive a restart, so wiring it here would fix nothing")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
||||
)
|
||||
|
||||
// COMPILE-TIME WITNESSES for OPTIONAL interfaces that are satisfied by a RUNTIME type assertion.
|
||||
//
|
||||
// WHY THIS FILE EXISTS. `localapi.BackupArchiveLister` is asserted at server.go's `newestArchiveOn`
|
||||
// via `tier.Service.(BackupArchiveLister)`. A failed assertion does not error — it degrades to
|
||||
// `archiveAbsent`, i.e. the pre-R-84 "ask the in-memory record only" behaviour. That degrade is
|
||||
// SILENT and it is behaviour-relevant: it is exactly the R-84 bug (a cold store after a restart
|
||||
// reading as "no backup ever") coming back, with nothing in any log to say so.
|
||||
//
|
||||
// The precedent is not hypothetical. During R-88 Part 2 the controller's `quiesceBackend` stopped
|
||||
// satisfying `quiesce.TieredBackend` when a signature changed, and `go build` AND `go vet` both
|
||||
// passed — because the interface is only ever asserted at runtime. Every box would have degraded to
|
||||
// the single-tier path, losing R-82's multi-tier backups, with no error anywhere. It was caught by
|
||||
// accident.
|
||||
//
|
||||
// A witness costs one line and converts that class of failure from a silent production degrade into
|
||||
// a compile error.
|
||||
//
|
||||
// THIS DOES NOT MAKE THE INTERFACE REQUIRED. The optionality is deliberate — it is what lets a
|
||||
// BackupService without a lister still work. The witness pins the IMPLEMENTATION (this concrete type
|
||||
// really does satisfy it), not the CONTRACT.
|
||||
var _ localapi.BackupArchiveLister = (*backup.BackupRunner)(nil)
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Every mode the dispatcher (`switch selftest.mode`) runs must be ACCEPTED by the --selftest flag. Found live
|
||||
// 2026-10-04: --selftest=os-update had a dispatch case and a function but the flag's allow-list refused it, so the
|
||||
// debug action could not run. Red-proof: drop the "os-update" case from selftestFlag.Set and this fails.
|
||||
func TestSelftestFlag_AcceptsEveryDispatchedMode(t *testing.T) {
|
||||
src, err := os.ReadFile("main.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := string(src)
|
||||
i := strings.Index(s, "switch selftest.mode {")
|
||||
if i < 0 {
|
||||
t.Fatal("dispatch switch not found")
|
||||
}
|
||||
block := s[i:]
|
||||
block = block[:strings.Index(block, "\n\t}\n")]
|
||||
modes := regexp.MustCompile(`(?m)^\tcase "([a-z-]+)":`).FindAllStringSubmatch(block, -1)
|
||||
if len(modes) < 5 {
|
||||
t.Fatalf("parsed only %d dispatch cases — the parser is wrong", len(modes))
|
||||
}
|
||||
var bad []string
|
||||
for _, m := range modes {
|
||||
var f selftestFlag
|
||||
if err := f.Set(m[1]); err != nil {
|
||||
bad = append(bad, m[1])
|
||||
}
|
||||
}
|
||||
if len(bad) > 0 {
|
||||
t.Fatalf("dispatched but refused by --selftest: %v", bad)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"go/ast"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
|
||||
)
|
||||
|
||||
// R-185 — a tier the box cannot READ must say so.
|
||||
//
|
||||
// THE OBSERVATION (demo-felhom, 2026-08-03, reproduced at the start of this session): root lists
|
||||
// three archives on `felhom-backup`; the agent's own token gets `{"data":[]}` from the same
|
||||
// endpoint; and `local`, which has the grant, lists through that same token. The token is the
|
||||
// variable, not the storage.
|
||||
//
|
||||
// The defect is NOT the missing grant — that is one command. It is that an empty content listing is
|
||||
// what a FORBIDDEN tier and a NEWBORN tier both return, so the box could not tell them apart and
|
||||
// said nothing. These tests pin the distinction.
|
||||
|
||||
// permAnswer is the shape /access/permissions really returns, taken from the live measurement:
|
||||
// an UNGRANTED path answers with the privileges inherited from the box-wide grant — NOT empty, and
|
||||
// NOT a 403.
|
||||
var (
|
||||
permGranted = map[string]int{"Datastore.Allocate": 1, "Datastore.AllocateSpace": 1}
|
||||
permUngranted = map[string]int{"Sys.Audit": 1, "SDN.Use": 1, "Datastore.Audit": 1}
|
||||
)
|
||||
|
||||
// probeWith calls the PRODUCTION decision with a permissions answer. **Naming the seam:** everything
|
||||
// below is true up to `storeGrantVerdict`; that the live call feeds it the real API answer is what
|
||||
// Part 0's measurement established and what the live run on the box demonstrates. An earlier draft
|
||||
// of this file re-implemented the branch here — it passed, and would have kept passing while
|
||||
// production diverged, which is the hollow shape this project keeps catching in its own tests.
|
||||
func probeWith(privs map[string]int, targetID string, critical bool) capability.Status {
|
||||
return storeGrantVerdict(targetID, critical, privs, nil)
|
||||
}
|
||||
|
||||
// ── SCENARIO A — a forbidden storage is REPORTED, not passed over ────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-03): delete the store-grant probes from `probeAll` in
|
||||
// main.go — i.e. restore `append(capProber.Probe(ctx), poolReadStatus(ctx, px))` — and
|
||||
// TestMainWiresTheStoreGrantProbe fails with "main.go never calls storeGrantStatuses". That is
|
||||
// today's behaviour on the live box: complete silence about a tier it cannot read.
|
||||
func TestStoreGrant_ForbiddenStorageIsDegradedAndNamed(t *testing.T) {
|
||||
s := probeWith(permUngranted, "felhom-backup", true)
|
||||
|
||||
if s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("a storage the agent may not read must be DEGRADED, not %q — silence is the defect", s.Status)
|
||||
}
|
||||
if !s.Critical {
|
||||
t.Fatal("it must be CRITICAL: the hub alerts only on critical, so a non-critical entry is the same silence with extra steps")
|
||||
}
|
||||
if !strings.Contains(s.Reason, "felhom-backup") {
|
||||
t.Fatalf("the reason must NAME the storage — 'a grant is missing' costs a diagnosis at 07:00; got %q", s.Reason)
|
||||
}
|
||||
if !strings.Contains(s.Reason, "FelhomAgentStore") {
|
||||
t.Fatalf("the reason must name the ROLE to grant, so the fix is in the alert; got %q", s.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// THE TRAP THE LIVE MEASUREMENT CAUGHT, pinned so it cannot be re-introduced: the ungranted answer
|
||||
// is not empty and not a 403 — it carries the INHERITED box-wide privileges. A probe that asked
|
||||
// "did the path come back?" or "does it have Datastore.Audit?" would report the blinded storage
|
||||
// healthy.
|
||||
func TestStoreGrant_InheritedPrivilegesAreNotAGrant(t *testing.T) {
|
||||
if len(permUngranted) == 0 {
|
||||
t.Fatal("fixture wrong: the ungranted answer is NOT empty — that is the whole trap")
|
||||
}
|
||||
if permUngranted["Datastore.Audit"] != 1 {
|
||||
t.Fatal("fixture wrong: the ungranted path DOES carry Datastore.Audit, inherited box-wide")
|
||||
}
|
||||
if s := probeWith(permUngranted, "felhom-backup", true); s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("checking for the wrong privilege reports a blinded storage healthy; got %q", s.Status)
|
||||
}
|
||||
// ...and the privilege actually checked is the one whose absence was measured to blind listing.
|
||||
if storeGrantRequiredPriv != "Datastore.AllocateSpace" {
|
||||
t.Fatalf("the probed privilege changed to %q — re-measure before trusting it", storeGrantRequiredPriv)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO B — a newborn tier is still silent ──────────────────────────────────────────────
|
||||
//
|
||||
// A storage the agent IS allowed to read but which simply holds no archives yet is HEALTHY. The
|
||||
// probe must not look at content at all, or every freshly provisioned box alarms and the signal dies.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): make the probe degrade on an empty content listing instead of on
|
||||
// the permission — a granted-but-empty storage then reports degraded, i.e. every newborn box alarms.
|
||||
func TestStoreGrant_GrantedButEmptyIsHealthy(t *testing.T) {
|
||||
s := probeWith(permGranted, "felhom-pbs", true)
|
||||
if s.Status != capability.StatusOK {
|
||||
t.Fatalf("a readable tier is healthy whether or not it holds archives yet; got %q (%s)", s.Status, s.Reason)
|
||||
}
|
||||
if s.Reason != "" {
|
||||
t.Fatalf("a healthy probe carries no reason; got %q", s.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO C — the two states are distinguishable at a glance ──────────────────────────────
|
||||
func TestStoreGrant_ForbiddenAndNewbornAreDistinguishable(t *testing.T) {
|
||||
forbidden := probeWith(permUngranted, "felhom-backup", true)
|
||||
newborn := probeWith(permGranted, "felhom-pbs", true)
|
||||
|
||||
if forbidden.Status == newborn.Status {
|
||||
t.Fatalf("the two states must differ — today both read as 'no settled archive yet'; got %q for both", forbidden.Status)
|
||||
}
|
||||
if forbidden.Name == newborn.Name {
|
||||
t.Fatalf("each tier needs its own capability id, or one tier's fault hides another's; got %q twice", forbidden.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// §8.3, weighed once and pinned: a box with NO dedicated target ("local" — host-install's own
|
||||
// DEGRADED fallback) must not turn an ordinary configuration into an operator page. It is still
|
||||
// probed and still reported; only the paging differs.
|
||||
func TestStoreGrant_TheFallbackTargetIsNotCritical(t *testing.T) {
|
||||
if storeGrantCritical("local") {
|
||||
t.Fatal("a box whose backup target is the 'local' fallback must not page the operator about " +
|
||||
"an ordinary, documented configuration")
|
||||
}
|
||||
for _, dedicated := range []string{"felhom-backup", "felhom-pbs", "some-nvme"} {
|
||||
if !storeGrantCritical(dedicated) {
|
||||
t.Fatalf("a DEDICATED target that cannot be read is user-facing and must be critical; %q was not", dedicated)
|
||||
}
|
||||
}
|
||||
// The fallback is still reported — silence for it would be the original defect, scoped smaller.
|
||||
if s := probeWith(permUngranted, "local", storeGrantCritical("local")); s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("the fallback target must still report degraded when unreadable; got %q", s.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// A probe that cannot ask must never answer "ok" — unknown reported as healthy is worse than no
|
||||
// probe, because it looks like coverage.
|
||||
func TestStoreGrant_UnreachablePVEIsDegradedNotOK(t *testing.T) {
|
||||
s := storeGrantStatus(context.Background(), nil, "felhom-backup", true, nil)
|
||||
if s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("an unaskable probe must be DEGRADED, never ok; got %q", s.Status)
|
||||
}
|
||||
if s.Reason == "" {
|
||||
t.Fatal("it must say why it could not ask")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO H — the seam ────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// This project's "built but never wired" count reached six last week. The fix for a SILENCE must not
|
||||
// itself be silent. AST, not grep: a commented-out call still contains the string.
|
||||
func TestMainWiresTheStoreGrantProbe(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var wired bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "storeGrantStatuses" {
|
||||
wired = true
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !wired {
|
||||
t.Error("main.go never calls storeGrantStatuses — the probe would exist and report to nobody, " +
|
||||
"which is precisely the silence R-185 is about")
|
||||
}
|
||||
}
|
||||
|
||||
// ── R-190 — the grant repairs itself, and the repair is VISIBLE ──────────────────────────────
|
||||
//
|
||||
// R-190 is a storage grant that demonstrably worked at 04:44 on 2026-08-03 and was gone by 09:24,
|
||||
// with a host reinstall, logged `pveum` activity and cluster-log entries all ruled out. The cause is
|
||||
// open; the resilience is not conditional on it.
|
||||
//
|
||||
// The half that matters is the RECORD. R-190's own words: the probe sees the state, nothing sees the
|
||||
// transition. A self-repair that leaves only "ok" behind destroys the only evidence a loss happened,
|
||||
// so a recurring loss becomes undetectable forever — strictly worse than the fault it fixes.
|
||||
|
||||
// fakeRepairRunner records wrapper invocations and can be made to fail.
|
||||
type fakeRepairRunner struct {
|
||||
calls [][]string
|
||||
fail bool
|
||||
}
|
||||
|
||||
func (f *fakeRepairRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||||
f.calls = append(f.calls, append([]string{name}, args...))
|
||||
if f.fail {
|
||||
return nil, []byte("pveum: refused"), errors.New("exit status 2")
|
||||
}
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func newRepairer(f *fakeRepairRunner) *storeGrantRepairer {
|
||||
return &storeGrantRepairer{run: f.Run, log: slog.New(slog.NewTextHandler(io.Discard, nil))}
|
||||
}
|
||||
|
||||
// ── SCENARIO F — the repair is BOUNDED ───────────────────────────────────────────────────────
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-04): make mayAttempt always return true (drop the
|
||||
// storeGrantRepairMinInterval check) →
|
||||
//
|
||||
// --- FAIL: TestGrantRepair_IsBounded
|
||||
// storegrant_test.go: a repair must not run on every cycle; 5 cycles produced 5 attempt(s)
|
||||
//
|
||||
// which is a re-grant every report cycle, forever, against a fault an ACL cannot fix. Restored.
|
||||
func TestGrantRepair_IsBounded(t *testing.T) {
|
||||
f := &fakeRepairRunner{}
|
||||
r := newRepairer(f)
|
||||
// Jittered, so the series never lands exactly on the interval boundary — a perfectly regular
|
||||
// series is how a threshold test passes its own mutation, which has happened here before.
|
||||
base := time.Date(2026, 8, 4, 9, 17, 43, 0, time.UTC)
|
||||
offsets := []time.Duration{0, 13*time.Minute + 7*time.Second, 27*time.Minute + 51*time.Second,
|
||||
41*time.Minute + 19*time.Second, 55*time.Minute + 3*time.Second}
|
||||
attempts := 0
|
||||
for _, off := range offsets {
|
||||
if r.mayAttempt("felhom-backup", base.Add(off)) {
|
||||
attempts++
|
||||
}
|
||||
}
|
||||
if attempts != 1 {
|
||||
t.Fatalf("a repair must not run on every cycle; %d cycles produced %d attempt(s) within %s",
|
||||
len(offsets), attempts, storeGrantRepairMinInterval)
|
||||
}
|
||||
// ...and once the interval has genuinely passed, it may try again — a bound is not a ban.
|
||||
if !r.mayAttempt("felhom-backup", base.Add(storeGrantRepairMinInterval+2*time.Minute+11*time.Second)) {
|
||||
t.Fatal("after the interval a repair must be allowed again — otherwise one failure disables the repair forever")
|
||||
}
|
||||
// A DIFFERENT tier is not throttled by this one's attempt.
|
||||
if !r.mayAttempt("felhom-pbs", base.Add(time.Minute)) {
|
||||
t.Fatal("the bound must be per tier — one tier's attempt must not suppress another's")
|
||||
}
|
||||
}
|
||||
|
||||
// A nil repairer (or one with no runner) never attempts, and never panics.
|
||||
func TestGrantRepair_NilIsSafe(t *testing.T) {
|
||||
var r *storeGrantRepairer
|
||||
if r.mayAttempt("felhom-backup", time.Now()) {
|
||||
t.Fatal("a nil repairer must never claim an attempt")
|
||||
}
|
||||
if (&storeGrantRepairer{}).mayAttempt("felhom-backup", time.Now()) {
|
||||
t.Fatal("a repairer with no runner must never claim an attempt")
|
||||
}
|
||||
}
|
||||
|
||||
// The repair calls the EXISTING wrapper verb, with the storage id — no new privileged surface.
|
||||
func TestGrantRepair_CallsTheExistingWrapperVerb(t *testing.T) {
|
||||
f := &fakeRepairRunner{}
|
||||
r := newRepairer(f)
|
||||
if err := r.repair(context.Background(), "felhom-backup"); err != nil {
|
||||
t.Fatalf("repair should succeed with a healthy runner: %v", err)
|
||||
}
|
||||
if len(f.calls) != 1 {
|
||||
t.Fatalf("exactly one wrapper invocation expected; got %d", len(f.calls))
|
||||
}
|
||||
got := f.calls[0]
|
||||
want := []string{"/usr/local/sbin/felhom-backup-target-apply", "grant", "felhom-backup"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("wrapper argv = %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("wrapper argv = %v, want %v — the sudoers vector is `grant *`; anything else is a policy change", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A repair that FAILS must surface the failure, not swallow it (Scenario E's precondition).
|
||||
func TestGrantRepair_FailureIsReturned(t *testing.T) {
|
||||
f := &fakeRepairRunner{fail: true}
|
||||
if err := newRepairer(f).repair(context.Background(), "felhom-backup"); err == nil {
|
||||
t.Fatal("a failed wrapper run must return its error — a repair that cannot run must never read as done")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO D (the half that matters) — the REPAIR MUST BE VISIBLE ──────────────────────────
|
||||
//
|
||||
// A repair that leaves only "ok" behind is worse than the fault: the tier works, and the fact that a
|
||||
// permission vanished is gone with it. R-190 exists because nothing saw the transition.
|
||||
//
|
||||
// The channel is the hub's EXISTING ok→degraded→ok edge (§8.5) — nothing new was built. That only
|
||||
// works if the agent deliberately reports ONE degraded cycle after repairing, and if the explanation
|
||||
// rides the field the hub actually puts in the operator's e-mail. The hub's message is built from the
|
||||
// capability NAME and FEATURE (`internal/monitor/host_capability.go` emitTransition) — **not** from
|
||||
// Reason — so the Feature must carry it.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-08-04): after a successful repair, report ok instead —
|
||||
//
|
||||
// s.Status = capability.StatusOK; s.Feature unchanged
|
||||
//
|
||||
// → --- FAIL: TestGrantRepair_ARepairedGrantIsReportedAsATransition
|
||||
//
|
||||
// storegrant_test.go: a self-repair must still report DEGRADED for one cycle so the hub raises
|
||||
// its edge; got "ok" — the loss would be invisible
|
||||
//
|
||||
// i.e. exactly the silence R-190 is about. Restored.
|
||||
func TestGrantRepair_ARepairedGrantIsReportedAsATransition(t *testing.T) {
|
||||
// THE PRODUCTION verdict, not a copy of it. An earlier draft of this test built the Status
|
||||
// itself and asserted its own construction — it would have passed while production reported ok,
|
||||
// which is precisely the silence being guarded against.
|
||||
if pre := probeWith(permUngranted, "felhom-backup", true); pre.Status != capability.StatusDegraded {
|
||||
t.Fatalf("precondition: a missing grant is degraded; got %q", pre.Status)
|
||||
}
|
||||
s := storeGrantRepairedVerdict("felhom-backup", true)
|
||||
|
||||
if s.Status != capability.StatusDegraded {
|
||||
t.Fatalf("a self-repair must still report DEGRADED for one cycle so the hub raises its edge; "+
|
||||
"got %q — the loss would be invisible", s.Status)
|
||||
}
|
||||
// The hub e-mails the FEATURE text. If the explanation is not there, the operator is told a
|
||||
// capability was degraded and never learns it repaired itself or that anything vanished.
|
||||
for _, want := range []string{"MISSING", "RESTORED", "felhom-backup", "R-190"} {
|
||||
if !strings.Contains(s.Feature, want) {
|
||||
t.Fatalf("the Feature text is what the hub puts in the operator's e-mail; it must contain %q. Got: %s", want, s.Feature)
|
||||
}
|
||||
}
|
||||
if !s.Critical {
|
||||
t.Fatal("the transition must be CRITICAL or the hub does not alert on it at all")
|
||||
}
|
||||
}
|
||||
|
||||
// ── SCENARIO H — the seam ────────────────────────────────────────────────────────────────────
|
||||
//
|
||||
// The wrapper's `grant` verb is itself a "built but never wired" example: it exists, is
|
||||
// sudoers-permitted for any id, and had only ever been called at storage CREATION. The repair must
|
||||
// not become the seventh instance. AST, not grep — a commented-out call still contains the string.
|
||||
func TestMainWiresTheGrantRepair(t *testing.T) {
|
||||
f := parseMainForWiring(t)
|
||||
|
||||
var built, passed bool
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch node := n.(type) {
|
||||
case *ast.CompositeLit:
|
||||
if id, ok := node.Type.(*ast.Ident); ok && id.Name == "storeGrantRepairer" {
|
||||
built = true
|
||||
}
|
||||
case *ast.CallExpr:
|
||||
if id, ok := node.Fun.(*ast.Ident); ok && id.Name == "storeGrantStatuses" && len(node.Args) == 4 {
|
||||
if a, ok := node.Args[3].(*ast.Ident); ok && a.Name == "grantRepairer" {
|
||||
passed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
if !built {
|
||||
t.Error("main.go never constructs a storeGrantRepairer — nothing would ever repair a lost grant")
|
||||
}
|
||||
if !passed {
|
||||
t.Error("storeGrantStatuses is not passed the repairer — the probe would detect the loss and " +
|
||||
"leave it, which is v0.123.0's behaviour and not R-190's mitigation")
|
||||
}
|
||||
}
|
||||
|
||||
// The transition must survive a probe that is NOT the one feeding the hub.
|
||||
//
|
||||
// MEASURED LIVE 2026-08-04, and this test exists because the first implementation failed it in
|
||||
// production while every unit test passed: `probeAll` is called independently by the self-check LOG
|
||||
// and by the collector building a host-report. The repairing call was the log's; the report three
|
||||
// seconds later found the grant present and reported `ok`. The agent's journal had the record and the
|
||||
// hub had nothing — the exact silence R-190 is about, re-created inside its own mitigation.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): delete the `recentlyRepaired` branch from the healthy path →
|
||||
//
|
||||
// --- FAIL: TestGrantRepair_TransitionSurvivesALaterProbe
|
||||
// storegrant_test.go: a probe AFTER the repair must still report the transition; got "ok" —
|
||||
// the host-report would carry ok and the operator would never learn the grant vanished
|
||||
//
|
||||
// Restored.
|
||||
func TestGrantRepair_TransitionSurvivesALaterProbe(t *testing.T) {
|
||||
r := newRepairer(&fakeRepairRunner{})
|
||||
// Jittered, never landing on the window boundary.
|
||||
repairedAt := time.Date(2026, 8, 4, 9, 39, 34, 0, time.UTC)
|
||||
r.noteRepaired("felhom-backup", repairedAt)
|
||||
|
||||
// The DECISION a later probe makes — the production function, not the helper it calls. An
|
||||
// earlier draft asserted `recentlyRepaired` directly and its red-proof PASSED, because removing
|
||||
// the latch's USE left the helper untouched.
|
||||
healthy := probeWith(permGranted, "felhom-backup", true)
|
||||
if healthy.Status != capability.StatusOK {
|
||||
t.Fatalf("precondition: a granted tier is ok; got %q", healthy.Status)
|
||||
}
|
||||
got := storeGrantHealthyVerdict("felhom-backup", true,
|
||||
healthy, r.recentlyRepaired("felhom-backup", repairedAt.Add(3*time.Second)))
|
||||
if got.Status != capability.StatusDegraded {
|
||||
t.Fatalf("a probe AFTER the repair must still report the transition; got %q — the host-report "+
|
||||
"would carry ok and the operator would never learn the grant vanished", got.Status)
|
||||
}
|
||||
if !strings.Contains(got.Feature, "RESTORED") {
|
||||
t.Fatalf("the later probe must carry the explanation into the hub's e-mail; got: %s", got.Feature)
|
||||
}
|
||||
// Outside the window it reports plain ok again.
|
||||
late := storeGrantHealthyVerdict("felhom-backup", true,
|
||||
healthy, r.recentlyRepaired("felhom-backup", repairedAt.Add(storeGrantRepairReportWindow+time.Minute)))
|
||||
if late.Status != capability.StatusOK {
|
||||
t.Fatalf("outside the window a healthy tier reports ok; got %q — a permanent degraded state "+
|
||||
"would be its own false alarm", late.Status)
|
||||
}
|
||||
if !r.recentlyRepaired("felhom-backup", repairedAt.Add(14*time.Minute+37*time.Second)) {
|
||||
t.Fatal("the latch must outlast the 900s hub report interval, or the record never reaches the hub")
|
||||
}
|
||||
// ...and it clears on its own rather than latching a box degraded forever.
|
||||
if r.recentlyRepaired("felhom-backup", repairedAt.Add(storeGrantRepairReportWindow+time.Minute+7*time.Second)) {
|
||||
t.Fatal("the latch must clear — a permanent degraded state would be its own false alarm")
|
||||
}
|
||||
// It is per tier.
|
||||
if r.recentlyRepaired("felhom-pbs", repairedAt.Add(time.Second)) {
|
||||
t.Fatal("one tier's repair must not latch another tier's status")
|
||||
}
|
||||
// The window MUST exceed the report interval — the property, asserted rather than assumed.
|
||||
if storeGrantRepairReportWindow <= 15*time.Minute {
|
||||
t.Fatalf("the report window (%s) must exceed the 900s hub report interval, or a transition can "+
|
||||
"be missed entirely", storeGrantRepairReportWindow)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
// Command felhom-opsign is the OPERATOR's offline signing CLI for destructive ops (slice 10B,
|
||||
// decision (a): offline operator key + signing CLI, hardware-key-ready).
|
||||
//
|
||||
// It constructs the canonical OpBlob bytes by REUSING internal/authz.CanonicalBlob — the exact
|
||||
// production path the agent's verifier authenticates over — so signer and verifier can never drift.
|
||||
// It signs that canonical message with the operator's key via `ssh-keygen -Y sign`, which makes it
|
||||
// hardware-ready: an `sk-`/YubiKey key works through ssh-keygen unchanged. The output is a signed-op
|
||||
// envelope { op_blob_b64, sig_armored } to hand to the hub's jobs queue (optionally uploaded with
|
||||
// --upload). This CLI touches ONLY the operator's signing key — never the hub's or agent's keys.
|
||||
//
|
||||
// Example — sign a data-bearing wipe (closing the 8C pending_signature gap):
|
||||
//
|
||||
// felhom-opsign -op storage_wipe -host demo-felhom-01 \
|
||||
// -durable-id byid:wwn-0x5000c500abcd1234 -fstype ext4 \
|
||||
// -key-id ops-key-1 -key ~/.ssh/felhom_op_ed25519 -ttl 30m
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/authz"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "felhom-opsign:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
var (
|
||||
op = flag.String("op", "", "op class to sign, e.g. storage_wipe | guest_destroy | decommission | agent_update | os_docker_step | os_pve_step | os_kernel_step | agent_config_update | add_internal_place")
|
||||
host = flag.String("host", "", "target host_id (anti-retarget — the op runs ONLY on this host)")
|
||||
guest = flag.String("guest", "", "target guest_id (\"\" = host-scoped op)")
|
||||
keyID = flag.String("key-id", "", "key id of the signing key (must match a pinned agent signer)")
|
||||
paramsRaw = flag.String("params", "", "op params as JSON (overrides -durable-id/-fstype)")
|
||||
durableID = flag.String("durable-id", "", "storage_wipe: the DURABLE device id (byid:…|byuuid:…); decommission: the drive's STORAGE durable-id (e.g. uuid:<fs-uuid>)")
|
||||
fstype = flag.String("fstype", "ext4", "for storage_wipe: the filesystem to mkfs after wipe")
|
||||
agentVer = flag.String("agent-version", "", "for agent_update: the target agent version (e.g. 0.70.1)")
|
||||
sha256Hex = flag.String("sha256", "", "for agent_update: the pinned lowercase-hex sha256 of the target binary")
|
||||
bundleSHA = flag.String("bundle-sha256", "", "for agent_config_update: the pinned sha256 of felhom-config-bundle.json (R-840)")
|
||||
keyFile = flag.String("key", "", "operator signing key (ssh private key / sk- key handle) for ssh-keygen -Y sign")
|
||||
ttl = flag.Duration("ttl", 30*time.Minute, "validity window from now (issued_at..expires_at)")
|
||||
nonce = flag.String("nonce", "", "explicit nonce (default: a fresh 128-bit random nonce)")
|
||||
uploadURL = flag.String("upload", "", "optional hub base URL to POST the signed op to the jobs queue")
|
||||
hubKey = flag.String("hub-key", "", "hub bearer key for --upload (operator's hub key — NOT a signing key)")
|
||||
out = flag.String("o", "", "write the envelope JSON to this file (default: stdout)")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
if *op == "" || *host == "" || *keyID == "" || *keyFile == "" {
|
||||
return fmt.Errorf("-op, -host, -key-id and -key are required")
|
||||
}
|
||||
|
||||
// Params: explicit JSON, or built from the convenience flags.
|
||||
params := strings.TrimSpace(*paramsRaw)
|
||||
if params == "" {
|
||||
switch *op {
|
||||
case "storage_wipe":
|
||||
if *durableID == "" {
|
||||
return fmt.Errorf("storage_wipe needs -durable-id (byid:…|byuuid:…) — a path-only binding is refused by the agent")
|
||||
}
|
||||
pj, _ := json.Marshal(map[string]string{"durable_id": *durableID, "fstype": *fstype})
|
||||
params = string(pj)
|
||||
case "decommission":
|
||||
// Decommission binds to the drive's STORAGE durable-id (the watchdog's key, e.g.
|
||||
// "uuid:<fs-uuid>"), NOT the device-level byid:/byuuid: scheme. The agent records this
|
||||
// id into the intent map, so it must match what the storage observer reports.
|
||||
if *durableID == "" {
|
||||
return fmt.Errorf("decommission needs -durable-id (the drive's storage durable-id, e.g. uuid:<fs-uuid>)")
|
||||
}
|
||||
pj, _ := json.Marshal(map[string]string{"durable_id": *durableID})
|
||||
params = string(pj)
|
||||
case "agent_update":
|
||||
// The agent downloads the binary for -agent-version and verifies it against -sha256.
|
||||
// The sha is the ONLY integrity root, so both are mandatory and the sha is strict-validated.
|
||||
if *agentVer == "" || *sha256Hex == "" {
|
||||
return fmt.Errorf("agent_update needs -agent-version and -sha256 (the pinned binary hash)")
|
||||
}
|
||||
if !isHex64(*sha256Hex) {
|
||||
return fmt.Errorf("agent_update -sha256 must be 64 lowercase hex chars (got %d)", len(*sha256Hex))
|
||||
}
|
||||
pj, _ := json.Marshal(map[string]string{"version": *agentVer, "sha256": *sha256Hex})
|
||||
params = string(pj)
|
||||
case "agent_config_update":
|
||||
// R-840: the box's root-owned files. The ROOT wrapper verifies this signature itself and refuses a bundle
|
||||
// whose sha256 is not exactly this one.
|
||||
if *agentVer == "" || *bundleSHA == "" {
|
||||
return fmt.Errorf("agent_config_update needs -agent-version and -bundle-sha256 (the pinned bundle hash)")
|
||||
}
|
||||
if !isHex64(*bundleSHA) {
|
||||
return fmt.Errorf("agent_config_update -bundle-sha256 must be 64 lowercase hex chars (got %d)", len(*bundleSHA))
|
||||
}
|
||||
pj, _ := json.Marshal(map[string]string{"agent_version": *agentVer, "bundle_sha256": *bundleSHA})
|
||||
params = string(pj)
|
||||
default:
|
||||
params = "{}"
|
||||
}
|
||||
}
|
||||
|
||||
n := *nonce
|
||||
if n == "" {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return fmt.Errorf("generating nonce: %w", err)
|
||||
}
|
||||
n = hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
issued := now
|
||||
if *ttl <= 0 || *ttl > authz.MaxWindow {
|
||||
fmt.Fprintf(os.Stderr, "felhom-opsign: -ttl %s refused: a box accepts a window of at most %s (R-1280)\n", *ttl, authz.MaxWindow)
|
||||
os.Exit(2)
|
||||
}
|
||||
expires := now.Add(*ttl)
|
||||
|
||||
// Canonical OpBlob bytes — the EXACT bytes the agent verifier authenticates over.
|
||||
blob, err := authz.CanonicalBlob(*op, *host, *guest, *keyID, n, params, issued, expires)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sigArmored, err := signWithSSHKeygen(blob, *keyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
env := map[string]string{
|
||||
"op_blob_b64": base64.StdEncoding.EncodeToString(blob),
|
||||
"sig_armored": sigArmored,
|
||||
}
|
||||
envJSON, _ := json.Marshal(env)
|
||||
|
||||
// Emit the envelope (stdout or file). Also print the human summary to stderr (never the key).
|
||||
fmt.Fprintf(os.Stderr, "signed: op=%s host=%s guest=%q key_id=%s nonce=%s expires=%s\n",
|
||||
*op, *host, *guest, *keyID, n, expires.Format(time.RFC3339))
|
||||
if *out != "" {
|
||||
if err := os.WriteFile(*out, envJSON, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "wrote envelope to", *out)
|
||||
} else {
|
||||
fmt.Println(string(envJSON))
|
||||
}
|
||||
|
||||
if *uploadURL != "" {
|
||||
if *hubKey == "" {
|
||||
return fmt.Errorf("--upload needs --hub-key (the operator's hub bearer key)")
|
||||
}
|
||||
if err := upload(*uploadURL, *hubKey, *host, envJSON); err != nil {
|
||||
return fmt.Errorf("upload to hub: %w", err)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr, "uploaded signed op to the hub jobs queue")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isHex64 reports whether s is exactly 64 lowercase hex chars (a sha256 hex digest).
|
||||
func isHex64(s string) bool {
|
||||
if len(s) != 64 {
|
||||
return false
|
||||
}
|
||||
for _, c := range s {
|
||||
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// signWithSSHKeygen signs `message` with `ssh-keygen -Y sign -n <namespace>`, the hardware-ready
|
||||
// path (sk-/YubiKey keys work unchanged). It writes the message to a temp file, runs ssh-keygen,
|
||||
// and reads the armored SSHSIG it produces. The namespace is the agent's FIXED domain separator.
|
||||
func signWithSSHKeygen(message []byte, keyFile string) (string, error) {
|
||||
dir, err := os.MkdirTemp("", "felhom-opsign-")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
msgPath := filepath.Join(dir, "op.blob")
|
||||
if err := os.WriteFile(msgPath, message, 0o600); err != nil {
|
||||
return "", err
|
||||
}
|
||||
cmd := exec.Command("ssh-keygen", "-Y", "sign", "-n", authz.Namespace, "-f", keyFile, msgPath)
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", fmt.Errorf("ssh-keygen -Y sign: %w: %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
sig, err := os.ReadFile(msgPath + ".sig")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading produced signature: %w", err)
|
||||
}
|
||||
return string(sig), nil
|
||||
}
|
||||
|
||||
// upload POSTs the signed-op envelope to the hub's jobs queue (POST /api/v1/admin/hosts/{id}/jobs).
|
||||
// The queued blob is base64(envelope JSON); the hub stores it opaquely (it cannot forge or open it).
|
||||
func upload(baseURL, hubKey, hostID string, envJSON []byte) error {
|
||||
body, _ := json.Marshal(map[string]string{
|
||||
"blob_b64": base64.StdEncoding.EncodeToString(envJSON),
|
||||
})
|
||||
// 10A's enqueue lives under /admin/hosts/{id}/jobs (operator/global key). The queued blob is
|
||||
// base64(envelope JSON); the hub stores it opaquely.
|
||||
url := strings.TrimRight(baseURL, "/") + "/api/v1/admin/hosts/" + hostID + "/jobs"
|
||||
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+hubKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
hc := &http.Client{Timeout: 15 * time.Second}
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(raw)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// TASK D1 Group D — agent_update opsign param validation. isHex64 is the sha gate the CLI applies
|
||||
// before it will build an agent_update envelope (the agent re-validates too, but a bad sha should
|
||||
// never even be signed).
|
||||
func TestIsHex64(t *testing.T) {
|
||||
good := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // 64 'a'
|
||||
if !isHex64(good) {
|
||||
t.Errorf("isHex64(%q) = false, want true", good)
|
||||
}
|
||||
for name, bad := range map[string]string{
|
||||
"too short": "abcdef",
|
||||
"too long": good + "a",
|
||||
"uppercase hex": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
"non-hex char": "g" + good[1:],
|
||||
"empty": "",
|
||||
} {
|
||||
if isHex64(bad) {
|
||||
t.Errorf("%s: isHex64(%q) = true, want false", name, bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"proxmox": {
|
||||
"endpoint": "https://127.0.0.1:8006",
|
||||
"node": "demo-felhom",
|
||||
"token": "felhom-agent@pve!agent=REPLACE_WITH_SECRET",
|
||||
"tls": {
|
||||
"ca_file": "",
|
||||
"fingerprint": "BA:7C:99:7D:45:D0:67:91:E2:F2:72:74:6E:D6:9F:83:51:D1:61:E5:C3:BD:F6:A0:B8:0B:E3:D8:DB:89:5B:CF",
|
||||
"insecure_skip_verify": false
|
||||
}
|
||||
},
|
||||
"privileged": {
|
||||
"mode": "sudo",
|
||||
"sudo_path": "sudo"
|
||||
},
|
||||
"authz": {
|
||||
"nonce_store_path": "/var/lib/felhom-agent/nonces.log",
|
||||
"signers": [
|
||||
{
|
||||
"key_id": "felhom-op-1",
|
||||
"role": "operational",
|
||||
"public_key": "ssh-ed25519 AAAA... felhom-op-1"
|
||||
},
|
||||
{
|
||||
"key_id": "felhom-recovery-1",
|
||||
"role": "recovery",
|
||||
"public_key": "ssh-ed25519 AAAA... felhom-recovery-1"
|
||||
}
|
||||
]
|
||||
},
|
||||
"hub": {
|
||||
"url": "https://hub.felhom.eu",
|
||||
"host_id": "demo-host-01",
|
||||
"api_key": "REPLACE_WITH_PER_HOST_HUB_KEY",
|
||||
"poll_seconds": 900,
|
||||
"timeout_seconds": 30,
|
||||
"ca_file": ""
|
||||
},
|
||||
"backup": {
|
||||
"local_backup_target": "local",
|
||||
"restore_storage": "local-lvm",
|
||||
"restore_test_cadence_seconds": 0,
|
||||
"scratch_vmid_min": 990000,
|
||||
"scratch_vmid_max": 990009,
|
||||
"pbs_verify_cadence_seconds": 0,
|
||||
"pbs_secret_dir": "/etc/pve/priv/storage",
|
||||
"backup_cadence_seconds": 0
|
||||
},
|
||||
"local_api": {
|
||||
"enable": true,
|
||||
"listen_addr": "169.254.253.1:8443",
|
||||
"cert_file": "/var/lib/felhom-agent/local-api.crt",
|
||||
"key_file": "/var/lib/felhom-agent/local-api.key",
|
||||
"token_store": "/var/lib/felhom-agent/local-tokens.log",
|
||||
"island_bridge": "vmbr9",
|
||||
"island_guest_addr": "169.254.253.2/30"
|
||||
},
|
||||
"lan_resolver": {
|
||||
"enable": true,
|
||||
"host_ip": "192.168.0.162"
|
||||
},
|
||||
"log_level": "info"
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-golden.sh — build the Felhom golden base LXC archive (slice 7).
|
||||
#
|
||||
# Produces a minimal Debian + Docker, unprivileged, nesting=1,keyctl=1, overlayfs LXC, baked
|
||||
# identity-clean, and archives it for a token-restore by the bring-up reconcile job
|
||||
# (internal/reconcile/bringup.go). Run as root@pam on a Proxmox host (the keyctl=1 feature flag
|
||||
# is root-only — phase3 #1; this is the ONE root step, off the per-customer path).
|
||||
#
|
||||
# Grounded by documentation/tests/slice7-bringup-spike-findings.md (commit 3342993):
|
||||
# - F3: removing the SSH host keys does NOT auto-regenerate them on Debian (pct restore runs no
|
||||
# keygen hook), so a baked, Condition-gated first-boot unit regenerates them — keeping the
|
||||
# agent's front half host-side-only. The gate (ConditionPathExists=!…) makes it fire on a
|
||||
# provision (golden, keys absent) and no-op on a DR restore (customer backup, keys present),
|
||||
# symmetric with machine-id.
|
||||
# - machine-id: truncated; systemd regenerates it on first boot for free (no unit needed).
|
||||
#
|
||||
# Slice 8A — the golden now also BAKES the in-guest controller (decision: image baked at golden
|
||||
# build on the trusted host, so NO registry credential ever enters a customer guest at deploy) and
|
||||
# a controller-bootstrap unit that, on boot, deploys the baked image from the agent-populated
|
||||
# config mount (/etc/felhom-bootstrap/bootstrap.json) — no docker login/pull at deploy. Refreshing
|
||||
# the golden bumps the controller baseline; controller self-update covers in-between drift.
|
||||
#
|
||||
# Usage: build-golden.sh [VMID] [TEMPLATE_VOLID] [ROOTFS_STORAGE] [ARCHIVE_STORAGE] [BRIDGE] CONTROLLER_IMAGE
|
||||
# CONTROLLER_IMAGE is REQUIRED (no default) — pass the released controller tag explicitly,
|
||||
# e.g. gitea.dooplex.hu/admin/felhom-controller:0.98.3.
|
||||
# Build-time registry login for the controller pull (used ONCE inside the build guest, then logged
|
||||
# out — never baked): set REGISTRY_USER + REGISTRY_TOKEN in the environment.
|
||||
#
|
||||
# OS / DATA SPLIT, and since v3.0.0 ONE DATA VOLUME (R-165, decision D-a + variant V-c).
|
||||
#
|
||||
# The golden is built with a SMALL OS rootfs and a SINGLE data volume (mp0, backup=1) mounted at a
|
||||
# NEUTRAL path, /var/lib/felhom. Both consumer paths are binds of subdirectories of it:
|
||||
#
|
||||
# /var/lib/felhom/docker --bind--> /var/lib/docker (Docker's data-root)
|
||||
# /var/lib/felhom/sys_drive --bind--> /mnt/sys_drive (the controller's system_data_path)
|
||||
#
|
||||
# WHAT THIS REPLACED, AND WHY. Until v2.1.0 these were TWO volumes (mp0 16 G at /var/lib/docker,
|
||||
# mp1 8 G at /mnt/sys_drive, grown separately at provision). The second one was a fixed ceiling: an
|
||||
# app whose local recovery unit outgrew it stopped being backed up even with free space next door.
|
||||
# D-a removed the wall rather than moving it — one volume, one free-space figure, no ceiling.
|
||||
#
|
||||
# WHY A NEUTRAL MOUNT AND NOT SIMPLY NESTING ONE PATH INSIDE THE OTHER. Both simpler shapes were
|
||||
# built and measured (SPIKE-r165-phase0-2026-08-03.md); both boot and reboot cleanly, and each breaks
|
||||
# a different documented guarantee:
|
||||
# * volume at /var/lib/docker -> customer backups live INSIDE Docker's data-root, so `du` there
|
||||
# stops meaning what it says and the ordinary "clear /var/lib/docker to fix Docker" reflex
|
||||
# destroys every local recovery unit on the box;
|
||||
# * volume at /mnt/sys_drive -> Docker's ENTIRE data-root lands under /mnt, which the controller
|
||||
# container mounts wholesale (`-v /mnt:/mnt:rslave`). Measured: the container then sees
|
||||
# /mnt/sys_drive/docker. The bootstrap's own claim that /mnt "holds only Felhom's
|
||||
# felhom-data-namespace mounts" would become false.
|
||||
# The neutral mount breaks neither, for one extra path and one extra fstab line.
|
||||
#
|
||||
# The split from the OS rootfs is still for RESILIENCE: an isolated rootfs stays bootable +
|
||||
# agent-recoverable if the data volume fills (the controller's prevention layer, and since
|
||||
# controller v0.192.0 the capture floor, keep it from filling). Size is env-overridable
|
||||
# (OS_SIZE_GB / GOLDEN_VOLUME_GB); provision GROWS the one volume (bringup.go DataVolGrowGB).
|
||||
# backup=1 is MANDATORY: without it vzdump EXCLUDES the volume (extra LXC mountpoints default
|
||||
# backup=0 — storage-split B3), so the archive would carry no images AND no user data.
|
||||
set -euo pipefail
|
||||
|
||||
# Script provenance — logged into every bake transcript next to the baked controller tag, so an
|
||||
# archive can always be traced to the script that produced it. Bump on any behavior change.
|
||||
GOLDEN_SCRIPT_VERSION="3.2.0"
|
||||
|
||||
VMID="${1:-9100}"
|
||||
TEMPLATE="${2:-local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst}"
|
||||
ROOTFS_STORAGE="${3:-local-lvm}"
|
||||
ARCHIVE_STORAGE="${4:-local}"
|
||||
BRIDGE="${5:-vmbr0}"
|
||||
# CONTROLLER_IMAGE is MANDATORY — no default. The hand-bumped default rotted twice (0.43.0 →
|
||||
# 0.85.1 → stale again): each time, a fresh provision would have booted a pre-floor controller
|
||||
# needing a manual install-day update (drill finding B5). A required argument cannot rot, and
|
||||
# auto-resolving "latest" could bake a tag the hub manifest never vouched — so the caller states
|
||||
# the released controller tag explicitly at every rebuild.
|
||||
CONTROLLER_IMAGE="${6:-}"
|
||||
if [ -z "$CONTROLLER_IMAGE" ]; then
|
||||
echo "[golden] FATAL: CONTROLLER_IMAGE (argument 6) is required — pass the released controller tag explicitly." >&2
|
||||
echo "Usage: build-golden.sh [VMID] [TEMPLATE_VOLID] [ROOTFS_STORAGE] [ARCHIVE_STORAGE] [BRIDGE] CONTROLLER_IMAGE" >&2
|
||||
echo " e.g.: build-golden.sh 9100 local:vztmpl/debian-13-standard_13.1-2_amd64.tar.zst local-lvm local vmbr0 gitea.dooplex.hu/admin/felhom-controller:0.98.3" >&2
|
||||
exit 1
|
||||
fi
|
||||
REGISTRY_HOST="${CONTROLLER_IMAGE%%/*}"
|
||||
# OS rootfs size (GiB) and the golden's SINGLE data volume size (GiB).
|
||||
#
|
||||
# ONE VOLUME MEANS ONE NUMBER (v3.0.0). The retired GOLDEN_SYSDATA_GB has no successor: there is
|
||||
# nothing left to size separately. Keep GOLDEN_VOLUME_GB just large enough for the baked images plus
|
||||
# headroom for the controller's felhom-data skeleton; provision grows the one volume to the
|
||||
# per-customer target (bringup.go DataVolGrowGB).
|
||||
OS_SIZE_GB="${OS_SIZE_GB:-32}"
|
||||
# 24 = the retired pair's 16 (docker) + 8 (user-data), so a golden archive carries the same content it
|
||||
# did before the merge. It is deliberately NOT a per-customer size: provision grows it.
|
||||
GOLDEN_VOLUME_GB="${GOLDEN_VOLUME_GB:-24}"
|
||||
# The neutral mount path of the single volume. Both consumer paths are binds of subdirectories of it.
|
||||
GOLDEN_VOLUME_MP="/var/lib/felhom"
|
||||
|
||||
echo "[golden] build-golden.sh v${GOLDEN_SCRIPT_VERSION} — baking controller ${CONTROLLER_IMAGE}"
|
||||
echo "[golden] creating build LXC $VMID (nesting=1,keyctl=1, unprivileged; rootfs ${OS_SIZE_GB}G + ONE data volume ${GOLDEN_VOLUME_GB}G @ ${GOLDEN_VOLUME_MP}, backup=1) …"
|
||||
# ONE mpN slot. There is deliberately no mp1: that slot held the retired user-data volume, and the
|
||||
# whole point of R-165 is that it stops existing rather than being made bigger.
|
||||
pct create "$VMID" "$TEMPLATE" \
|
||||
--hostname felhom-golden --unprivileged 1 \
|
||||
--features nesting=1,keyctl=1 \
|
||||
--rootfs "${ROOTFS_STORAGE}:${OS_SIZE_GB}" --cores 2 --memory 2048 \
|
||||
--mp0 "${ROOTFS_STORAGE}:${GOLDEN_VOLUME_GB},mp=${GOLDEN_VOLUME_MP},backup=1" \
|
||||
--net0 "name=eth0,bridge=${BRIDGE},ip=dhcp" --onboot 0
|
||||
|
||||
echo "[golden] starting + installing Docker (official repo, trixie channel) …"
|
||||
pct start "$VMID"
|
||||
# wait for DHCP/DNS
|
||||
for i in $(seq 1 30); do
|
||||
if pct exec "$VMID" -- getent hosts download.docker.com >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
# v3.1.0 (`11` §5.8): GOLDEN_DOCKER_PKGS pins the APPROVED Docker engine set (the hub's newest Docker release, all six
|
||||
# "name=version"); without it the newest stable set is installed and the bake log says so.
|
||||
GOLDEN_DOCKER_PKGS="${GOLDEN_DOCKER_PKGS:-}"
|
||||
if [[ -n "$GOLDEN_DOCKER_PKGS" ]]; then
|
||||
echo "[golden] Docker engine set PINNED to the approved release: $GOLDEN_DOCKER_PKGS"
|
||||
else
|
||||
echo "[golden] WARNING: GOLDEN_DOCKER_PKGS not set — installing the newest stable Docker set, not an approved one"
|
||||
fi
|
||||
pct exec "$VMID" -- env GOLDEN_DOCKER_PKGS="$GOLDEN_DOCKER_PKGS" bash -c '
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq ca-certificates curl >/dev/null
|
||||
install -m0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
|
||||
echo "deb [signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian trixie stable" \
|
||||
> /etc/apt/sources.list.d/docker.list
|
||||
apt-get update -qq
|
||||
if [ -n "$GOLDEN_DOCKER_PKGS" ]; then
|
||||
apt-get install -y -qq $GOLDEN_DOCKER_PKGS >/dev/null
|
||||
else
|
||||
apt-get install -y -qq docker-ce docker-ce-cli containerd.io >/dev/null
|
||||
fi
|
||||
dpkg-query -W containerd.io docker-buildx-plugin docker-ce docker-ce-cli docker-ce-rootless-extras docker-compose-plugin 2>/dev/null | sed "s/^/ installed: /"
|
||||
'
|
||||
# v3.2.0 (Part F of the R-840 brief, `11` §5.3): GOLDEN_GUEST_PKGS = the newest APPROVED guest release, as
|
||||
# "name=version …" (the hub's os_releases row for layer guest, IN FORCE — never a cancelled test approval). The bake
|
||||
# brings every package the template HAS to exactly that version, under felhom-os-apply's rules: never a package the
|
||||
# template lacks (--only-upgrade), never newer than approved, never a removal or a new package (a simulation is checked
|
||||
# first and the bake FAILS on either). Empty = no approved guest release in force: the template's versions stay, and
|
||||
# the box's first night installs whatever release is approved then. Either way the bake PRINTS the first-night count:
|
||||
# how many installed packages are older than the approved version (target 0).
|
||||
GOLDEN_GUEST_PKGS="${GOLDEN_GUEST_PKGS:-}"
|
||||
pct exec "$VMID" -- env GOLDEN_GUEST_PKGS="$GOLDEN_GUEST_PKGS" bash -c '
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
if [ -z "$GOLDEN_GUEST_PKGS" ]; then
|
||||
echo "[golden] no approved guest release given - the template versions stay; first-night count vs an approved release: n/a"
|
||||
echo "[golden] pending Debian upgrades in the baked guest (what a FUTURE approval may bring): $(apt list --upgradable 2>/dev/null | grep -c /)"
|
||||
exit 0
|
||||
fi
|
||||
want=""
|
||||
for nv in $GOLDEN_GUEST_PKGS; do
|
||||
n=${nv%%=*}; v=${nv#*=}
|
||||
cur=$(dpkg-query -W -f="\${Version}" "$n" 2>/dev/null) || continue # not in the template: never added
|
||||
dpkg --compare-versions "$cur" lt "$v" && want="$want $n=$v"
|
||||
done
|
||||
if [ -n "$want" ]; then
|
||||
sim=$(apt-get -s install --only-upgrade -o Dpkg::Options::=--force-confold $want)
|
||||
if echo "$sim" | grep -q "^Remv "; then echo "[golden] FATAL: the approved guest set would REMOVE a package"; echo "$sim" | grep "^Remv "; exit 1; fi
|
||||
for p in $(echo "$sim" | awk "/^Inst /{print \$2}"); do
|
||||
dpkg-query -W "$p" >/dev/null 2>&1 || { echo "[golden] FATAL: the approved guest set would ADD $p - not in the template"; exit 1; }
|
||||
done
|
||||
apt-get install -y -qq --only-upgrade -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-confdef $want >/dev/null
|
||||
echo "[golden] approved guest release installed: $(echo $want | wc -w) package(s) brought to the approved version"
|
||||
else
|
||||
echo "[golden] approved guest release: the template already runs every approved version"
|
||||
fi
|
||||
left=0
|
||||
for nv in $GOLDEN_GUEST_PKGS; do
|
||||
n=${nv%%=*}; v=${nv#*=}
|
||||
cur=$(dpkg-query -W -f="\${Version}" "$n" 2>/dev/null) || continue
|
||||
dpkg --compare-versions "$cur" lt "$v" && { left=$((left+1)); echo " still older: $n $cur < $v"; }
|
||||
done
|
||||
echo "[golden] first-night count vs the approved guest release: $left (target 0)"
|
||||
[ "$left" -eq 0 ] || { echo "[golden] FATAL: $left package(s) stayed older than the approved release"; exit 1; }
|
||||
'
|
||||
echo "[golden] baking daemon.json: classic overlay2 driver (containerd-snapshotter OFF) + log rotation …"
|
||||
# containerd-snapshotter (Docker 28+/29 default) keeps the IMAGE content store under
|
||||
# /var/lib/containerd — which is NOT /var/lib/docker, so it would stay on the OS rootfs and the split
|
||||
# would only move named volumes, defeating the whole point (validated: images landed on the rootfs).
|
||||
# The classic overlay2 driver stores EVERYTHING (images + overlay + volumes) under data-root
|
||||
# (/var/lib/docker) = the data volume, which is exactly what "one data-root = one partition for all
|
||||
# images + overlay" requires. It also makes the controller's statfs("/") (its overlay root) report the
|
||||
# DATA volume, which the prevention layer depends on — MEASURED to still hold under the v3.0.0 merged
|
||||
# layout (a container's `df /` reports the single volume, phase-0 spike). Since v3.0.0 /var/lib/docker
|
||||
# is a BIND of <volume>/docker rather than the mp0 mount itself, wired immediately below; data-root
|
||||
# still needs no override because the path is unchanged. Log caps kill the most common runaway.
|
||||
# v3.1.0: "live-restore": true (`09` decision 87) — a Docker engine update then restarts no app (`11` C5). A box made
|
||||
# from this golden never needs the agent's one-time live-restore-on step. NEVER removed by a plain restart (R-835).
|
||||
pct exec "$VMID" -- bash -c 'mkdir -p /etc/docker; cat > /etc/docker/daemon.json <<JSON
|
||||
{
|
||||
"features": { "containerd-snapshotter": false },
|
||||
"live-restore": true,
|
||||
"log-driver": "json-file",
|
||||
"log-opts": { "max-size": "10m", "max-file": "3" }
|
||||
}
|
||||
JSON'
|
||||
echo "[golden] wiring the single data volume (R-165 variant V-c): ${GOLDEN_VOLUME_MP}/{docker,sys_drive} -> binds …"
|
||||
# docker-ce has already populated /var/lib/docker ON THE ROOTFS by now (it auto-starts on install), so
|
||||
# the content is MOVED onto the volume before the bind is laid over the top. Doing it the other way
|
||||
# round would hide those files under the bind and silently ship a golden whose baked images are on the
|
||||
# rootfs — the exact failure class the assertions below exist to catch.
|
||||
#
|
||||
# /etc/fstab, not a hand-run mount: systemd's fstab generator orders both binds under local-fs.target,
|
||||
# which precedes basic.target and therefore docker.service. MEASURED across 3 reboots per variant in
|
||||
# the phase-0 spike — the ordering worry that motivated the probe did not materialise.
|
||||
pct exec "$VMID" -- bash -c "
|
||||
set -e
|
||||
systemctl stop docker docker.socket containerd 2>/dev/null || true
|
||||
mkdir -p '${GOLDEN_VOLUME_MP}/docker' '${GOLDEN_VOLUME_MP}/sys_drive'
|
||||
if [ -d /var/lib/docker ] && [ -n \"\$(ls -A /var/lib/docker 2>/dev/null)\" ]; then
|
||||
cp -a /var/lib/docker/. '${GOLDEN_VOLUME_MP}/docker'/
|
||||
rm -rf /var/lib/docker/*
|
||||
fi
|
||||
mkdir -p /var/lib/docker /mnt/sys_drive
|
||||
printf '%s /var/lib/docker none bind 0 0\n' '${GOLDEN_VOLUME_MP}/docker' >> /etc/fstab
|
||||
printf '%s /mnt/sys_drive none bind 0 0\n' '${GOLDEN_VOLUME_MP}/sys_drive' >> /etc/fstab
|
||||
systemctl daemon-reload
|
||||
mount /var/lib/docker
|
||||
mount /mnt/sys_drive
|
||||
systemctl start containerd
|
||||
"
|
||||
|
||||
echo "[golden] verifying Docker works in the build guest (storage driver should be overlay2 on the ext4 data volume) …"
|
||||
# RESTART (not start): docker-ce auto-starts on install with the DEFAULT config, so it is already
|
||||
# running by now; only a restart picks up the daemon.json just written (overlay2 + log caps).
|
||||
pct exec "$VMID" -- bash -c 'systemctl restart docker; sleep 3; docker run --rm hello-world >/dev/null && echo " docker OK ($(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"); data-root $(docker info 2>/dev/null | sed -n "s/.*Docker Root Dir: //p"))"'
|
||||
# Guard: the image store MUST be on the data volume now. /var/lib/containerd holding the images would
|
||||
# mean containerd-snapshotter is still on (the split would leave images on the rootfs).
|
||||
pct exec "$VMID" -- bash -c 'drv=$(docker info 2>/dev/null | sed -n "s/.*Storage Driver: //p"); [ "$drv" = "overlay2" ] || { echo "[golden] FATAL: storage driver is $drv, expected overlay2 — images would not land on the data volume"; exit 1; }'
|
||||
# v3.1.0 ASSERTION: live-restore is ON in the running daemon (decision 87), or the bake fails closed.
|
||||
pct exec "$VMID" -- bash -c 'lr=$(docker info --format "{{.LiveRestoreEnabled}}" 2>/dev/null); [ "$lr" = "true" ] && echo " live-restore: on" || { echo "[golden] FATAL: live-restore is $lr, expected true (decision 87)"; exit 1; }'
|
||||
# ASSERTION 1 (RETARGETED v3.0.0, not removed). /var/lib/docker must be a real mount — now the V-c
|
||||
# bind of <volume>/docker rather than the mp0 mount itself. Still fails closed on the same failure:
|
||||
# if the bind did not take, Docker's data-root silently sits on the OS rootfs and the golden ships
|
||||
# its baked images there.
|
||||
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /var/lib/docker | grep -q . && echo " /var/lib/docker is a real mount: $(findmnt -no SOURCE,FSTYPE /var/lib/docker | head -1)" || { echo "[golden] FATAL: /var/lib/docker is NOT a mount — the V-c docker bind did not take, so the baked images would land on the OS rootfs"; exit 1; }'
|
||||
# ASSERTION 2 (RETARGETED v3.0.0). /mnt/sys_drive must be a real mount — now the V-c bind of
|
||||
# <volume>/sys_drive. Otherwise the controller's system_data_path lands on the OS drive and it warns
|
||||
# (clearing that warning is the whole point of the volume).
|
||||
pct exec "$VMID" -- bash -c 'findmnt -no SOURCE,FSTYPE /mnt/sys_drive | grep -q . && echo " /mnt/sys_drive is a real mount: $(findmnt -no SOURCE,FSTYPE /mnt/sys_drive | head -1)" || { echo "[golden] FATAL: /mnt/sys_drive is NOT a mount — the V-c sys_drive bind did not take, so the controller system_data_path would be the OS rootfs"; exit 1; }'
|
||||
# ASSERTION 2b (NEW v3.0.0 — the invariant the merge is FOR). Both paths must be backed by the SAME
|
||||
# device, i.e. ONE filesystem with ONE free-space figure. Two devices here is the S2 shape the R-165
|
||||
# spike ranked strictly WORSE than the split it replaced: every assertion satisfied, the ceiling still
|
||||
# there, and a shared pool neither `df` can see coming.
|
||||
pct exec "$VMID" -- bash -c 'n=$(df --output=source /var/lib/docker /mnt/sys_drive | tail -n +2 | sort -u | wc -l); [ "$n" = "1" ] && echo " both paths are ONE filesystem: $(df --output=source,avail /var/lib/docker | tail -1)" || { echo "[golden] FATAL: /var/lib/docker and /mnt/sys_drive are on $n DIFFERENT filesystems — that is the S2 shape (two ceilings), not the R-165 merge"; exit 1; }'
|
||||
|
||||
echo "[golden] baking the in-guest controller image $CONTROLLER_IMAGE (no registry cred at deploy) …"
|
||||
# docker login is used ONCE here on the trusted build host, then logged out before archiving so
|
||||
# the credential is NEVER baked into the golden. The IMAGE is what gets baked (in Docker storage).
|
||||
if [ -n "${REGISTRY_USER:-}" ] && [ -n "${REGISTRY_TOKEN:-}" ]; then
|
||||
pct exec "$VMID" -- bash -c "systemctl start docker; sleep 1; echo '$REGISTRY_TOKEN' | docker login '$REGISTRY_HOST' -u '$REGISTRY_USER' --password-stdin >/dev/null"
|
||||
fi
|
||||
pct exec "$VMID" -- bash -c "docker pull '$CONTROLLER_IMAGE'"
|
||||
# Record the baked image ref for the bootstrap unit (so the unit needs no login/pull).
|
||||
pct exec "$VMID" -- bash -c "printf '%s\n' '$CONTROLLER_IMAGE' > /etc/felhom-controller-image"
|
||||
|
||||
# Bake EVERY controller-managed infra image so enabling an infra stack on a fresh box is near-instant
|
||||
# instead of a multi-minute silent registry pull.
|
||||
#
|
||||
# THE LIST COMES FROM THE CONTROLLER WE JUST PULLED, not from a copy here. This script used to carry
|
||||
# a hand-maintained array of three tags with a comment telling the reader to keep it in sync with the
|
||||
# controller's internal/infra constants. It drifted the moment a fourth stack was added: felhom-samba
|
||||
# was never added here, so the golden baked 3 of 4 and turning on Megosztás pulled from the registry
|
||||
# with zero UI feedback — observed live, twice. Asking the binary (`--print-infra-images`, backed by
|
||||
# infra.Images()) makes golden-vs-controller drift structurally impossible: the golden bakes exactly
|
||||
# what THIS controller version will request.
|
||||
echo "[golden] asking the controller which infra images it manages …"
|
||||
INFRA_LIST="$(pct exec "$VMID" -- bash -c "docker run --rm --entrypoint /usr/local/bin/felhom-controller '$CONTROLLER_IMAGE' --print-infra-images 2>/dev/null" || true)"
|
||||
mapfile -t INFRA_IMAGES < <(printf '%s\n' "$INFRA_LIST" | grep -E '^[a-z0-9._/-]+:[A-Za-z0-9._-]+$' || true)
|
||||
if [ "${#INFRA_IMAGES[@]}" -eq 0 ]; then
|
||||
# Controllers older than v0.147.0 have no --print-infra-images. Fall back to the historical list so
|
||||
# an old-controller bake still works — but say so loudly, because the fallback is exactly the
|
||||
# drift-prone thing this change removed, and it CANNOT know about stacks added after it was written.
|
||||
echo "[golden] WARN: '$CONTROLLER_IMAGE' does not support --print-infra-images (pre-0.147.0)."
|
||||
echo "[golden] WARN: falling back to the historical 3-image list. felhom-samba will NOT be baked,"
|
||||
echo "[golden] WARN: so enabling Megosztás on this golden will pull at runtime. Bake >=0.147.0."
|
||||
INFRA_IMAGES=(
|
||||
"traefik:v3.6.7"
|
||||
"cloudflare/cloudflared:2026.6.0"
|
||||
"gtstef/filebrowser:1.3.3-stable"
|
||||
)
|
||||
fi
|
||||
echo "[golden] baking infra images (${#INFRA_IMAGES[@]}): ${INFRA_IMAGES[*]} …"
|
||||
for img in "${INFRA_IMAGES[@]}"; do
|
||||
# Hard gate: fail loudly BEFORE pulling if a pinned tag doesn't resolve (a bad pin otherwise fails
|
||||
# mid-bake with a confusing error).
|
||||
pct exec "$VMID" -- bash -c "docker manifest inspect '$img' >/dev/null 2>&1" \
|
||||
|| { echo "[golden] FATAL: pinned infra image does not resolve: $img"; exit 1; }
|
||||
pct exec "$VMID" -- bash -c "docker pull '$img'"
|
||||
done
|
||||
|
||||
# Log out LAST — not right after the controller pull. felhom-samba lives on the same private registry
|
||||
# as the controller, so the infra loop above needs the credential; logging out first made that pull
|
||||
# 401. The credential is still never archived: the logout + config.json removal happen before the
|
||||
# guest is stopped and templated.
|
||||
pct exec "$VMID" -- bash -c "docker logout '$REGISTRY_HOST' >/dev/null 2>&1 || true; rm -f /root/.docker/config.json"
|
||||
pct exec "$VMID" -- bash -c '[ ! -s /root/.docker/config.json ]' \
|
||||
|| { echo "[golden] FATAL: registry credential still present in the guest — refusing to archive"; exit 1; }
|
||||
|
||||
echo "[golden] baking the controller-bootstrap unit (deploys the BAKED controller from the config mount) …"
|
||||
pct push "$VMID" /dev/stdin /usr/local/sbin/felhom-controller-bootstrap.sh --perms 700 <<'BOOTSH'
|
||||
#!/bin/bash
|
||||
# felhom controller-bootstrap (slice 8A): the host agent's back-half populated the read-only
|
||||
# config mount /etc/felhom-bootstrap; this golden-baked oneshot deploys the BAKED controller image
|
||||
# with that config. NO docker login / NO docker pull — the image is already in this golden's Docker
|
||||
# storage (and self-update handles version drift). Host-side only; the agent never enters the guest.
|
||||
set -euo pipefail
|
||||
CFG=/etc/felhom-bootstrap/bootstrap.json
|
||||
[ -r "$CFG" ] || { echo "[ctrl-bootstrap] no $CFG — not provisioned, nothing to do"; exit 0; }
|
||||
IMAGE=$(cat /etc/felhom-controller-image 2>/dev/null || true)
|
||||
[ -n "$IMAGE" ] || { echo "[ctrl-bootstrap] FATAL: /etc/felhom-controller-image missing"; exit 1; }
|
||||
# Per-guest container hostname (slice base-infra/3A): derive from the bootstrap's customer.id so the
|
||||
# controller's os.Hostname() (its hub-reported hostname) is the customer id, not the Docker container
|
||||
# ID. Portable, dependency-free parse (NO jq in the golden) — bootstrap.json has exactly one "id" key
|
||||
# (customer.id). Falls back to no --hostname if the parse yields nothing (fail-safe).
|
||||
CUSTOMER_ID=$(sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$CFG" | head -1)
|
||||
# SECURITY: $CUSTOMER_ID is interpolated into `docker run` — reject anything that isn't a DNS-safe
|
||||
# label so a malformed/hostile customer.id can't smuggle extra docker flags (e.g. --privileged, -v).
|
||||
# Then pass it via a quoted array (never word-split). Empty/invalid → no --hostname (fail-safe).
|
||||
case "$CUSTOMER_ID" in
|
||||
""|*[!a-zA-Z0-9._-]*|-*) CUSTOMER_ID="" ;;
|
||||
esac
|
||||
HOSTNAME_ARGS=()
|
||||
[ -n "$CUSTOMER_ID" ] && HOSTNAME_ARGS=(--hostname "$CUSTOMER_ID")
|
||||
echo "[ctrl-bootstrap] deploying $IMAGE from $CFG (hostname=${CUSTOMER_ID:-<unset>})"
|
||||
docker rm -f felhom-controller >/dev/null 2>&1 || true
|
||||
# Section-G fix (base-infra slice): the controller writes app/infra compose stacks under
|
||||
# /opt/docker/stacks INSIDE the container, but `docker compose up` is executed by the GUEST daemon
|
||||
# (shared socket), which resolves every relative bind source on the GUEST filesystem. Without a
|
||||
# SAME-PATH host bind for /opt/docker/stacks, the daemon can't see those bind sources and silently
|
||||
# creates empty dirs → every bind-mounted stack (base infra AND customer apps) breaks. A named volume
|
||||
# would NOT fix this (it resolves to /var/lib/docker/volumes/...). Pre-create the dir, then same-path bind.
|
||||
mkdir -p /opt/docker/stacks
|
||||
# slice 10 P2: the controller must SEE enrolled external data drives, which the agent binds into the
|
||||
# guest at /mnt/<name>. Make /mnt a SHARED mount so those binds (and later host-remount self-heal)
|
||||
# propagate into the controller container, then bind /mnt :rslave (host->container propagation ONLY —
|
||||
# the container can't mutate the guest mount tree). This is the ONE bind the 8C de-privileging left
|
||||
# out; scoped to /mnt, which (Model A) holds only Felhom's felhom-data-namespace mounts, never the
|
||||
# customer's other on-drive data. rbind preserves existing submounts; re-running re-shares (safe).
|
||||
mkdir -p /mnt
|
||||
mountpoint -q /mnt || mount --rbind /mnt /mnt
|
||||
mount --make-rshared /mnt
|
||||
# Otherwise still DE-PRIVILEGED: disk EXECUTION (scan/format/mount) stays the agent's — NO --privileged,
|
||||
# no /dev, no /etc/fstab. Bootstrap config (ro), data volume, stacks dir (same-path), the /mnt :rslave
|
||||
# view, and the docker socket. The controller reaches the agent's local API for disk management.
|
||||
# R-523: `always`, not `unless-stopped`. It covers ONE extra case only — a Docker daemon restart after
|
||||
# the container was stopped by hand. Neither policy restarts a container that `docker kill`/`docker
|
||||
# stop` ended (measured 2026-09-15, Docker 29.8.0, evidence-p1fixes-2026-09-15/A1); the host agent's
|
||||
# controller supervisor (felhom-agent v0.131.0, internal/localapi/controllersupervisor.go) covers that.
|
||||
docker run -d --name felhom-controller --restart always "${HOSTNAME_ARGS[@]}" \
|
||||
-e FELHOM_BOOTSTRAP_PATH=/etc/felhom-bootstrap/bootstrap.json \
|
||||
-v /etc/felhom-bootstrap:/etc/felhom-bootstrap:ro \
|
||||
-v felhom-controller-data:/opt/docker/felhom-controller \
|
||||
-v /opt/docker/stacks:/opt/docker/stacks \
|
||||
-v /mnt:/mnt:rslave \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
"$IMAGE"
|
||||
echo "[ctrl-bootstrap] controller started"
|
||||
BOOTSH
|
||||
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-controller-bootstrap.service <<UNIT
|
||||
[Unit]
|
||||
Description=Felhom controller bootstrap (deploy the baked controller from the agent-populated config mount)
|
||||
After=docker.service network-online.target
|
||||
Wants=docker.service network-online.target
|
||||
ConditionPathExists=/etc/felhom-bootstrap/bootstrap.json
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/sbin/felhom-controller-bootstrap.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
systemctl enable felhom-controller-bootstrap.service'
|
||||
|
||||
# B1 (DRILL-day0-cleanroom-2026-07-03 R6/B1): the service's ConditionPathExists is evaluated only
|
||||
# when the service is STARTED — i.e. at boot via multi-user.target — but the agent back-half
|
||||
# hot-plugs the bootstrap mount into the ALREADY-RUNNING guest, so on a provision the boot-time
|
||||
# start races the mount and loses on slow hardware. This path unit watches for bootstrap.json and
|
||||
# starts the service when it APPEARS — covering the provision hot-plug without a reboot. The boot
|
||||
# case is still served by the enabled service itself; RemainAfterExit=yes on the service means the
|
||||
# path unit cannot re-trigger it in a loop once it has run.
|
||||
echo "[golden] baking the controller-bootstrap PATH unit (starts the service on bootstrap-mount hot-plug — B1) …"
|
||||
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-controller-bootstrap.path <<UNIT
|
||||
[Unit]
|
||||
Description=Watch for the agent-populated bootstrap config; start the controller bootstrap when it appears
|
||||
|
||||
[Path]
|
||||
PathExists=/etc/felhom-bootstrap/bootstrap.json
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
systemctl enable felhom-controller-bootstrap.path'
|
||||
|
||||
echo "[golden] baking the first-boot SSH host-key regeneration unit (F3) …"
|
||||
pct exec "$VMID" -- bash -c 'cat > /etc/systemd/system/felhom-regen-hostkeys.service <<UNIT
|
||||
[Unit]
|
||||
Description=Regenerate SSH host keys on first boot if absent
|
||||
ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key
|
||||
DefaultDependencies=no
|
||||
After=local-fs.target
|
||||
Before=ssh.service sshd.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/bin/ssh-keygen -A
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
systemctl enable felhom-regen-hostkeys.service'
|
||||
|
||||
echo "[golden] identity-clean + minimize …"
|
||||
pct exec "$VMID" -- bash -c '
|
||||
systemctl stop docker containerd 2>/dev/null || true
|
||||
apt-get clean; rm -rf /var/lib/apt/lists/*
|
||||
rm -f /etc/ssh/ssh_host_* # regenerated on first boot by the baked unit (F3)
|
||||
truncate -s 0 /etc/machine-id # systemd regenerates on first boot (free)
|
||||
rm -f /var/lib/dbus/machine-id; ln -sf /etc/machine-id /var/lib/dbus/machine-id
|
||||
rm -rf /var/log/*; : > /root/.bash_history
|
||||
rm -f /etc/hostname # set per-guest at provision (host-side token config)
|
||||
'
|
||||
|
||||
echo "[golden] stop + archive …"
|
||||
pct stop "$VMID"
|
||||
# --mode stop with mp0 backup=1 → the SINGLE data volume (baked images AND the user-data area) is
|
||||
# INCLUDED. The log MUST show "including mount point mp0" and must NOT show it being excluded — an
|
||||
# exclusion means the backup flag was lost and the archive carries neither (storage-split B3 trap).
|
||||
# Since v3.0.0 there is no mp1; the guard that covered it is retargeted below rather than deleted,
|
||||
# because a guard whose pattern can no longer match is a guard that has silently stopped guarding.
|
||||
vzdump "$VMID" --storage "$ARCHIVE_STORAGE" --mode stop --compress zstd 2>&1 | tee /tmp/golden-vzdump.log | grep -iE "including mount point|excluding|archive file size|Finished Backup" || true
|
||||
if grep -q "excluding volume mount point mp0" /tmp/golden-vzdump.log; then
|
||||
echo "[golden] FATAL: mp0 (/var/lib/docker) was EXCLUDED from the archive — backup=1 was lost; the golden would carry no images. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
# ASSERTION 4 (RETARGETED v3.0.0). The mp1 guard used to catch "the user-data volume fell out of the
|
||||
# archive". After the merge there is no mp1 — so the same failure now looks like the volume being
|
||||
# mounted at the WRONG PATH, which would carry the images but not the user-data area. Assert the
|
||||
# inclusion line names the volume's actual mount path.
|
||||
if ! grep -q "including mount point mp0 ('${GOLDEN_VOLUME_MP}')" /tmp/golden-vzdump.log; then
|
||||
echo "[golden] FATAL: the archive's mp0 is not ${GOLDEN_VOLUME_MP} — the single data volume is mounted somewhere unexpected, so the archive would not carry both the baked images and the user-data area. Aborting."
|
||||
grep -iE "mount point" /tmp/golden-vzdump.log || true
|
||||
exit 1
|
||||
fi
|
||||
# ASSERTION 5 (RETARGETED v3.0.0). There must be NO mp1 in the archive at all. A leftover second
|
||||
# volume means the merge did not take and this golden would ship the very ceiling R-165 removed.
|
||||
if grep -qE "mount point mp1" /tmp/golden-vzdump.log; then
|
||||
echo "[golden] FATAL: the archive still carries an mp1 — the R-165 merge did not take and this golden would ship a second, ceilinged volume. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
grep -q "including mount point mp0" /tmp/golden-vzdump.log \
|
||||
|| echo "[golden] WARN: could not confirm mp0 inclusion in the vzdump log — verify manually before using this archive."
|
||||
|
||||
VOLID=$(pvesm list "$ARCHIVE_STORAGE" --content backup 2>/dev/null | awk -v v="$VMID" '$1 ~ ("vzdump-lxc-" v "-") {print $1}' | sort | tail -1)
|
||||
echo "[golden] DONE. golden archive volid: ${VOLID:-<check ${ARCHIVE_STORAGE} dump dir>} (rootfs ${OS_SIZE_GB}G + ONE data volume ${GOLDEN_VOLUME_GB}G @ ${GOLDEN_VOLUME_MP}, all in the archive)"
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Publish to Gitea (BUNDLE slice) — make this golden fetchable by the host-bootstrap script.
|
||||
#-------------------------------------------------------------------------------
|
||||
# The host-install script fetches the golden from Gitea
|
||||
# (/api/packages/admin/generic/felhom-golden/<golden-version>/golden.tar.zst) and verifies its sha256
|
||||
# against the hub-vouched artifact manifest before importing it. <golden-version> = the BAKED
|
||||
# controller version (so the golden's published version tracks what it ships). Publishing is OPT-IN:
|
||||
# only runs when GITEA_USER + GITEA_TOKEN (or REGISTRY_USER/REGISTRY_TOKEN) are set. The local-golden
|
||||
# auto-discovery in the host-install script stays as a fallback either way.
|
||||
GITEA_BASE="${GITEA_BASE:-https://gitea.dooplex.hu}"
|
||||
GITEA_OWNER="${GITEA_OWNER:-admin}"
|
||||
PUB_USER="${GITEA_USER:-${REGISTRY_USER:-}}"
|
||||
PUB_TOKEN="${GITEA_TOKEN:-${REGISTRY_TOKEN:-}}"
|
||||
# Golden version = the baked controller tag's version (strip the image path + any leading 'v').
|
||||
GOLDEN_VERSION="${GOLDEN_VERSION:-${CONTROLLER_IMAGE##*:}}"; GOLDEN_VERSION="${GOLDEN_VERSION#v}"
|
||||
|
||||
if [ -z "$VOLID" ]; then
|
||||
echo "[golden] WARN: could not resolve the archive volid — skipping Gitea publish."
|
||||
elif [ -z "$PUB_USER" ] || [ -z "$PUB_TOKEN" ]; then
|
||||
echo "[golden] Gitea publish SKIPPED (set GITEA_USER+GITEA_TOKEN or REGISTRY_USER+REGISTRY_TOKEN to enable)."
|
||||
echo "[golden] would publish version=$GOLDEN_VERSION from volid $VOLID"
|
||||
else
|
||||
# Resolve the archive's on-disk path (pvesm path turns a volid into a filesystem path).
|
||||
ARCHIVE_PATH="$(pvesm path "$VOLID" 2>/dev/null || true)"
|
||||
if [ -z "$ARCHIVE_PATH" ] || [ ! -f "$ARCHIVE_PATH" ]; then
|
||||
echo "[golden] WARN: cannot resolve archive path for $VOLID — skipping publish."
|
||||
else
|
||||
GOLDEN_SHA256="$(sha256sum "$ARCHIVE_PATH" | awk '{print $1}')"
|
||||
PUB_URL="${GITEA_BASE}/api/packages/${GITEA_OWNER}/generic/felhom-golden/${GOLDEN_VERSION}/golden.tar.zst"
|
||||
echo "[golden] publishing golden ($(wc -c < "$ARCHIVE_PATH") bytes, sha256 ${GOLDEN_SHA256:0:16}…) → $PUB_URL"
|
||||
# Delete-then-PUT so re-publishing the same version overwrites cleanly (idempotent).
|
||||
dcode="$(curl -fsS -o /dev/null -w '%{http_code}' -u "${PUB_USER}:${PUB_TOKEN}" -X DELETE "$PUB_URL" 2>/dev/null || true)"
|
||||
echo "[golden] pre-delete existing: HTTP ${dcode} (404/204 expected)"
|
||||
ucode="$(curl -sS -o /dev/null -w '%{http_code}' -u "${PUB_USER}:${PUB_TOKEN}" -X PUT --upload-file "$ARCHIVE_PATH" "$PUB_URL")"
|
||||
if [ "$ucode" = "201" ] || [ "$ucode" = "200" ]; then
|
||||
echo "[golden] upload OK (HTTP $ucode)"
|
||||
echo "GOLDEN_VERSION=${GOLDEN_VERSION}"
|
||||
echo "GOLDEN_SHA256=${GOLDEN_SHA256}"
|
||||
echo "[golden] Record in the hub operator UI (Configs → Day-0 artifacts): golden ${GOLDEN_VERSION} / ${GOLDEN_SHA256}"
|
||||
else
|
||||
echo "[golden] WARN: golden upload failed (HTTP $ucode) — the local archive is still usable via auto-discovery."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[golden] (the build guest $VMID is stopped; destroy it with: pct destroy $VMID --purge)"
|
||||
@@ -0,0 +1,8 @@
|
||||
# /etc/felhom/crash-guard.conf — read by /usr/local/sbin/felhom-crash-guard (`11` §5.9).
|
||||
# The LIMIT-th unclean stop within WINDOW_MINUTES leaves the box off. Decided by CC unattended — operator may reverse.
|
||||
LIMIT=3
|
||||
WINDOW_MINUTES=60
|
||||
# kernel.panic while armed: seconds after a crash before the kernel restarts the box.
|
||||
PANIC_SECONDS=10
|
||||
# A tripped guard re-arms after this many hours of normal running (or `felhom-crash-guard rearm`).
|
||||
REARM_HOURS=24
|
||||
@@ -0,0 +1,19 @@
|
||||
# felhom-agent-limits.conf — start-limit + rollback-trigger drop-in for felhom-agent.service
|
||||
# (TASK D1). Install as /etc/systemd/system/felhom-agent.service.d/felhom-agent-limits.conf
|
||||
# and `systemctl daemon-reload`.
|
||||
#
|
||||
# Values are the SPIKE-agent-selfupdate-2026-07-05 tuned set, verbatim [SF-2]: with the unit's
|
||||
# Restart=on-failure + RestartSec=5s and systemd 257's compiled defaults (10s/5), a crash-looping
|
||||
# binary NEVER trips the start limit and loops forever (35 starts/180s measured). 120s/4 gives a
|
||||
# terminal `failed` ≈20s after the first crash.
|
||||
#
|
||||
# PLACEMENT TRAP [SF-3]: these keys MUST be in [Unit]. On systemd 257 a [Service] placement is
|
||||
# HALF-APPLIED — StartLimitBurst is accepted while StartLimitIntervalSec is silently ignored
|
||||
# (journal warning only). Never split them; never put them in [Service].
|
||||
#
|
||||
# OnFailure fires on EVERY crash on systemd 257 [SF-1] — see the comment block in
|
||||
# felhom-agent-rollback.service for why that is safe (pending-marker guard).
|
||||
[Unit]
|
||||
StartLimitIntervalSec=120
|
||||
StartLimitBurst=4
|
||||
OnFailure=felhom-agent-rollback.service
|
||||
@@ -0,0 +1,21 @@
|
||||
# felhom-agent-rollback.service — the OnFailure= target that auto-reverts a crash-looping agent
|
||||
# self-update (TASK D1; SPIKE-agent-selfupdate-2026-07-05).
|
||||
#
|
||||
# Install as /etc/systemd/system/felhom-agent-rollback.service. It is referenced by the
|
||||
# felhom-agent-limits.conf drop-in's OnFailure= line.
|
||||
#
|
||||
# THE PER-CRASH-FIRING REALITY [SF-1]: on systemd 257, OnFailure= fires on EVERY crash of the main
|
||||
# unit — even while it is merely `activating (auto-restart)`, long before (and regardless of) the
|
||||
# terminal start-limit `failed` state. During one crash incident this unit therefore runs MANY
|
||||
# times. That is safe BY DESIGN: the wrapper's rollback verb is pending-marker-guarded — the first
|
||||
# fire after a bad update reverts the binary and clears the marker; every later fire (and any fire
|
||||
# with no update in flight at all) is a logged no-op that touches nothing. Consequence: a bad
|
||||
# update is rolled back at the FIRST crash (~seconds), not after the start-limit burst — the tuned
|
||||
# start-limit in the drop-in is the terminal BACKSTOP (e.g. an environmental crash loop of the
|
||||
# known-good binary), not the rollback trigger.
|
||||
[Unit]
|
||||
Description=Felhom agent self-update auto-rollback (pending-guarded; fires per crash, no-ops without a pending update)
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/felhom-selfupdate-guarded rollback
|
||||
@@ -0,0 +1,50 @@
|
||||
# felhom-agent.service — canonical systemd unit for the Felhom host agent.
|
||||
#
|
||||
# Install as /etc/systemd/system/felhom-agent.service (the host-bootstrap script installs this from
|
||||
# the Gitea-published binary; previously this unit was hand-made on each host). The agent runs as the
|
||||
# NON-root `felhom-agent` service user (the documented production model — README "Process model";
|
||||
# `privileged.mode: "sudo"`) and shells the few host-root ops out via `sudo -n` against the fixed-arg
|
||||
# allowlist in /etc/sudoers.d/felhom-agent (configs/felhom-agent.sudoers). The User= here and the
|
||||
# sudoers `felhom-agent ALL=(root) NOPASSWD: …` MUST name the SAME user.
|
||||
#
|
||||
# Paths MUST match what the sudoers / host-install script expect:
|
||||
# binary /usr/local/bin/felhom-agent
|
||||
# config /etc/felhom-agent/agent.json (0600 felhom-agent:felhom-agent — secrets live here)
|
||||
# state /var/lib/felhom-agent (nonces, local-api cert/key/tokens, staged units, guests)
|
||||
#
|
||||
# === DELIBERATELY NO SANDBOXING — read before adding any hardening directive ===
|
||||
#
|
||||
# 1. NoNewPrivileges is NOT set. It is INCOMPATIBLE with the agent's privilege model: it blocks the
|
||||
# setuid `sudo` the agent relies on for EVERY host-root op (mount, format, pct, dnsmasq …), so the
|
||||
# agent would silently lose all privileged capability. The narrow surface comes from the sudoers
|
||||
# fixed-arg allowlist + the agent's in-process fine validation (internal/storage/validate.go), NOT
|
||||
# from NoNewPrivileges. Do not add it.
|
||||
#
|
||||
# 2. NO mount-namespacing hardening (ProtectHome, ProtectSystem, PrivateTmp, ReadOnlyPaths,
|
||||
# ProtectControlGroups, …). Any of these give the unit a PRIVATE mount namespace — and the agent's
|
||||
# intermediary-mount drive model does `mount --make-shared /mnt/felhom-drives` + `mount --bind` and
|
||||
# relies on those propagating into the RUNNING customer guest. In a private namespace the binds
|
||||
# would be invisible to the host/guest and every external-drive enrollment would silently break.
|
||||
# The agent MUST share the host mount namespace. The security boundary is the sudoers allowlist.
|
||||
|
||||
[Unit]
|
||||
Description=Felhom host agent (Proxmox host tier; hub control loop + PBS verify + storage watchdog)
|
||||
Documentation=https://gitea.dooplex.hu/admin/felhom-agent
|
||||
After=network-online.target pve-cluster.service pveproxy.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=felhom-agent
|
||||
Group=felhom-agent
|
||||
ExecStart=/usr/local/bin/felhom-agent --config /etc/felhom-agent/agent.json
|
||||
Restart=on-failure
|
||||
RestartSec=5s
|
||||
|
||||
# State dir: created 0750 felhom-agent:felhom-agent on start if absent (local-api cert/key/tokens,
|
||||
# nonces, staged .mount units, per-guest bootstrap dirs all live here).
|
||||
StateDirectory=felhom-agent
|
||||
StateDirectoryMode=0750
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,345 @@
|
||||
# felhom-agent sudoers allowlist — the NARROW host-root surface (slice 5 Phase B, doc 03 §3/§7; narrowed R-861).
|
||||
#
|
||||
# Install as a drop-in: /etc/sudoers.d/felhom-agent (mode 0440, root:root), validated with `visudo -cf`. It rides the
|
||||
# signed config bundle (R-840). The agent runs as the non-root `felhom-agent` user and shells out via `sudo -n` with
|
||||
# FIXED argument vectors (no shell).
|
||||
#
|
||||
# R-861 (agent v0.146.0) — EXACT PATTERNS, NOT GLOBS. A sudoers `*` in the ARGUMENTS also matches spaces, so
|
||||
# `pct set [0-9]* -onboot 1` matched `pct set 100 --dev0 /dev/sda -onboot 1` (a raw host disk for the guest), and
|
||||
# `mount --bind /mnt/*/felhom-data /mnt/felhom-drives/*` matched a `..` path onto /etc. Every argument list that varies
|
||||
# is now a sudo regular expression (`^...$`, sudo >= 1.9.10; Debian 13 / PVE 9 ship 1.9.16): one value per slot, a
|
||||
# fixed character set, no `..`, no extra argument. Lines with no variable part stay literal. The patterns are pinned
|
||||
# by the capability manifest (every real call must match: TestManifestCoveredBySudoers) and by injection cases that
|
||||
# must NOT match (configs/test_sudoers_patterns.py, and live with `sudo -l -U felhom-agent` on the demo boxes).
|
||||
#
|
||||
# R-861 — NO FILE THE AGENT WROTE IS INSTALLED WHERE ROOT READS IT. The `install` lines are gone: a mount unit, a
|
||||
# dnsmasq drop-in, the WireGuard config and the OOB sshd config + key go through `felhom-priv-apply`, a root wrapper
|
||||
# from the bundle that checks the CONTENT against the agent's own renderers; the guest pre-start hook and the shared
|
||||
# drive parent are FIXED files that come with the bundle itself; the agent binary is replaced only by an
|
||||
# operator-signed agent_update that `felhom-os-apply` verifies as root (`felhom-selfupdate-guarded apply` is no longer
|
||||
# here). The two remaining root runs of agent code (FELHOM_ESCROW, the guest hook) therefore run only a signed binary.
|
||||
#
|
||||
# Binary paths MUST match the agent config (privileged.systemctl/install/smartctl/lvs). A missing/declined entry
|
||||
# degrades the agent with a warning (SMART→UNKNOWN, mount→logged error), it does not crash; the capability probe
|
||||
# reports it to the hub.
|
||||
|
||||
Cmnd_Alias FELHOM_MOUNT = \
|
||||
/usr/local/sbin/felhom-priv-apply ^unit mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$, \
|
||||
/usr/local/sbin/felhom-priv-apply ^unit-forget mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
||||
/usr/bin/systemctl daemon-reload, \
|
||||
/usr/bin/systemctl ^enable --now -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
||||
/usr/bin/systemctl ^disable -- mnt-[A-Za-z0-9_.\\-]+\.mount$, \
|
||||
/usr/bin/systemctl ^stop -- mnt-[A-Za-z0-9_.\\-]+\.mount$
|
||||
|
||||
Cmnd_Alias FELHOM_DISK = \
|
||||
/usr/sbin/smartctl ^-a -j /dev/(sd[a-z]+|nvme[0-9]+n[0-9]+|vd[a-z]+|hd[a-z]+)$, \
|
||||
/usr/sbin/lvs ^--reportformat json --units b -o lv_name\,data_percent\,metadata_percent -- [A-Za-z0-9_.+-]+(/[A-Za-z0-9_.+-]+)?$, \
|
||||
/usr/sbin/pvs --reportformat json --noheadings -o pv_name, \
|
||||
/usr/sbin/zpool status -P
|
||||
|
||||
# Provisioning back-half (slice 8A, doc 03 §6): populate a guest's bootstrap config mount
|
||||
# host-side (internal/provision). These are host-root ops the API token cannot do — a bind mount
|
||||
# is root@pam-only, and the chown maps the 0600 bootstrap.json to the unprivileged-LXC guest-root
|
||||
# (uid/gid 100000, spike gotcha 1). The host dir is AGENT-OWNED state under /var/lib/felhom-agent/
|
||||
# (the wildcard only ever names a path the agent itself created), and the bootstrap file the agent
|
||||
# writes there is the only thing these touch. ':' is escaped per sudoers grammar.
|
||||
Cmnd_Alias FELHOM_PROVISION = \
|
||||
/usr/bin/chown ^-R 100000\:100000 /var/lib/felhom-agent/guests/[0-9]+(/bootstrap)?$, \
|
||||
/usr/sbin/pct ^set [0-9]+ -mp[0-9]+ /var/lib/felhom-agent/guests/[0-9]+/bootstrap\,mp\=/[A-Za-z0-9/_.-]+(\,ro\=1)?$, \
|
||||
/usr/sbin/pct ^set [0-9]+ -onboot 1$
|
||||
|
||||
# Disk inspection + format (slice 8C + Impl-1). blkid/lsblk read the device's data-bearing evidence
|
||||
# (the agent decides data-bearing-ness from THIS, never the caller's claim). Format goes ONLY through
|
||||
# felhom-mkfs-guarded (Impl-1 Part B): raw mkfs.* is NO LONGER allowlisted, so even a bad agent cannot
|
||||
# mkfs the OS disk — the wrapper re-checks the catastrophic cases (system disk / LVM PV / foreign mount)
|
||||
# as root and refuses, and the agent's unclaimed-disk filter (claim.go) is the primary guard above it.
|
||||
Cmnd_Alias FELHOM_FORMAT = \
|
||||
/usr/sbin/blkid ^-p -o export /dev/[^ ]+$, \
|
||||
/usr/bin/lsblk ^-J -o NAME\,FSTYPE\,PTTYPE\,MOUNTPOINT /dev/[^ ]+$, \
|
||||
/usr/local/sbin/felhom-mkfs-guarded ^/dev/[^ ]+ (ext4|xfs)$
|
||||
|
||||
# LAN split-horizon resolver (internal/lanresolver): the agent manages a host-side dnsmasq that
|
||||
# answers *.<customer-domain> with each guest's live LAN IP. A felhom-*.conf drop-in reaches /etc/dnsmasq.d
|
||||
# only through felhom-priv-apply, which allows exactly the lines the resolver renders (R-861: a `dhcp-script=` would
|
||||
# run as root); the two `pct exec` reads are FIXED command vectors
|
||||
# (the guest's eth0 IPv4 + the controller's pulled controller.yaml for the domain) — NOT a general
|
||||
# `pct exec`. systemctl is scoped to the dnsmasq unit only. The agent never edits /etc/resolv.conf.
|
||||
Cmnd_Alias FELHOM_DNSMASQ = \
|
||||
/usr/bin/apt-get install -y -q dnsmasq, \
|
||||
/usr/local/sbin/felhom-priv-apply ^dnsmasq /tmp/felhom-resolver-[0-9]+\.conf felhom-[a-z0-9][a-z0-9._-]*\.conf$, \
|
||||
/usr/bin/systemctl enable --now dnsmasq, \
|
||||
/usr/bin/systemctl reload dnsmasq, \
|
||||
/usr/bin/systemctl restart dnsmasq, \
|
||||
/usr/bin/rm ^-f /etc/dnsmasq\.d/felhom-[a-z0-9][a-z0-9._-]*\.conf$, \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- ip -4 -o addr show dev eth0$, \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- docker exec felhom-controller cat /opt/docker/felhom-controller/controller\.yaml$
|
||||
|
||||
# Guest mountpoint lifecycle (intermediary-mount re-architecture + C1 net). The pre-start self-heal hook is a FIXED file
|
||||
# from the config bundle (/var/lib/vz/snippets/felhom-guest-hook.sh — R-861: the agent no longer installs it from /tmp;
|
||||
# Proxmox runs it as root at every guest start). The agent only registers it per guest, deletes a dead mountpoint slot
|
||||
# (the B3 C1 fix) and reboots a guest to activate binds — each with an exact vmid / slot.
|
||||
Cmnd_Alias FELHOM_GUESTHOOK = \
|
||||
/usr/sbin/pct ^set [0-9]+ --hookscript local\:snippets/felhom-guest-hook\.sh$, \
|
||||
/usr/sbin/pct ^set [0-9]+ --delete mp[0-9]+$, \
|
||||
/usr/sbin/pct ^reboot [0-9]+$, \
|
||||
/usr/sbin/pct ^shutdown [0-9]+ --timeout [0-9]+$
|
||||
|
||||
# Host power-off (R-998, doc 14): the household's „A gép kikapcsolása" — guest shutdown above, then this one fixed
|
||||
# vector. Guarded agent-side (10 min after boot, one per boot, never during a backup/restore/update/drive job).
|
||||
Cmnd_Alias FELHOM_POWEROFF = \
|
||||
/usr/bin/systemctl poweroff
|
||||
|
||||
# Dead half-written vzdump removal (R-1006, power-cut-fixes-2026-10-14): a power cut mid-backup leaves
|
||||
# vzdump-lxc-<vmid>-<date>.tar.dat + .tmp/ on the dir storage; nothing removed them and every later backup was skipped
|
||||
# „not enough space". ONE root helper (configs/felhom-vzdump-partial-clean, from the config bundle) re-checks everything
|
||||
# as root — a dir storage from storage.cfg, the exact name of THAT vmid, not in use, dead — and refuses otherwise. The
|
||||
# arguments here are its exact shape (storage id, vmid, one half-written name); the vmid-in-name match is the helper's.
|
||||
Cmnd_Alias FELHOM_VZPARTIAL = \
|
||||
/usr/local/sbin/felhom-vzdump-partial-clean ^[A-Za-z][A-Za-z0-9_-]{0\,63} [1-9][0-9]{2\,8} vzdump-(lxc|qemu)-[1-9][0-9]{2\,8}-[0-9]{4}_[0-9]{2}_[0-9]{2}-[0-9]{2}_[0-9]{2}_[0-9]{2}\.(tar\.dat|vma\.dat|tmp)$
|
||||
|
||||
# Internal data place („Belső tárhely", felhom.eu architecture/16 §5, §7; agent v0.158.0). ONE root helper
|
||||
# (configs/felhom-internal-place, from the config bundle) creates <vg>/felhom-belso in the local-lvm thin pool by the
|
||||
# ruled size rule and formats ONLY that volume, ONLY when it carries no signature. The agent calls it from the signed
|
||||
# `add_internal_place` job (an old box), always with 0 GiB planned: its guest's volumes already exist. `plan` changes
|
||||
# nothing. The installer runs the helper as root itself (a new box).
|
||||
# R-1012 (agent v0.159.0): the READ-ONLY orphan-thin check before and after a whole-guest backup — lists the pool's thin
|
||||
# device ids (through a metadata snapshot it reserves and releases) next to LVM's. It sends no other pool message; the
|
||||
# repair stays a human step. Pinned by configs/test_felhom_thin_check.py.
|
||||
Cmnd_Alias FELHOM_THINCHECK = \
|
||||
/usr/local/sbin/felhom-thin-check ^[A-Za-z0-9_+][A-Za-z0-9_.+-]{0\,63} [A-Za-z0-9_+][A-Za-z0-9_.+-]{0\,63}$
|
||||
Cmnd_Alias FELHOM_INTERNALPLACE = \
|
||||
/usr/local/sbin/felhom-internal-place ^(plan|create) 0$
|
||||
|
||||
# Intermediary mount model (the drive hot-swap re-architecture). The agent keeps a SHARED host parent
|
||||
# /mnt/felhom-drives (self-bind + make-shared) and binds/unbinds each drive's felhom-data namespace UNDERNEATH it so the
|
||||
# change propagates into the running guest live. The boot-persistence script + unit are FIXED files from the config
|
||||
# bundle (R-861: the agent no longer installs them from /tmp); the agent only enables the unit. A drive name is one
|
||||
# path segment that cannot start with a dot (no `..`); `lxc-info -n <vmid> -p -H` resolves the guest init PID for the
|
||||
# GuestSeesMount check; `make-private` isolates the parent's peer group on FIRST setup only.
|
||||
Cmnd_Alias FELHOM_INTERMEDIARY = \
|
||||
/usr/bin/mkdir -p /mnt/felhom-drives, \
|
||||
/usr/bin/mkdir ^-p /mnt/felhom-drives/[A-Za-z0-9_-][A-Za-z0-9_.-]*$, \
|
||||
/usr/bin/mkdir ^-p /mnt/[A-Za-z0-9_-][A-Za-z0-9_.-]*/felhom-data$, \
|
||||
/usr/bin/chown ^100000\:100000 /mnt/[A-Za-z0-9_-][A-Za-z0-9_.-]*/felhom-data$, \
|
||||
/usr/bin/mount --bind /mnt/felhom-drives /mnt/felhom-drives, \
|
||||
/usr/bin/mount --make-shared /mnt/felhom-drives, \
|
||||
/usr/bin/mount --make-private /mnt/felhom-drives, \
|
||||
/usr/bin/mount ^--bind /mnt/[A-Za-z0-9_-][A-Za-z0-9_.-]*/felhom-data /mnt/felhom-drives/[A-Za-z0-9_-][A-Za-z0-9_.-]*$, \
|
||||
/usr/bin/umount ^/mnt/felhom-drives/[A-Za-z0-9_-][A-Za-z0-9_.-]*$, \
|
||||
/usr/bin/systemctl enable felhom-shared-parent.service, \
|
||||
/usr/bin/lxc-info ^-n [0-9]+ -p -H$, \
|
||||
/usr/sbin/pct ^set [0-9]+ -mp8 /mnt/felhom-drives\,mp\=/mnt/felhom-drives$
|
||||
|
||||
# Controller-swap / managed auto-update (Option A, non-root). The agent owns the in-guest controller
|
||||
# image SWAP (it survives the controller being killed mid-swap): read the baked image ref, check the
|
||||
# pre-pulled target is present, rewrite /etc/felhom-controller-image, restart the bootstrap unit,
|
||||
# health-check, roll back on failure. Each grant is bounded — NO general `pct exec` and NO `bash -c`:
|
||||
# cat <fixed file> — read the current image ref (read-only)
|
||||
# docker image inspect * — is the pre-pulled target present? (read-only)
|
||||
# docker inspect -f * — container running/health/image (read-only; `*` spans the -f template
|
||||
# + container across spaces, spike-confirmed)
|
||||
# systemctl restart <fixed unit> — re-run the golden's bootstrap (the only state change)
|
||||
# felhom-priv-apply controller-image <vmid> — WRITE the ref (R-861 (a) A1, `09` §3 decision 165): the ref goes on
|
||||
# STDIN to the ROOT wrapper, which requires our registry + repository + an x.y.z tag
|
||||
# and writes the guest file itself. The agent's own `tee` grant is GONE: before, a
|
||||
# compromised agent could hand the guest's bootstrap ANY image (sudo cannot see stdin).
|
||||
# Validated GO: felhom.eu/documentation/audits/SPIKE-controllerswap-narrow-grants-2026-06-29.md.
|
||||
Cmnd_Alias FELHOM_CONTROLLERSWAP = \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- cat /etc/felhom-controller-image$, \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- docker image inspect gitea\.dooplex\.hu/admin/felhom-controller\:[0-9]+\.[0-9]+\.[0-9]+$, \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- docker inspect -f .+ (felhom-controller|cloudflared)$, \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- systemctl restart felhom-controller-bootstrap\.service$, \
|
||||
/usr/local/sbin/felhom-priv-apply ^controller-image [0-9]+$
|
||||
|
||||
# Stale-lock recovery (F2-b, v0.49.0). A host reboot DURING a vzdump backup leaves the guest with a
|
||||
# `snapshot-delete`/`backup` lock + `onboot:1` then can't start it → the customer box stays DOWN. The
|
||||
# agent clears the STALE lock at startup (only when no vzdump is in-flight). `pct unlock` is the one op
|
||||
# with no API equivalent (snapshot-delete + start go through the API token); the agent fine-validates the
|
||||
# vmid (numeric) before exec — the `[0-9]*` is the coarse allowlist.
|
||||
Cmnd_Alias FELHOM_STALELOCK = \
|
||||
/usr/sbin/pct ^unlock [0-9]+$
|
||||
|
||||
# Weekly guest disk trim (R-444, operator ruling `09` §3 decision 139). A thin pool only ever grows from blocks the
|
||||
# guest has FREED: `fstrim` inside the unprivileged container is refused (FITRIM: Operation not permitted), so the host
|
||||
# trims the guest's mounts. Measured on demo-hp 2026-10-06: `pct fstrim 9201` rc 0 in 24.4 s, pool 65.53 % -> 33.40 %,
|
||||
# apps kept answering. ONE exact pattern: a vmid and nothing else — no `--ignore-mountpoints`, no second argument
|
||||
# (pinned: TestSudoersFstrimRuleIsExact). The agent runs it on a weekly daytime timer under the heavy-op gate.
|
||||
Cmnd_Alias FELHOM_FSTRIM = \
|
||||
/usr/sbin/pct ^fstrim [0-9]+$
|
||||
|
||||
# Restore-test scratch teardown (F-LEAK, Campaign 8, v0.110.0). A restore-test whose restore FAILS
|
||||
# leaves a scratch guest the API token CANNOT destroy: `FelhomAgentGuest` is granted at /pool/felhom and
|
||||
# a guest joins that pool only when its restore COMPLETES, so a failed restore leaves a pool-less guest
|
||||
# out of reach (403 VM.Allocate) holding its disks until a human removes it.
|
||||
#
|
||||
# TWO API-SIDE FIXES WERE TRIED AND BOTH REFUTED LIVE on 2026-07-28, which is why this grant exists:
|
||||
# 1. Adopt the stranded guest into the pool, then retry. `PUT /pools/{pool}` ALSO requires
|
||||
# VM.Allocate on the VM being added — pool membership cannot bootstrap its own authority.
|
||||
# 2. Grant FelhomAgentGuest per-path at /vms/990000..990009. Durable for exactly one use per slot:
|
||||
# PVE's own destroy path calls `AccessControl::remove_vm_access($vmid)` (LXC.pm:906), which DELETES
|
||||
# every ACL at /vms/<vmid> (AccessControl.pm:1898). The grant is consumed by the operation it
|
||||
# authorises, so after ten teardowns the band is ungranted and the defect returns.
|
||||
#
|
||||
# WHY THIS IS THE TIGHTEST AVAILABLE FENCE, not a widening: sudo matches the vmid LITERALLY, so
|
||||
# `99000[0-9]` is exactly the ten-slot scratch band the restore-test picks from — nothing else. There is
|
||||
# no `[0-9]*` coarse allowlist here on purpose: unlike `pct unlock`, this op DESTROYS, so the band must
|
||||
# be in the policy and not merely validated in the agent. Even a compromised agent asking for
|
||||
# `pct destroy 9201` is refused by sudo itself. Unlike an ACL, a sudoers rule is not consumed by use.
|
||||
# The agent re-checks the band in code before exec (defence in depth); this is the outer fence.
|
||||
Cmnd_Alias FELHOM_SCRATCH_TEARDOWN = \
|
||||
/usr/sbin/pct destroy 99000[0-9] --purge
|
||||
|
||||
# Network storage / NAS (Part A1, SPIKE-nas-storage-2026-06-29). The agent mounts a customer NAS share
|
||||
# HOST-SIDE under /mnt/felhom-drives/<name> via a systemd .automount (+ .mount) pair so it propagates
|
||||
# into the guest through the existing shared bind (an unprivileged LXC cannot mount NFS/CIFS itself).
|
||||
# A NAS is NOT a drive — no durable-id, no SMART, no wipe; these grants only install/enable/remove the
|
||||
# unit pair. The agent fine-validates every value (share name, server, export, uid/gid, creds path) before
|
||||
# any unit is rendered (internal/storage/netmount.go ValidateNetworkMountSpec); the unit FILE reaches
|
||||
# /etc/systemd/system only through `felhom-priv-apply unit` (FELHOM_MOUNT), which requires nosuid,nodev on a network
|
||||
# share (R-861). The `.mount` enable/disable/stop reuse FELHOM_MOUNT; this alias adds the
|
||||
# `.automount` variants + the unit-file removal. The unit FILE name is the systemd-escaped mountpoint,
|
||||
# which always begins `mnt-felhom` (the mountpoint is /mnt/felhom-drives/<name>), so the rm glob is scoped
|
||||
# to felhom mount units only. mkdir of the mountpoint reuses FELHOM_INTERMEDIARY's /mnt/felhom-drives/*.
|
||||
# CAMPAIGN-3 additions (loud, per the no-widening rule):
|
||||
# - `systemctl reset-failed -- mnt-felhom*`: F10 (CRITICAL) — a NAS automount that hit
|
||||
# mount-start-limit-hit during an outage was re-armable by NO platform path; the reassert now
|
||||
# reset-failed's the stuck unit before `enable --now` (which the start-limit otherwise refuses),
|
||||
# and RemoveNetworkMount clears failed-state residue (F2). Scoped to felhom mount units (the unit
|
||||
# name is the systemd-escaped mountpoint, always beginning `mnt-felhom`). reset-failed only clears
|
||||
# a unit's failed latch — it cannot start/stop/alter anything.
|
||||
# - `rmdir /mnt/felhom-drives/*`: F1 — remove the now-empty mountpoint dir a removed share leaves
|
||||
# behind (the campaign accumulated 10 stub-shaped leftovers). rmdir ONLY (never rm -rf): it refuses
|
||||
# a non-empty dir, so unexpected data is preserved, not destroyed — a fail-safe grant.
|
||||
Cmnd_Alias FELHOM_NETMOUNT = \
|
||||
/usr/bin/systemctl ^enable --now -- mnt-[A-Za-z0-9_.\\-]+\.automount$, \
|
||||
/usr/bin/systemctl ^disable -- mnt-[A-Za-z0-9_.\\-]+\.automount$, \
|
||||
/usr/bin/systemctl ^stop -- mnt-[A-Za-z0-9_.\\-]+\.automount$, \
|
||||
/usr/bin/systemctl ^reset-failed -- mnt-felhom[A-Za-z0-9_.\\-]*\.(mount|automount)$, \
|
||||
/usr/bin/rmdir ^/mnt/felhom-drives/[A-Za-z0-9_-][A-Za-z0-9_.-]*$, \
|
||||
/usr/bin/rm ^-f /etc/systemd/system/mnt-felhom[A-Za-z0-9_.\\-]*\.(mount|automount)$
|
||||
|
||||
# Offsite WG tunnel (S3, doc 06 §3.3). The agent manages wg-quick@wg-felhom as an agent-managed
|
||||
# host service (the dnsmasq/lanresolver shape): conf staged in the agent-owned StateDir (never
|
||||
# /tmp), installed 0600 to the FIXED destination by `felhom-priv-apply wg`, which refuses any key renderConf never
|
||||
# writes (R-861: PostUp/PreUp run as root under wg-quick), unit enable/restart/disable. The ONLY wg read
|
||||
# is `latest-handshakes` — `wg show <if> dump` is FORBIDDEN everywhere (its interface line
|
||||
# carries the PRIVATE KEY; the S1 session-log incident). Source and destination are fixed in the wrapper.
|
||||
Cmnd_Alias FELHOM_WG = \
|
||||
/usr/bin/apt-get install -y -q wireguard-tools, \
|
||||
/usr/local/sbin/felhom-priv-apply wg, \
|
||||
/usr/bin/systemctl enable --now wg-quick@wg-felhom, \
|
||||
/usr/bin/systemctl restart wg-quick@wg-felhom, \
|
||||
/usr/bin/systemctl disable --now wg-quick@wg-felhom, \
|
||||
/usr/bin/wg show wg-felhom latest-handshakes
|
||||
|
||||
# Agent self-update (TASK D1; R-861). The A/B flip (`felhom-selfupdate-guarded apply`) is NO LONGER the agent's: the
|
||||
# agent hands the operator-SIGNED agent_update to felhom-os-apply (FELHOM_OSAPPLY, mode agent_update), which verifies
|
||||
# the signature as root and only then runs the flip. Until v0.146.0 the agent passed the sha itself, so a compromised
|
||||
# agent could install any binary — the binary FELHOM_ESCROW and the guest hook run as root. `commit` (clear the pending
|
||||
# marker) and `rollback` (pending-guarded revert, normally run by felhom-agent-rollback.service) stay.
|
||||
Cmnd_Alias FELHOM_SELFUPDATE = \
|
||||
/usr/local/sbin/felhom-selfupdate-guarded commit, \
|
||||
/usr/local/sbin/felhom-selfupdate-guarded rollback
|
||||
|
||||
# Dedicated OOB sshd (TASK H1). The agent manages felhom-sshd like wg-felhom/dnsmasq: it RENDERS the
|
||||
# config (Port from its claim) + the operator's authorized_keys, validates with `sshd -t`, and reloads
|
||||
# (never restart-on-change [SF-2]). Both files reach /etc/felhom-sshd only through felhom-priv-apply (R-861): the config
|
||||
# must be the ONE template with only the Port varying (an AuthorizedKeysFile the agent owns + `StrictModes no` would be
|
||||
# a root login), the key file one plain public key without options. `sshd -t/-T` are the validate/discover reads. The
|
||||
# systemctl verbs are SCOPED to felhom-sshd only. reset-failed precedes a deliberate restart [SF-5].
|
||||
# NOTHING here can touch the stock sshd, :22, or /etc/ssh.
|
||||
Cmnd_Alias FELHOM_SSHD = \
|
||||
/usr/local/sbin/felhom-priv-apply sshd-config, \
|
||||
/usr/local/sbin/felhom-priv-apply sshd-key, \
|
||||
/usr/sbin/sshd -t -f /var/lib/felhom-agent/felhom-sshd/sshd_config, \
|
||||
/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config, \
|
||||
/usr/sbin/sshd -T -f /etc/felhom-sshd/sshd_config, \
|
||||
/usr/bin/systemctl enable --now felhom-sshd, \
|
||||
/usr/bin/systemctl reload felhom-sshd, \
|
||||
/usr/bin/systemctl restart felhom-sshd, \
|
||||
/usr/bin/systemctl reset-failed felhom-sshd, \
|
||||
/usr/bin/wg show wg-felhom latest-handshakes
|
||||
|
||||
# PBS DR tier apply (slice 2, SPIKE-pbs-tier-provisioning-2026-07-10 §2b). Storage-entry
|
||||
# lifecycle is /storage-ROOT-gated in the PVE API (spike Probe 1: create/modify/delete all check
|
||||
# Datastore.Allocate on /storage), so the agent token cannot do it — this wrapper is the pinned
|
||||
# vector. THE SET-ONLY LAW: the wrapper contains NO deletion path (entry deletion destroys the
|
||||
# client encryption key = un-decryptable backups); verbs are create/reconcile/grant only. The
|
||||
# token secret rides the wrapper's STDIN — sudo logs argv, so it must never appear here. The
|
||||
# agent fine-validates every field (charset + descriptor equality) before exec; these globs are
|
||||
# the coarse allowlist.
|
||||
#
|
||||
# `read` (R-39 leg b, agent v0.91.0) is the ONE added verb. It prints a token secret to stdout and
|
||||
# performs no mutation. It exists because the agent writes that file through this wrapper but could
|
||||
# never read it back (/etc/pve/priv is 0700 root:www-data), leaving its PBS verify loop permanently
|
||||
# blind to an `applied`-but-401 tier. It is NOT a general file-read: the wrapper pins the directory
|
||||
# and prefix-asserts the resolved path, and the id grammar admits no slash. The secret goes to
|
||||
# STDOUT, never argv — sudo logs argv.
|
||||
# E-2a: the backup-target storage shim. Creating a PVE storage needs Datastore.Allocate at /storage
|
||||
# and the grant needs Permissions.Modify -- the agent holds NEITHER by design (blast-radius
|
||||
# containment; Permissions.Modify would let it rewrite its own authority). Both live behind this
|
||||
# fixed-vocabulary root shim instead, exactly like the mkfs and pbs-apply wrappers. The wrapper has
|
||||
# NO storage-removal path, enforces is_mountpoint 1, and refuses a target on the root device.
|
||||
Cmnd_Alias FELHOM_BACKUPTARGET = \
|
||||
/usr/local/sbin/felhom-backup-target-apply create *, \
|
||||
/usr/local/sbin/felhom-backup-target-apply grant *
|
||||
|
||||
Cmnd_Alias FELHOM_PBSDR = \
|
||||
/usr/local/sbin/felhom-pbs-apply create *, \
|
||||
/usr/local/sbin/felhom-pbs-apply reconcile *, \
|
||||
/usr/local/sbin/felhom-pbs-apply grant *, \
|
||||
/usr/local/sbin/felhom-pbs-apply read *
|
||||
|
||||
# OOB nft belt (TASK H1). The STATIC table `inet felhom_oob` is installed once by host-install; the
|
||||
# agent mutates ONLY its two SETS — @operator_ips (the operator /32) + @ssh_port (the claimed port).
|
||||
# SET ELEMENTS ONLY [trap 4]: NO `nft add rule`, NO `nft -f`, NO `flush ruleset/table` — a rule grant
|
||||
# would let the agent firewall anything. The agent fine-validates every element (netip / int range)
|
||||
# before exec; the trailing wildcards are the coarse allowlist (values only).
|
||||
Cmnd_Alias FELHOM_OOB = \
|
||||
/usr/sbin/nft list set inet felhom_oob operator_ips, \
|
||||
/usr/sbin/nft list set inet felhom_oob ssh_port, \
|
||||
/usr/sbin/nft flush set inet felhom_oob operator_ips, \
|
||||
/usr/sbin/nft flush set inet felhom_oob ssh_port, \
|
||||
/usr/sbin/nft ^add element inet felhom_oob operator_ips \{ [0-9.]+(/[0-9]+)? \}$, \
|
||||
/usr/sbin/nft ^add element inet felhom_oob ssh_port \{ [0-9]+ \}$
|
||||
|
||||
# Escrow ceremony (controller-driven, TASK 2026-07-13; mechanics validated by
|
||||
# SPIKE-controller-escrow-2026-07-13). ONE fixed argv — sudoers matches the argument vector
|
||||
# byte-for-byte (spike §2.2): any alteration (value, extra flag, order, config path) is refused.
|
||||
# --config pinned: env_reset strips FELHOM_AGENT_CONFIG and the pin closes alternate-config
|
||||
# injection. R rides the subprocess stdout pipe only; sudo logs argv = secrets-free. The argv
|
||||
# MUST stay byte-identical to escrow.CeremonyArgs() (internal/escrow/ceremony.go) — the
|
||||
# capability manifest entry + TestManifestCoveredBySudoers lock the three copies together.
|
||||
Cmnd_Alias FELHOM_ESCROW = \
|
||||
/usr/local/bin/felhom-agent --config /etc/felhom-agent/agent.json --selftest=escrow-create --upload --output=json
|
||||
|
||||
# Node self-heal (CAMPAIGN-3 Part 6, F12-class defense in depth). The ONE fixed unit the appliance
|
||||
# watchdog may (re)start when a boot leaves networking down — the exact command the morning recovery
|
||||
# ran by hand after the F12 host loss. FIXED unit, no glob: this grant alone cannot harm — starting
|
||||
# networking.service is precisely what the boot should have done. The remedy is ALSO code-gated on
|
||||
# deployment_mode="appliance" (the Manager refuses to invoke it on a byo host); the sudoers grant is
|
||||
# the coarse floor, the mode gate is the fine one.
|
||||
Cmnd_Alias FELHOM_SELFHEAL = \
|
||||
/usr/bin/systemctl start networking.service
|
||||
|
||||
# Guest-network watchdog (internal/guestnet, R-54). The guest's DHCP client is unsupervised — when it
|
||||
# died on 2026-07-20 the box lost its address ~80 minutes later and went off the internet for 1h15m
|
||||
# (INCIDENT-guest-dhclient-killed-2026-07-20). Four FIXED read vectors plus ONE fixed heal vector; the
|
||||
# heal is the incident's own restored invocation, byte for byte. This is NOT a general `pct exec`: every
|
||||
# argument after the numeric vmid is a literal, so the grant cannot be widened by anything the guest or
|
||||
# the hub says. The address read is deliberately NOT duplicated here — it is already FELHOM_DNSMASQ's,
|
||||
# and the same command must not be granted twice under two names.
|
||||
# OS updates, guest fast lane (`11-os-updates.md` §5.4.1, agent v0.140.0). The ONLY entry: the root wrapper with
|
||||
# one plan file in the agent's own os/ dir. Every safety rule (no removal, no downgrade, no new or unlisted package,
|
||||
# Debian origin only, the box's own customer guest only) lives in the wrapper, red-proved per rule
|
||||
# (configs/test_felhom_os_apply.py). The agent gets NO apt grant of its own.
|
||||
Cmnd_Alias FELHOM_OSAPPLY = \
|
||||
/usr/local/sbin/felhom-os-apply --plan /var/lib/felhom-agent/os/plan-*.json
|
||||
|
||||
Cmnd_Alias FELHOM_GUESTNET = \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- ip route show default$, \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- cat /etc/network/interfaces$, \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- pgrep -x dhclient$, \
|
||||
/usr/sbin/pct ^exec [0-9]+ -- dhclient -pf /run/dhclient\.eth0\.pid -lf /var/lib/dhcp/dhclient\.eth0\.leases eth0$
|
||||
|
||||
felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_FSTRIM, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_BACKUPTARGET, FELHOM_SELFHEAL, FELHOM_ESCROW, FELHOM_GUESTNET, FELHOM_SCRATCH_TEARDOWN, FELHOM_OSAPPLY, FELHOM_POWEROFF, FELHOM_VZPARTIAL, FELHOM_INTERNALPLACE, FELHOM_THINCHECK
|
||||
Executable
+111
@@ -0,0 +1,111 @@
|
||||
#!/bin/bash
|
||||
#===============================================================================
|
||||
# felhom-backup-target-apply — the ONLY path the felhom-agent sudoers permits for creating the
|
||||
# whole-guest backup TARGET storage and granting the agent access to it (E-2a).
|
||||
#
|
||||
# WHY A WRAPPER AT ALL. Creating a PVE storage needs `Datastore.Allocate` at `/storage`, and the ACL
|
||||
# grant needs `Permissions.Modify`. The agent holds NEITHER by design — its token is scoped per
|
||||
# storage path for blast-radius containment, and `Permissions.Modify` would let it rewrite its own
|
||||
# authority. Widening the PVE role to make the move possible would trade the entire containment model
|
||||
# for one feature. So the privileged half lives here: a minimal, auditable root shim with a fixed
|
||||
# vocabulary, exactly like felhom-mkfs-guarded and felhom-pbs-apply.
|
||||
#
|
||||
# THE NO-DELETE LAW (inherited from felhom-pbs-apply, same reasoning class). This wrapper contains NO
|
||||
# storage-removal path of any kind. `pvesm remove` on a dir storage does not delete the archives, but
|
||||
# it DOES silently orphan a configured backup tier, and a "cleanup" verb here would be reachable by
|
||||
# any bug in the agent. Retiring a target is a deliberate operator op, not this tool. Grep-assertable;
|
||||
# do not add one.
|
||||
#
|
||||
# THE TWO LAWS E-1 PAID FOR ON LIVE HARDWARE, both enforced here rather than trusted to the caller:
|
||||
#
|
||||
# F-1 the storage path must BE the drive's own mountpoint. A subdirectory fails the agent's
|
||||
# exactMount check, so the target reports `disconnected` FOREVER and its durable id degrades
|
||||
# off the filesystem UUID. Enforced: `mountpoint -q` must pass on the exact path given.
|
||||
#
|
||||
# F-2 --is_mountpoint 1 is not optional. Without it, an unplugged or late-mounting drive leaves a
|
||||
# bare directory on the ROOT filesystem and vzdump writes the whole-guest backup onto the
|
||||
# system drive — the exact device the whole change exists to escape — while PVE reports the
|
||||
# storage `active` and advertises the root filesystem's free space. Proven live: the unguarded
|
||||
# form had already created dump/ on pve-root. Hardcoded below; not a caller-supplied flag.
|
||||
#
|
||||
# Ops (all non-secret; nothing here touches a credential, so nothing arrives on stdin):
|
||||
# create <id> <mountpoint>
|
||||
# Create a `dir` storage with content=backup at <mountpoint>, is_mountpoint 1.
|
||||
# IDEMPOTENT: an existing entry with the SAME path is accepted (re-run safe, and the
|
||||
# installer re-run path depends on it). An existing entry with a DIFFERENT path is REFUSED
|
||||
# — silently repointing a live backup target is the failure this whole arc closes.
|
||||
# grant <id>
|
||||
# The dual grant: FelhomAgentStore on /storage/<id> to the agent user AND token (privsep
|
||||
# intersection — a token's rights are the intersection, so granting one is granting neither).
|
||||
# Without it every backup 403s on first run (E-1 finding F-3, found by the first real backup).
|
||||
#===============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
die() { echo "felhom-backup-target-apply: REFUSED: $*" >&2; exit 1; }
|
||||
|
||||
op="${1:-}"; id="${2:-}"
|
||||
[[ -n "$op" && -n "$id" ]] || die "usage: felhom-backup-target-apply <create|grant> <storage-id> [mountpoint]"
|
||||
|
||||
# Storage id: PVE grammar, conservative. Also the ACL path component — no slashes possible.
|
||||
[[ "$id" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,27}$ ]] || die "bad storage id ($id)"
|
||||
|
||||
STORECFG=/etc/pve/storage.cfg
|
||||
|
||||
# current_path_of <id> — the configured `path` of dir storage <id>, or "" when absent/not-a-dir.
|
||||
current_path_of() {
|
||||
awk -v want="dir: $1" '
|
||||
$0 == want { found=1; next }
|
||||
found && /^[a-z]+: / { exit }
|
||||
found && $1 == "path" { print $2; exit }
|
||||
' "$STORECFG" 2>/dev/null || true
|
||||
}
|
||||
|
||||
case "$op" in
|
||||
create)
|
||||
[[ $# -eq 3 ]] || die "create takes <id> <mountpoint>"
|
||||
mp="$3"
|
||||
# Absolute, normalized, no traversal, no shell metacharacters. The value reaches pvesm and the
|
||||
# filesystem, so it is validated here rather than assumed well-formed.
|
||||
[[ "$mp" = /* ]] || die "mountpoint must be absolute ($mp)"
|
||||
[[ "$mp" != *".."* ]] || die "mountpoint must not contain .. ($mp)"
|
||||
[[ "$mp" =~ ^[A-Za-z0-9/_.-]+$ ]] || die "mountpoint has unexpected characters ($mp)"
|
||||
[[ "$mp" != "/" ]] || die "refusing / as a backup target"
|
||||
|
||||
# F-1 + F-2, checked as one: the path must BE a mountpoint right now. A bare directory here is
|
||||
# precisely the silent-retarget shape, and is_mountpoint would make PVE refuse it later anyway —
|
||||
# better to refuse now, with a reason, than to create a storage that can never activate.
|
||||
mountpoint -q "$mp" || die "$mp is not a mountpoint — the backup target must be the drive's OWN mountpoint (F-1), and an unmounted path would silently retarget onto the system drive (F-2)"
|
||||
|
||||
# Never the system disk: a target on the root filesystem is not drive-loss protection, it is the
|
||||
# thing we are escaping. The root device and the candidate's device are compared, not their paths.
|
||||
root_dev="$(findmnt -no SOURCE / 2>/dev/null || true)"
|
||||
mp_dev="$(findmnt -no SOURCE "$mp" 2>/dev/null || true)"
|
||||
[[ -n "$mp_dev" ]] || die "could not resolve the backing device of $mp"
|
||||
[[ "$mp_dev" != "$root_dev" ]] || die "$mp is backed by the ROOT device ($root_dev) — a backup target there protects against corruption only, never drive loss"
|
||||
|
||||
existing="$(current_path_of "$id")"
|
||||
if [[ -n "$existing" ]]; then
|
||||
if [[ "$existing" == "$mp" ]]; then
|
||||
echo "felhom-backup-target-apply: storage $id already exists at $mp — nothing to do (idempotent)" >&2
|
||||
exit 0
|
||||
fi
|
||||
die "storage $id already exists at $existing — refusing to repoint it at $mp (a live backup target is never silently moved)"
|
||||
fi
|
||||
|
||||
# is_mountpoint 1 is HARDCODED (F-2). content=backup only: this storage exists for vzdump archives
|
||||
# and must never become a place guests are allocated on.
|
||||
pvesm add dir "$id" --path "$mp" --content backup --is_mountpoint 1 >&2
|
||||
echo "felhom-backup-target-apply: created dir storage $id at $mp (content=backup, is_mountpoint 1)" >&2
|
||||
;;
|
||||
grant)
|
||||
[[ $# -eq 2 ]] || die "grant takes only <id>"
|
||||
# BOTH, always. A privsep token's rights are the intersection of the user's and the token's ACLs,
|
||||
# so granting one of the two grants nothing usable.
|
||||
pveum acl modify "/storage/$id" --users felhom-agent@pve --roles FelhomAgentStore >&2
|
||||
pveum acl modify "/storage/$id" --tokens 'felhom-agent@pve!agent' --roles FelhomAgentStore >&2
|
||||
echo "felhom-backup-target-apply: granted FelhomAgentStore on /storage/$id (user + token)" >&2
|
||||
;;
|
||||
*)
|
||||
die "unknown op ($op)"
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/python3
|
||||
# felhom-crash-guard — a crashed host restarts by itself, but not forever (`09` decision 88, R-851, `11` §5.9).
|
||||
#
|
||||
# Install as /usr/local/sbin/felhom-crash-guard (0755 root:root), with felhom-crash-guard.service (boot / clean-stop)
|
||||
# and felhom-crash-guard-check.timer (hourly re-arm check). Python 3, standard library only.
|
||||
# Tests: configs/test_felhom_crash_guard.py (temp dirs; nothing real is touched).
|
||||
#
|
||||
# WHAT IT DOES
|
||||
# boot early at every boot. Was the previous boot ended CLEANLY? (the clean-stop marker exists). If not, this
|
||||
# boot follows an UNCLEAN stop — a kernel crash, a power cut or a hard reset (they cannot be told apart
|
||||
# on these boxes: measured 2026-10-04 on demo-hp, efi_pstore is on yet saved NOTHING for a real panic;
|
||||
# the journal and `last` show only "no shutdown"). It records the unclean boot, counts those in the last
|
||||
# WINDOW_MINUTES, and sets kernel.panic:
|
||||
# - fewer than LIMIT-1 recent unclean boots → kernel.panic = PANIC_SECONDS (a crash restarts the box);
|
||||
# - LIMIT-1 or more → the guard TRIPS: kernel.panic = 0, so the LIMIT-th crash within the window
|
||||
# leaves the box OFF (operator's own words: "if it crashes 3 times within one hour, it stays off").
|
||||
# A tripped guard stays tripped across further boots until it re-arms.
|
||||
# clean-stop ExecStop of the service: writes the clean-stop marker during an orderly shutdown or reboot.
|
||||
# check hourly: a tripped guard re-arms after REARM_HOURS of normal running (since the trip AND since boot).
|
||||
# rearm the operator re-arms by hand (`felhom-crash-guard rearm`).
|
||||
# status prints the state.
|
||||
# The state is /var/lib/felhom-crash-guard/state.json (0644: the non-root agent reads it into its host report).
|
||||
# Before the service runs (very early boot) the kernel default kernel.panic = 0 applies, so a crash THAT early leaves
|
||||
# the box off — the safe side: a box that cannot reach userspace must not loop.
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
CONF = "/etc/felhom/crash-guard.conf"
|
||||
STATE_DIR = "/var/lib/felhom-crash-guard"
|
||||
DEFAULTS = {"LIMIT": 3, "WINDOW_MINUTES": 60, "PANIC_SECONDS": 10, "REARM_HOURS": 24}
|
||||
|
||||
|
||||
class Env:
|
||||
"""Paths and clock; tests replace them."""
|
||||
|
||||
def __init__(self, conf=CONF, state_dir=STATE_DIR, panic_path="/proc/sys/kernel/panic",
|
||||
uptime_path="/proc/uptime", boot_id_path="/proc/sys/kernel/random/boot_id"):
|
||||
self.conf, self.state_dir = conf, state_dir
|
||||
self.panic_path, self.uptime_path, self.boot_id_path = panic_path, uptime_path, boot_id_path
|
||||
|
||||
def now(self):
|
||||
return time.time()
|
||||
|
||||
def log(self, line):
|
||||
print(line, file=sys.stderr, flush=True)
|
||||
try:
|
||||
import subprocess
|
||||
subprocess.run(["logger", "-t", "felhom-crash-guard", line], timeout=10)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def iso(t):
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t))
|
||||
|
||||
|
||||
def parse_iso(s):
|
||||
import calendar
|
||||
return calendar.timegm(time.strptime(s, "%Y-%m-%dT%H:%M:%SZ"))
|
||||
|
||||
|
||||
def load_conf(env):
|
||||
c = dict(DEFAULTS)
|
||||
try:
|
||||
for line in open(env.conf):
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = (x.strip() for x in line.split("=", 1))
|
||||
if k in c and v.isdigit() and int(v) >= (1 if k != "PANIC_SECONDS" else 1):
|
||||
c[k] = int(v)
|
||||
except OSError:
|
||||
pass
|
||||
return c
|
||||
|
||||
|
||||
def state_path(env):
|
||||
return os.path.join(env.state_dir, "state.json")
|
||||
|
||||
|
||||
def marker_path(env):
|
||||
return os.path.join(env.state_dir, "clean-stop")
|
||||
|
||||
|
||||
def load_state(env):
|
||||
try:
|
||||
with open(state_path(env)) as f:
|
||||
s = json.load(f)
|
||||
return s if isinstance(s, dict) else None
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def save_state(env, s):
|
||||
os.makedirs(env.state_dir, mode=0o755, exist_ok=True)
|
||||
tmp = state_path(env) + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(s, f, indent=2, sort_keys=True)
|
||||
f.write("\n")
|
||||
os.chmod(tmp, 0o644)
|
||||
os.replace(tmp, state_path(env))
|
||||
|
||||
|
||||
def set_panic(env, seconds):
|
||||
with open(env.panic_path, "w") as f:
|
||||
f.write(f"{seconds}\n")
|
||||
|
||||
|
||||
def read(path, default=""):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return f.read().strip()
|
||||
except OSError:
|
||||
return default
|
||||
|
||||
|
||||
def summarize(s, c, now):
|
||||
window = c["WINDOW_MINUTES"] * 60
|
||||
times = [parse_iso(t) for t in s.get("unclean_boots", [])]
|
||||
after = parse_iso(s["rearmed_at"]) if s.get("rearmed_at") else 0
|
||||
# a re-arm starts a fresh window (or the next unclean boot would trip again at once); the history stays
|
||||
s["unclean_boots_in_window"] = sum(1 for t in times if now - t <= window and t > after)
|
||||
s["unclean_boots_24h"] = sum(1 for t in times if now - t <= 86400)
|
||||
s["config"] = c
|
||||
s["updated_at"] = iso(now)
|
||||
|
||||
|
||||
def boot(env):
|
||||
c = load_conf(env)
|
||||
now = env.now()
|
||||
try:
|
||||
up = float(read(env.uptime_path, "0").split()[0])
|
||||
except (ValueError, IndexError):
|
||||
up = 0.0
|
||||
boot_at = now - up
|
||||
prev = load_state(env)
|
||||
first = prev is None
|
||||
s = prev or {"version": 1, "unclean_boots": [], "tripped": False}
|
||||
clean = os.path.exists(marker_path(env))
|
||||
unclean = (not first) and (not clean)
|
||||
try:
|
||||
os.remove(marker_path(env))
|
||||
except OSError:
|
||||
pass
|
||||
# keep 7 days of history (the 24 h figure and the operator's view), drop older
|
||||
s["unclean_boots"] = [t for t in s.get("unclean_boots", []) if now - parse_iso(t) <= 7 * 86400]
|
||||
if unclean:
|
||||
s["unclean_boots"].append(iso(boot_at))
|
||||
s["last_boot_at"] = iso(boot_at)
|
||||
s["last_boot_unclean"] = unclean
|
||||
s["boot_id"] = read(env.boot_id_path, "unknown")
|
||||
summarize(s, c, now)
|
||||
if not s.get("tripped") and s["unclean_boots_in_window"] >= c["LIMIT"] - 1:
|
||||
s["tripped"], s["tripped_at"] = True, iso(now)
|
||||
s["tripped_reason"] = (f"{s['unclean_boots_in_window']} unclean boots within {c['WINDOW_MINUTES']} minutes — "
|
||||
f"the next crash leaves the box off (limit {c['LIMIT']})")
|
||||
env.log(f"crash-guard: TRIPPED: {s['tripped_reason']}")
|
||||
panic = 0 if s.get("tripped") else c["PANIC_SECONDS"]
|
||||
set_panic(env, panic)
|
||||
s["kernel_panic"] = panic
|
||||
s["armed"] = not s.get("tripped")
|
||||
save_state(env, s)
|
||||
env.log(f"crash-guard: boot first={first} unclean={unclean} in-window={s['unclean_boots_in_window']} "
|
||||
f"tripped={s.get('tripped')} kernel.panic={panic}")
|
||||
return 0
|
||||
|
||||
|
||||
def clean_stop(env):
|
||||
os.makedirs(env.state_dir, mode=0o755, exist_ok=True)
|
||||
with open(marker_path(env), "w") as f:
|
||||
f.write(iso(env.now()) + "\n")
|
||||
env.log("crash-guard: clean stop recorded")
|
||||
return 0
|
||||
|
||||
|
||||
def rearm(env, by):
|
||||
c = load_conf(env)
|
||||
now = env.now()
|
||||
s = load_state(env) or {"version": 1, "unclean_boots": []}
|
||||
was = bool(s.get("tripped"))
|
||||
s["tripped"] = False
|
||||
s["armed"] = True
|
||||
s["rearmed_at"], s["rearmed_by"] = iso(now), by
|
||||
if was:
|
||||
s["last_trip"] = {"at": s.get("tripped_at"), "reason": s.get("tripped_reason")}
|
||||
s.pop("tripped_at", None)
|
||||
s.pop("tripped_reason", None)
|
||||
summarize(s, c, now)
|
||||
set_panic(env, c["PANIC_SECONDS"])
|
||||
s["kernel_panic"] = c["PANIC_SECONDS"]
|
||||
save_state(env, s)
|
||||
env.log(f"crash-guard: RE-ARMED by {by} (was tripped: {was}); kernel.panic={c['PANIC_SECONDS']}")
|
||||
return 0
|
||||
|
||||
|
||||
def check(env):
|
||||
c = load_conf(env)
|
||||
now = env.now()
|
||||
s = load_state(env)
|
||||
if not s:
|
||||
return 0
|
||||
if s.get("tripped"):
|
||||
since = max(parse_iso(s["tripped_at"]), parse_iso(s.get("last_boot_at", s["tripped_at"])))
|
||||
if now - since >= c["REARM_HOURS"] * 3600:
|
||||
return rearm(env, f"timer ({c['REARM_HOURS']} h of normal running)")
|
||||
summarize(s, c, now)
|
||||
save_state(env, s)
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv, env=None):
|
||||
env = env or Env()
|
||||
cmd = argv[1] if len(argv) == 2 else ""
|
||||
if cmd == "boot":
|
||||
return boot(env)
|
||||
if cmd == "clean-stop":
|
||||
return clean_stop(env)
|
||||
if cmd == "check":
|
||||
return check(env)
|
||||
if cmd == "rearm":
|
||||
return rearm(env, "operator")
|
||||
if cmd == "status":
|
||||
print(json.dumps(load_state(env), indent=2, sort_keys=True))
|
||||
return 0
|
||||
print("usage: felhom-crash-guard boot|clean-stop|check|rearm|status", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if os.geteuid() != 0:
|
||||
print("felhom-crash-guard: must run as root", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -0,0 +1,7 @@
|
||||
# Hourly: a tripped crash guard re-arms after REARM_HOURS of normal running (felhom-crash-guard check).
|
||||
[Unit]
|
||||
Description=Felhom crash guard re-arm check
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/felhom-crash-guard check
|
||||
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Felhom crash guard re-arm check (hourly)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=15min
|
||||
OnUnitActiveSec=1h
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,19 @@
|
||||
# felhom-crash-guard — a crashed host restarts by itself, with a limit (`09` decision 88, R-851, `11` §5.9).
|
||||
# Starts early at boot (sets kernel.panic for THIS boot); its ExecStop writes the clean-stop marker during an orderly
|
||||
# shutdown or reboot. A boot that finds no marker followed a crash, a power cut or a hard reset.
|
||||
[Unit]
|
||||
Description=Felhom crash guard (restart after a kernel crash, with a limit)
|
||||
DefaultDependencies=no
|
||||
After=local-fs.target
|
||||
Before=sysinit.target shutdown.target
|
||||
Conflicts=shutdown.target
|
||||
RequiresMountsFor=/var/lib
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/sbin/felhom-crash-guard boot
|
||||
ExecStop=/usr/local/sbin/felhom-crash-guard clean-stop
|
||||
|
||||
[Install]
|
||||
WantedBy=sysinit.target
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
# /etc/grub.d/42_felhom_oneshot — the kernel lane's one-shot ENTRIES (R-836, `09` §3 decision 172, `11` §5.11).
|
||||
# Installed by the config bundle (felhom-os-apply BUNDLE_FILES), 0755 root. update-grub runs it.
|
||||
#
|
||||
# One menu entry per installed Proxmox kernel, id `felhom-oneshot-<version>`, booted ONLY when 01_felhom_oneshot found
|
||||
# the flag naming it. It is the normal entry plus option C (decision 172): softlockup_panic=1 hardlockup_panic=1
|
||||
# hung_task_panic=1 panic=10 — a lockup the kernel can detect becomes a panic, and a panic restarts the box in 10 s into
|
||||
# the default (the old kernel). A true dead freeze still needs a person (spike candidate 3 failed on all three boxes).
|
||||
# It sorts AFTER 10_linux, so it is never entry 0 and never the default.
|
||||
#
|
||||
# No vfat ESP at /boot/efi → prints nothing (no flag can name these entries).
|
||||
set -e
|
||||
prefix="/usr"
|
||||
exec_prefix="/usr"
|
||||
datarootdir="/usr/share"
|
||||
. "$datarootdir/grub/grub-mkconfig_lib"
|
||||
esp_uuid=$(findmnt -n -o UUID,FSTYPE /boot/efi 2>/dev/null | awk '$2 == "vfat" { print $1 }')
|
||||
[ -n "$esp_uuid" ] || exit 0
|
||||
kernels=$(ls /boot/vmlinuz-*-pve 2>/dev/null | sed 's#^/boot/vmlinuz-##' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-pve$' || true)
|
||||
[ -n "$kernels" ] || exit 0
|
||||
case "${GRUB_DEVICE}" in
|
||||
/dev/mapper/*|/dev/dm-*|"") root_arg="root=${GRUB_DEVICE}" ;;
|
||||
*) if [ -n "${GRUB_DEVICE_UUID}" ]; then root_arg="root=UUID=${GRUB_DEVICE_UUID}"; else root_arg="root=${GRUB_DEVICE}"; fi ;;
|
||||
esac
|
||||
[ -n "${GRUB_DEVICE}" ] || root_arg="root=$(findmnt -n -o SOURCE /)"
|
||||
rel=$(make_system_path_relative_to_its_root /boot)
|
||||
prep=$(prepare_grub_to_access_device "$(${grub_probe:-grub-probe} --target=device /boot)" | sed 's/^/ /')
|
||||
for k in $kernels; do
|
||||
[ -f "/boot/initrd.img-$k" ] || continue
|
||||
cat <<EOF
|
||||
menuentry 'Felhom one-shot: $k' --class proxmox --id felhom-oneshot-$k {
|
||||
insmod gzio
|
||||
$prep
|
||||
echo 'Loading Linux $k (felhom one-shot) ...'
|
||||
linux $rel/vmlinuz-$k $root_arg ro ${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT} softlockup_panic=1 hardlockup_panic=1 hung_task_panic=1 panic=10
|
||||
initrd $rel/initrd.img-$k
|
||||
}
|
||||
EOF
|
||||
done
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
# /etc/grub.d/01_felhom_oneshot — the kernel lane's ONE-SHOT boot (R-836, `09` §3 decisions 164 + 172, `11` §5.11).
|
||||
# Installed by the config bundle (felhom-os-apply BUNDLE_FILES), 0755 root. update-grub runs it; it prints GRUB script.
|
||||
#
|
||||
# At boot, GRUB reads `felhom_next` from an environment block on the ESP (vfat — GRUB can rewrite a file there; it
|
||||
# cannot on the LVM /boot, R-836), CLEARS it, and — only if it names an installed kernel — boots that kernel's one-shot
|
||||
# entry (42_felhom_oneshot) instead of the default. The next boot uses the default again whatever happens: a new kernel
|
||||
# that panics comes back on the old one by itself. felhom-os-apply writes the flag (mode apply) and never the default
|
||||
# for a new kernel. Measured on the Tester 1 VM, demo-felhom and demo-hp (Secure Boot on):
|
||||
# `audits/kernel-spike-2026-10-07/` (candidate 2).
|
||||
#
|
||||
# No vfat ESP at /boot/efi, or no Proxmox kernel → prints nothing (the box boots exactly as before).
|
||||
set -e
|
||||
esp_uuid=$(findmnt -n -o UUID,FSTYPE /boot/efi 2>/dev/null | awk '$2 == "vfat" { print $1 }')
|
||||
[ -n "$esp_uuid" ] || exit 0
|
||||
kernels=$(ls /boot/vmlinuz-*-pve 2>/dev/null | sed 's#^/boot/vmlinuz-##' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-pve$' || true)
|
||||
[ -n "$kernels" ] || exit 0
|
||||
cat <<EOF
|
||||
# felhom kernel lane: a one-shot kernel named on the ESP, read and cleared before the menu
|
||||
insmod part_gpt
|
||||
insmod fat
|
||||
search --no-floppy --fs-uuid --set=felhom_esp $esp_uuid
|
||||
if [ -f (\$felhom_esp)/EFI/felhom/oneshot.env ]; then
|
||||
load_env -f (\$felhom_esp)/EFI/felhom/oneshot.env felhom_next
|
||||
if [ "\${felhom_next}" ]; then
|
||||
set felhom_boot="\${felhom_next}"
|
||||
set felhom_next=
|
||||
save_env -f (\$felhom_esp)/EFI/felhom/oneshot.env felhom_next
|
||||
EOF
|
||||
for k in $kernels; do
|
||||
printf ' if [ "${felhom_boot}" = "%s" ]; then set default="felhom-oneshot-%s"; fi\n' "$k" "$k"
|
||||
done
|
||||
cat <<EOF
|
||||
fi
|
||||
fi
|
||||
EOF
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
# felhom-agent guest pre-start self-heal hook (C1 net). PVE calls: <script> <vmid> <phase>.
|
||||
/usr/local/bin/felhom-agent guest-hook "$1" "$2" || true
|
||||
exit 0
|
||||
Executable
+118
@@ -0,0 +1,118 @@
|
||||
#!/bin/bash
|
||||
# felhom-internal-place — make the one-disk box's internal data place („Belső tárhely"), felhom.eu
|
||||
# architecture/16-storage-physical-identity.md §5, §7 (sizes ruled by the operator 2026-10-16). Installed as
|
||||
# /usr/local/sbin/felhom-internal-place (config bundle). Called by felhom-host-install.sh (a new box) and by the agent's
|
||||
# signed `add_internal_place` job (an old box) — ONE implementation of the size rule and of the only mkfs it does.
|
||||
#
|
||||
# felhom-internal-place plan <planned GiB> → prints the numbers, changes nothing
|
||||
# felhom-internal-place create <planned GiB> → makes the place (or finds it), prints the result
|
||||
#
|
||||
# <planned GiB> is thin space still to be promised to the guest (rootfs + data volume) that does not exist yet:
|
||||
# the installer passes 32 + the data volume; an old box passes 0 (its guest's volumes are already counted).
|
||||
#
|
||||
# The rule (16 §7, ruled): reserve = max(16 GiB, 10 % of the pool), at most 48 GiB; internal = pool − reserve −
|
||||
# every thin volume already promised in the pool − <planned>, made only when ≥ 20 GiB. So nothing in the pool is ever
|
||||
# promised beyond pool − reserve: a pool at 100 % stops every volume on the box.
|
||||
#
|
||||
# It trusts nothing it is told. Every rule is checked here, as root, and any doubt REFUSES (exit 1):
|
||||
# - the pool is the `lvmthin: local-lvm` section of /etc/pve/storage.cfg (its vgname/thinpool), nothing else;
|
||||
# - the only volume it creates, formats or reads is <vg>/felhom-belso;
|
||||
# - it formats ONLY a felhom-belso that carries no signature at all (blkid -p finds nothing) — the one it just made,
|
||||
# or one a crash left between lvcreate and mkfs; any other content REFUSES; it never resizes or removes anything.
|
||||
# Output, one line: „created uuid=<fs-uuid> gib=<n>", „formatted uuid=… gib=…" (a crash left it empty), „existing uuid=<fs-uuid> gib=<n>", „skipped <why>" (exit 0),
|
||||
# or „plan pool=… reserve=… promised=… planned=… internal=…". Pinned by configs/test_felhom_internal_place.py.
|
||||
set -u
|
||||
STORAGE_CFG=${FELHOM_STORAGE_CFG:-/etc/pve/storage.cfg}
|
||||
LV_NAME=felhom-belso
|
||||
LABEL=felhom-belso
|
||||
MIN_GIB=20
|
||||
|
||||
die() { echo "felhom-internal-place: REFUSED: $*" >&2; exit 1; }
|
||||
|
||||
pool_of_local_lvm() { # prints "<vg> <thinpool>" of the lvmthin storage local-lvm, or nothing
|
||||
awk '
|
||||
/^[a-z]+:[ \t]/ { split($0, h, /:[ \t]+/); type = h[1]; id = h[2]; sub(/[ \t]+$/, "", id); in_s = (type == "lvmthin" && id == "local-lvm"); next }
|
||||
in_s && /^[ \t]+vgname[ \t]/ { vg = $2 }
|
||||
in_s && /^[ \t]+thinpool[ \t]/ { tp = $2 }
|
||||
END { if (vg != "" && tp != "") print vg, tp }
|
||||
' "$STORAGE_CFG"
|
||||
}
|
||||
|
||||
gib_floor() { echo $(( $1 / 1073741824 )); }
|
||||
|
||||
[[ $# -eq 2 ]] || die "usage: felhom-internal-place plan|create <planned GiB>"
|
||||
MODE=$1; PLANNED=$2
|
||||
[[ "$MODE" == plan || "$MODE" == create ]] || die "mode must be plan or create"
|
||||
[[ "$PLANNED" =~ ^(0|[1-9][0-9]{0,5})$ ]] || die "planned GiB must be a whole number"
|
||||
|
||||
read -r VG POOL < <(pool_of_local_lvm) || true
|
||||
[[ -n "${VG:-}" && -n "${POOL:-}" ]] || die "no lvmthin storage local-lvm in $STORAGE_CFG"
|
||||
[[ "$VG" =~ ^[A-Za-z0-9_.+-]+$ && "$POOL" =~ ^[A-Za-z0-9_.+-]+$ ]] || die "unexpected vg/pool name"
|
||||
|
||||
pool_b=$(lvs --noheadings --nosuffix --units b -o lv_size "$VG/$POOL" 2>/dev/null | tr -d ' ')
|
||||
[[ "$pool_b" =~ ^[0-9]+$ ]] || die "cannot read the size of $VG/$POOL"
|
||||
pool_gib=$(gib_floor "$pool_b")
|
||||
|
||||
# Every thin volume promised in the pool (its virtual size) — except the place itself and vzdump's short-lived
|
||||
# snapshots (snap_*_vzdump), which exist only during a whole-box copy and are what the reserve is for.
|
||||
# FAIL CLOSED: a failed or empty listing must never read as „nothing promised" (that would size the place too big).
|
||||
listing=$(lvs --noheadings --nosuffix --units b -o lv_name,pool_lv,lv_size "$VG" 2>/dev/null) || die "cannot list the volumes of $VG"
|
||||
pool_seen=false; promised_b=0; have_lv=false
|
||||
while read -r name pl size; do
|
||||
[[ -n "${name:-}" ]] || continue
|
||||
[[ "$name" == "$POOL" ]] && pool_seen=true
|
||||
[[ "$pl" == "$POOL" ]] || continue
|
||||
if [[ "$name" == "$LV_NAME" ]]; then have_lv=true; continue; fi
|
||||
[[ "$name" =~ ^snap_.*_vzdump$ ]] && continue
|
||||
[[ "$size" =~ ^[0-9]+$ ]] || die "cannot read the size of $VG/$name"
|
||||
promised_b=$(( promised_b + size ))
|
||||
done <<< "$listing"
|
||||
$pool_seen || die "the listing of $VG does not contain the pool $POOL — refusing to guess what is promised"
|
||||
promised_gib=$(( (promised_b + 1073741823) / 1073741824 )) # rounded UP: never promise a partial GiB twice
|
||||
|
||||
reserve=$(( pool_gib / 10 )); (( reserve < 16 )) && reserve=16; (( reserve > 48 )) && reserve=48
|
||||
internal=$(( pool_gib - reserve - promised_gib - PLANNED ))
|
||||
|
||||
if [[ "$MODE" == plan ]]; then
|
||||
echo "plan pool=$pool_gib reserve=$reserve promised=$promised_gib planned=$PLANNED internal=$internal exists=$have_lv"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
DEV=/dev/$VG/$LV_NAME
|
||||
is_block() { [[ "$(stat -L -c %F "$1" 2>/dev/null)" == "block special file" ]]; }
|
||||
fs_uuid() { blkid -p -s UUID -o value "$DEV" 2>/dev/null; }
|
||||
fs_type() { blkid -p -s TYPE -o value "$DEV" 2>/dev/null; }
|
||||
fs_label() { blkid -p -s LABEL -o value "$DEV" 2>/dev/null; }
|
||||
# has_signature: FAIL CLOSED. blkid -p exits 0 when it found something and 2 when it found NOTHING; any other result (an
|
||||
# error, an ambivalent probe, blkid missing) must never read as „empty" — that is the only gate in front of mkfs.
|
||||
has_signature() {
|
||||
local rc=0
|
||||
blkid -p -o value "$DEV" >/dev/null 2>&1 || rc=$?
|
||||
case $rc in
|
||||
0) return 0 ;;
|
||||
2) return 1 ;;
|
||||
*) die "cannot probe $DEV for a signature (blkid exit $rc) — refusing" ;;
|
||||
esac
|
||||
}
|
||||
lv_gib() { gib_floor "$(lvs --noheadings --nosuffix --units b -o lv_size "$VG/$LV_NAME" 2>/dev/null | tr -d ' ')"; }
|
||||
|
||||
if $have_lv; then
|
||||
is_block "$DEV" || die "$DEV is not an active block device"
|
||||
t=$(fs_type)
|
||||
if [[ -n "$t" ]]; then
|
||||
[[ "$t" == ext4 && "$(fs_label)" == "$LABEL" ]] || die "$DEV holds something that is not Felhom's ($t) — left alone"
|
||||
echo "existing uuid=$(fs_uuid) gib=$(lv_gib)"; exit 0
|
||||
fi
|
||||
# A felhom-belso with NO signature: a crash between lvcreate and mkfs. Nothing can be on it — format it.
|
||||
has_signature && die "$DEV carries a signature blkid cannot name — left alone"
|
||||
else
|
||||
(( internal >= MIN_GIB )) || { echo "skipped only $internal GiB free for it (pool $pool_gib, reserve $reserve, promised $promised_gib, planned $PLANNED; needs $MIN_GIB)"; exit 0; }
|
||||
lvcreate -q -y -V "${internal}G" -T "$VG/$POOL" -n "$LV_NAME" >/dev/null || die "lvcreate failed"
|
||||
is_block "$DEV" || die "$DEV did not appear after lvcreate"
|
||||
has_signature && die "the new $DEV already carries a signature — left alone"
|
||||
made=created
|
||||
fi
|
||||
mkfs.ext4 -q -L "$LABEL" "$DEV" >/dev/null || die "mkfs.ext4 failed on $DEV"
|
||||
u=$(fs_uuid)
|
||||
[[ "$u" =~ ^[0-9a-fA-F-]{36}$ ]] || die "no filesystem UUID after mkfs on $DEV"
|
||||
echo "${made:-formatted} uuid=$u gib=$(lv_gib)"
|
||||
@@ -0,0 +1,49 @@
|
||||
# felhom-agent local API — host firewall narrowing (doc 03 §6; R-50 island update 2026-07-25)
|
||||
#
|
||||
# Defense-in-depth for the per-guest local API (the controller→agent channel). The PER-GUEST BEARER
|
||||
# TOKEN + the served-leaf pin are the authorization gate; a firewall rule is only an ADDITIONAL layer
|
||||
# limiting who can even open the port.
|
||||
#
|
||||
# === R-50 ISLAND INSTALL (the default on a fresh appliance) =================================
|
||||
# The agent binds local_api.listen_addr on the HOST-INTERNAL island bridge — 169.254.253.1:8443 on
|
||||
# vmbr9, a bridge with NO physical port (bridge-ports none). That bind is the security win:
|
||||
# * Nothing listens on the LAN IP at all, so no LAN host (or off-site attacker on the LAN) can
|
||||
# reach the local API — the LAN:8443 surface is CLOSED by the bind, not by a rule.
|
||||
# * vmbr9 has no uplink, so 169.254.253.1:8443 is reachable ONLY from the one guest wired to the
|
||||
# /30 (169.254.253.2) — the controller. The portless bridge is the isolation.
|
||||
# So on an island install NO firewall rule is required for exposure; the topology provides it. If you
|
||||
# want belt-and-suspenders, restrict the port to the island bridge (it changes nothing, since nothing
|
||||
# off-bridge can route to a portless bridge anyway):
|
||||
#
|
||||
# nft add rule inet filter input iifname != "vmbr9" ip daddr 169.254.253.1 tcp dport 8443 drop
|
||||
#
|
||||
# Verify: from the guest, a TLS connect to 169.254.253.1:8443 succeeds; there is no LAN listener to
|
||||
# probe (`ss -lnt 'sport = :8443'` shows only the island IP).
|
||||
#
|
||||
# === LEGACY LAN BIND (byo, --no-island, or an explicit --bridge-ip) =========================
|
||||
# When the agent still binds a LAN bridge IP (e.g. 192.168.0.162:8443), the port is exposed to the
|
||||
# whole LAN and the subnet-narrowing rule below is worth applying. Replace the bridge IP, port, and
|
||||
# the guest bridge subnet with this host's values.
|
||||
#
|
||||
# Option A — nftables (recommended on PVE 8/9; inet filter table). Insert ABOVE any accept:
|
||||
#
|
||||
# nft add rule inet filter input ip daddr 192.168.0.162 tcp dport 8443 \
|
||||
# ip saddr != 192.168.0.0/24 drop
|
||||
# nft add rule inet filter input ip daddr 192.168.0.162 tcp dport 8443 \
|
||||
# ip saddr 192.168.0.0/24 accept
|
||||
#
|
||||
# Option B — iptables:
|
||||
#
|
||||
# iptables -A INPUT -d 192.168.0.162 -p tcp --dport 8443 -s 192.168.0.0/24 -j ACCEPT
|
||||
# iptables -A INPUT -d 192.168.0.162 -p tcp --dport 8443 -j DROP
|
||||
#
|
||||
# Option C — PVE host firewall (/etc/pve/nodes/<node>/host.fw), if the PVE firewall is enabled.
|
||||
# Add under [RULES] (and ensure the firewall is enabled in cluster.fw / host.fw):
|
||||
#
|
||||
# [RULES]
|
||||
# IN ACCEPT -source 192.168.0.0/24 -dport 8443 -proto tcp -log nolog
|
||||
# IN DROP -dport 8443 -proto tcp -log nolog
|
||||
#
|
||||
# Apply at HOST SETUP — a host-level packet-filter change, intentionally OUTSIDE the agent's
|
||||
# 3-exception privileged fence (the agent never mutates the host firewall at runtime). The token +
|
||||
# leaf-pin still gate the request regardless of which bind is in force.
|
||||
@@ -0,0 +1,19 @@
|
||||
# felhom-mgmt-watchdog.service (TASK G1) — install as /etc/systemd/system/felhom-mgmt-watchdog.service.
|
||||
#
|
||||
# LAYER 2 (auto-heal without login): a DUMB oneshot that runs /usr/local/sbin/felhom-mgmt-watchdog,
|
||||
# triggered every ~60s by felhom-mgmt-watchdog.timer. It recreates a missing /run/sshd and clears a
|
||||
# start-limited stock sshd — so a management-plane lockout self-corrects in ≤1 tick with NOBODY logged
|
||||
# in and WITHOUT the felhom-agent binary running (the agent only OBSERVES/reports; the heal must not
|
||||
# depend on it — SPIKE-felhom-sshd-2026-07-05 finding #9 / TASK G1 trap 1).
|
||||
#
|
||||
# CRITICAL: this unit MUST NOT declare `RuntimeDirectory=` — that directive (with value `sshd`) is the
|
||||
# incident it exists to fix. It writes only to /run (dir + heal marker) as root; no state dir needed.
|
||||
[Unit]
|
||||
Description=Felhom management-plane watchdog (privsep-dir auto-heal; agent-independent)
|
||||
# Ordering only — the timer drives cadence; no hard dependency so it runs even in degraded boots.
|
||||
After=systemd-tmpfiles-setup.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/felhom-mgmt-watchdog
|
||||
# Deliberately NO RuntimeDirectory=, NO PrivateTmp, NO sandboxing that would hide /run/sshd from it.
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
# felhom-mgmt-watchdog — the DUMB, agent-INDEPENDENT management-plane healer (TASK G1).
|
||||
#
|
||||
# WHY THIS EXISTS: /run/sshd is OpenSSH's compiled-in privilege-separation directory, SHARED by
|
||||
# every sshd on the host. If it goes missing, sshd's listener stays up but every new session RESETS
|
||||
# right after SSH2_MSG_KEXINIT — a management-plane lockout on an otherwise-healthy box
|
||||
# (SPIKE-felhom-sshd-2026-07-05 §8, caused live by a second unit's `RuntimeDirectory=sshd` cleanup).
|
||||
# The tmpfiles.d entry (felhom-privsep.tmpfiles) PREVENTS the known cause; this script AUTO-HEALS any
|
||||
# novel cause with ZERO login and ZERO dependency on the felhom-agent binary (so it self-corrects even
|
||||
# when the agent is down — the whole point). It is run by felhom-mgmt-watchdog.timer every ~60s.
|
||||
#
|
||||
# CONTRACT (all four are load-bearing):
|
||||
# 1. STAT-FIRST / IDEMPOTENT — a HEALTHY /run/sshd is NEVER touched (no mkdir/chmod/chown call), so
|
||||
# steady state is zero mutation and the dir's mtime is preserved. Only a MISSING or
|
||||
# wrong-mode/owner dir is corrected.
|
||||
# 2. reset-failed the STOCK sshd ONLY when it is in the `failed` state (a start-limit lockout) —
|
||||
# never otherwise, and NEVER `restart` it (mkdir alone restores it; PID stays, spike-proven).
|
||||
# 3. HEAL MARKER — writes an RFC3339 UTC timestamp to $MARKER only on a real heal, so the agent can
|
||||
# REPORT the condition to the hub (a recurring clobber must surface BEFORE it becomes a lockout).
|
||||
# 4. NO `RuntimeDirectory=` anywhere in this feature (that IS the incident cause) — the unit that
|
||||
# runs this script must not declare one either.
|
||||
#
|
||||
# Touches host /run + the stock sshd unit ONLY — no guests (pool-scoping is moot here). Fail-safe:
|
||||
# any single step failing is logged and does not abort the others (`|| true`); the next tick retries.
|
||||
|
||||
set -u
|
||||
|
||||
PRIVSEP=/run/sshd
|
||||
MARKER=/run/felhom-mgmt-watchdog.healed
|
||||
SSHD_UNIT=ssh.service # Debian/PVE stock sshd unit name (sshd.service is an alias)
|
||||
healed=0
|
||||
|
||||
log() { logger -t felhom-mgmt-watchdog "$*" 2>/dev/null || true; }
|
||||
|
||||
# 1. Privsep dir — stat-first: create only when missing; correct mode/owner only when wrong.
|
||||
if [ ! -d "$PRIVSEP" ]; then
|
||||
if mkdir -p "$PRIVSEP" && chown root:root "$PRIVSEP" && chmod 0755 "$PRIVSEP"; then
|
||||
healed=1
|
||||
log "recreated missing privsep dir $PRIVSEP (0755 root:root)"
|
||||
else
|
||||
log "ERROR: failed to recreate $PRIVSEP"
|
||||
fi
|
||||
else
|
||||
mode=$(stat -c %a "$PRIVSEP" 2>/dev/null || echo "")
|
||||
owner=$(stat -c %U:%G "$PRIVSEP" 2>/dev/null || echo "")
|
||||
if [ "$mode" != "755" ]; then
|
||||
chmod 0755 "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP mode ($mode -> 755)"
|
||||
fi
|
||||
if [ "$owner" != "root:root" ]; then
|
||||
chown root:root "$PRIVSEP" && healed=1 && log "corrected $PRIVSEP owner ($owner -> root:root)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 2. Stock sshd — clear a start-limit lockout ONLY when the unit is genuinely `failed`. Never restart
|
||||
# it (unnecessary — a recreated privsep dir is picked up by the next fork; the listener never died).
|
||||
if systemctl is-failed --quiet "$SSHD_UNIT" 2>/dev/null; then
|
||||
if systemctl reset-failed "$SSHD_UNIT" 2>/dev/null; then
|
||||
healed=1
|
||||
log "reset-failed $SSHD_UNIT (was in the failed state)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. Heal marker — only on a real heal, so the agent reports the condition (hub raises a warning).
|
||||
if [ "$healed" = "1" ]; then
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ > "$MARKER" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,18 @@
|
||||
# felhom-mgmt-watchdog.timer (TASK G1) — install as /etc/systemd/system/felhom-mgmt-watchdog.timer.
|
||||
#
|
||||
# Drives felhom-mgmt-watchdog.service on a ~60s cadence (the auto-heal tick). OnBootSec fires shortly
|
||||
# after boot (belt-and-suspenders with the tmpfiles layer); OnUnitActiveSec=60s gives the ≤1-tick
|
||||
# heal budget the G1 acceptance drill measures. Persistent=true runs a missed tick immediately after a
|
||||
# resume/late boot. The oneshot is idempotent, so a fast cadence never churns a healthy host.
|
||||
[Unit]
|
||||
Description=Felhom management-plane watchdog timer (~60s privsep-dir auto-heal tick)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitActiveSec=60s
|
||||
AccuracySec=5s
|
||||
Persistent=true
|
||||
Unit=felhom-mgmt-watchdog.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,106 @@
|
||||
#!/bin/bash
|
||||
#===============================================================================
|
||||
# felhom-mkfs-guarded — the ONLY mkfs path the felhom-agent sudoers permits (Impl-1 Part B,
|
||||
# SPIKE-drive-enrollment-2026-07-01 §SQ3). Defense-in-depth BELOW the agent: even a buggy or
|
||||
# compromised agent cannot mkfs a catastrophic target through this — it re-checks, as root, the
|
||||
# cheap catastrophic cases (OS/system disk, LVM physical volume, a foreign mount, a read-only
|
||||
# device, and any LVM/ZFS/mdraid/LUKS/swap member signature — audit D1) and refuses.
|
||||
#
|
||||
# The agent's full unclaimed-disk filter (internal/storage/claim.go) is the PRIMARY guard; this
|
||||
# wrapper is a deliberately minimal, auditable second gate. It is NOT the place for the full filter.
|
||||
#
|
||||
# Usage: felhom-mkfs-guarded <device> <fstype:ext4|xfs>
|
||||
#===============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
die() { echo "felhom-mkfs-guarded: REFUSED: $*" >&2; exit 1; }
|
||||
|
||||
dev="${1:-}"; fstype="${2:-}"
|
||||
[[ -n "$dev" && -n "$fstype" ]] || die "usage: felhom-mkfs-guarded <device> <fstype>"
|
||||
|
||||
# The device must be a REAL block-device node under /dev — no symlink (e.g. /dev/disk/by-*), no traversal.
|
||||
[[ "$dev" == /dev/* ]] || die "device must be under /dev ($dev)"
|
||||
[[ "$dev" != *..* ]] || die "path traversal ($dev)"
|
||||
[[ -b "$dev" ]] || die "not a block device ($dev)"
|
||||
[[ ! -L "$dev" ]] || die "device must be a real node, not a symlink ($dev)"
|
||||
|
||||
# Whole-disk of the target (a partition's parent, else the disk itself).
|
||||
pk="$(lsblk -ndo PKNAME "$dev" 2>/dev/null || true)"
|
||||
whole="$dev"; [[ -n "$pk" ]] && whole="/dev/$pk"
|
||||
|
||||
# 1) OS/system disk — does the target's whole-disk back /, /boot or /boot/efi?
|
||||
while read -r src mnt _rest; do
|
||||
case "$mnt" in
|
||||
/|/boot|/boot/efi)
|
||||
spk="$(lsblk -ndo PKNAME "$src" 2>/dev/null || true)"
|
||||
swhole="$src"; [[ -n "$spk" ]] && swhole="/dev/$spk"
|
||||
[[ "$swhole" == "$whole" || "$src" == "$dev" || "$src" == "$whole" ]] && die "system/OS disk ($dev backs $mnt)"
|
||||
;;
|
||||
esac
|
||||
done < /proc/mounts
|
||||
|
||||
# 2) LVM physical volume anywhere on the target disk or its partitions. pvs is resolved by ABSOLUTE
|
||||
# path (audit D1: `command -v pvs` silently skipped this check when pvs wasn't on the caller's
|
||||
# PATH); if neither candidate exists, check 5's LVM2_member FSTYPE loop still catches a PV
|
||||
# independently — pvs-absent never silently drops LVM detection.
|
||||
pvsbin=""
|
||||
for c in /usr/sbin/pvs /sbin/pvs; do
|
||||
[[ -x "$c" ]] && { pvsbin="$c"; break; }
|
||||
done
|
||||
if [[ -n "$pvsbin" ]]; then
|
||||
while read -r pv; do
|
||||
pv="${pv//[[:space:]]/}"; [[ -z "$pv" ]] && continue
|
||||
pvpk="$(lsblk -ndo PKNAME "$pv" 2>/dev/null || true)"
|
||||
pvwhole="$pv"; [[ -n "$pvpk" ]] && pvwhole="/dev/$pvpk"
|
||||
[[ "$pvwhole" == "$whole" ]] && die "device holds an LVM physical volume ($pv)"
|
||||
done < <("$pvsbin" --noheadings -o pv_name 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
# 3) mounted OUTSIDE Felhom's own drive area = a live foreign filesystem → catastrophic. Mounts under
|
||||
# /mnt/felhom-drives are our own drives (the agent detaches before a re-init) → allowed.
|
||||
# Agent v0.158.0: read from the kernel's mount table for EVERY node of the disk (lsblk's MOUNTPOINT shows one mount
|
||||
# per node; a bind elsewhere hid behind it), and a disk whose nodes cannot be listed REFUSES (it passed before).
|
||||
# Deliberately NO exemption for a sibling that is Felhom's own enrolled drive (raw /mnt/<name> + /mnt/felhom-drives):
|
||||
# the only corroboration available here is a mount the agent itself can make, so trusting it would let a compromised
|
||||
# agent unlock a host owner's disk. Consequence (felhom.eu `16` §13): the free part next to an in-use Felhom drive
|
||||
# cannot be formatted; the dashboard does not offer it.
|
||||
PROC_MOUNTS=/proc/mounts
|
||||
node_mounts() { awk -v d="$1" '$1 == d { print $2 }' "$PROC_MOUNTS"; }
|
||||
disk_nodes="$(lsblk -nrpo NAME "$whole" 2>/dev/null || true)"
|
||||
[[ -n "$disk_nodes" ]] || die "cannot list the nodes of $whole — refusing"
|
||||
while read -r node; do
|
||||
[[ -z "$node" ]] && continue
|
||||
while read -r mp; do
|
||||
[[ -z "$mp" ]] && continue
|
||||
case "$mp" in
|
||||
/mnt/felhom-drives|/mnt/felhom-drives/*) : ;;
|
||||
*) die "device (or a partition) is mounted at $mp ($node)" ;;
|
||||
esac
|
||||
done < <(node_mounts "$node")
|
||||
done <<< "$disk_nodes"
|
||||
# 4) read-only device (audit D1): a device the kernel marks RO is never a formattable data disk.
|
||||
wbase="${whole#/dev/}"
|
||||
rof="/sys/block/$wbase/ro"
|
||||
if [[ -r "$rof" ]]; then
|
||||
ro="$(cat "$rof" 2>/dev/null || true)"
|
||||
[[ "$ro" == "1" ]] && die "read-only device ($whole)"
|
||||
fi
|
||||
|
||||
# 5) member/active FSTYPEs anywhere on the target disk or its partitions (audit D1). Mirrors
|
||||
# claim.go memberFSTypes exactly: a member of LVM/ZFS/mdraid/LUKS or active-swap signature is
|
||||
# always a claim, never a plain formattable data disk. This also independently catches an LVM PV
|
||||
# when pvs is not installed (check 2's belt-and-suspenders).
|
||||
while read -r fst; do
|
||||
[[ -z "$fst" ]] && continue
|
||||
case "$fst" in
|
||||
LVM2_member|zfs_member|linux_raid_member|crypto_LUKS|swap)
|
||||
die "device holds a $fst signature ($whole)" ;;
|
||||
esac
|
||||
done < <(lsblk -nro FSTYPE "$whole" 2>/dev/null || true)
|
||||
|
||||
# Passed the catastrophic checks → format. exec so the mkfs exit status is the wrapper's.
|
||||
case "$fstype" in
|
||||
ext4) exec /usr/sbin/mkfs.ext4 -F "$dev" ;;
|
||||
xfs) exec /usr/sbin/mkfs.xfs -f "$dev" ;;
|
||||
*) die "unsupported fstype ($fstype)" ;;
|
||||
esac
|
||||
@@ -0,0 +1,19 @@
|
||||
# felhom-oob-nft.service (TASK H1) — install as /etc/systemd/system/felhom-oob-nft.service.
|
||||
#
|
||||
# Loads the STATIC felhom_oob belt table on boot (the agent then fills its sets each tick). Oneshot +
|
||||
# RemainAfterExit so `systemctl status` reflects "loaded". Independent of pve-firewall/nftables.service
|
||||
# (the table is a self-contained `inet felhom_oob`, additive — it never flushes the ruleset).
|
||||
[Unit]
|
||||
Description=Felhom OOB belt — load the static felhom_oob nft table (H1)
|
||||
After=network-pre.target
|
||||
Before=network.target felhom-agent.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/sbin/nft -f /etc/felhom-oob.nft
|
||||
# On stop, remove the table (best-effort) so an uninstall/disable leaves nft clean.
|
||||
ExecStop=-/usr/sbin/nft delete table inet felhom_oob
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/sbin/nft -f
|
||||
# felhom OOB belt — STATIC table (TASK H1). Installed by host-install; loaded on boot by
|
||||
# felhom-oob-nft.service. The agent mutates ONLY the two SETS (@operator_ips, @ssh_port) via the
|
||||
# narrow FELHOM_OOB sudoers — it NEVER touches the rules [trap 4], so the enforcement semantics are
|
||||
# fixed here and cannot be changed by the agent.
|
||||
#
|
||||
# Enforcement (scoped to iifname "wg-felhom" — the offsite tunnel — so :22 and every other interface
|
||||
# are UNTOUCHED [SF-3/OF-5]):
|
||||
# - ESTABLISHED,RELATED replies always pass (covers the PBS path + reply traffic; PMTU-safe).
|
||||
# - felhom-sshd's port is reachable from the operator /32 (in @operator_ips) over the tunnel ONLY.
|
||||
# - any other tunnel source to that port is DROPPED at the host (defense-in-depth vs the endpoint).
|
||||
# - the port is unreachable OFF the tunnel entirely.
|
||||
# Empty sets (before the agent's first sync) = the port rules match nothing = no enforcement yet
|
||||
# (felhom-sshd is not started until the agent renders its config either). priority -5 sits just above
|
||||
# the default; policy accept so this table only ever ADDS drops for the felhom-sshd port.
|
||||
#
|
||||
# Idempotent load: create-then-delete-then-define (a re-run/boot yields a clean table with EMPTY sets;
|
||||
# the agent refills them within one tick).
|
||||
|
||||
table inet felhom_oob
|
||||
delete table inet felhom_oob
|
||||
|
||||
table inet felhom_oob {
|
||||
set operator_ips {
|
||||
type ipv4_addr
|
||||
}
|
||||
set ssh_port {
|
||||
type inet_service
|
||||
}
|
||||
chain input {
|
||||
type filter hook input priority -5; policy accept;
|
||||
ct state established,related accept
|
||||
iifname "wg-felhom" tcp dport @ssh_port ip saddr @operator_ips accept
|
||||
iifname "wg-felhom" tcp dport @ssh_port drop
|
||||
tcp dport @ssh_port iifname != "wg-felhom" drop
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# felhom-op operator sudoers (TASK H1) — install as /etc/sudoers.d/felhom-op (0440 root:root).
|
||||
#
|
||||
# The DEFAULT operator identity (login via felhom-sshd only; key in /etc/felhom-sshd/authorized_keys/
|
||||
# felhom-op, outside ~/.ssh, so the customer's sshd never honours it [SF-3]). felhom-op is a normal
|
||||
# login user with NO ambient privilege — it escalates ONLY through these explicit repair verbs, so a
|
||||
# leaked operator key's blast radius is this verb set, not the box (locked decision §6). Deep repairs
|
||||
# that genuinely need root (pct, deeper systemctl) remain available via `root` as a tunnel-gated
|
||||
# break-glass account (PermitRootLogin prohibit-password on felhom-sshd), but the day-to-day identity
|
||||
# is felhom-op.
|
||||
Cmnd_Alias FELHOM_OP_REPAIR = \
|
||||
/usr/bin/mkdir -p /run/sshd, \
|
||||
/usr/bin/systemctl reset-failed wg-quick@wg-felhom, \
|
||||
/usr/bin/systemctl restart wg-quick@wg-felhom, \
|
||||
/usr/bin/systemctl start felhom-agent, \
|
||||
/usr/bin/systemctl restart felhom-agent, \
|
||||
/usr/bin/systemctl reset-failed felhom-sshd, \
|
||||
/usr/bin/systemctl restart felhom-sshd, \
|
||||
/usr/sbin/pct list, \
|
||||
/usr/sbin/pct ^start [0-9]+$, \
|
||||
/usr/sbin/pct ^stop [0-9]+$, \
|
||||
/usr/sbin/pct ^unlock [0-9]+$
|
||||
|
||||
# R-861 (b) B2 (`09` §3 decision 165, hygiene): one numeric vmid per pct verb, anchored — the old glob `[0-9]*` also
|
||||
# matched spaces, so `pct stop 9201 --skiplock 1` passed. Pinned by TestFelhomOpSudoersPctIsExact.
|
||||
felhom-op ALL=(root) NOPASSWD: FELHOM_OP_REPAIR
|
||||
Executable
+2406
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,159 @@
|
||||
#!/bin/bash
|
||||
#===============================================================================
|
||||
# felhom-pbs-apply — the ONLY storage-entry path the felhom-agent sudoers permits for the PBS DR
|
||||
# tier (slice 2; SPIKE-pbs-tier-provisioning-2026-07-10 §2b). The guarded-mkfs shape: the agent's
|
||||
# in-process validation is the PRIMARY gate (descriptor-field charset + equality checks BEFORE
|
||||
# exec); this wrapper is the minimal, auditable second gate as root.
|
||||
#
|
||||
# THE SET-ONLY LAW (spike §4, data-loss class): `pvesm` entry deletion DESTROYS the client
|
||||
# encryption key file (<id>.enc = K) — un-decryptable backups. This wrapper therefore contains
|
||||
# NO deletion path of any kind, and a re-apply is `pvesm set`-only. Grep-assertable; do not add
|
||||
# a "cleanup" verb here, ever. Deprovision is a deliberate future operator op, not this tool.
|
||||
#
|
||||
# SECRET DISCIPLINE (spike §2b): sudo logs its full argv to auth.log → the PBS token secret
|
||||
# arrives on STDIN, never as an argument to this wrapper. Inside, it is passed to pvesm's
|
||||
# --password (root-local, transient ps exposure — the accepted spike posture); it is never
|
||||
# echoed, never written anywhere except by pvesm itself (the 0600 .pw store).
|
||||
#
|
||||
# Ops (non-secret args on argv):
|
||||
# create <id> <server> <datastore> <namespace> <token-id> <fingerprint> <secret-dir>
|
||||
# secret on stdin (required). Creates the pbs entry with --encryption-key autogen
|
||||
# (K born at /etc/pve/priv/storage/<id>.enc), then places .pw/.enc copies in
|
||||
# <secret-dir> when it differs (the §4b WARN-fix dir; escrow-create's PBSEncKeyPath
|
||||
# must find K there — spike §4 flag).
|
||||
# reconcile <id> <server> <namespace> <token-id> <fingerprint> <secret-dir>
|
||||
# secret on stdin (optional; empty = no credential change). `pvesm set` ONLY:
|
||||
# server/fingerprint (+ --password when a secret is fed, e.g. after a hub re-issue).
|
||||
# NOTE datastore is deliberately NOT settable, and namespace/token-id are accepted
|
||||
# for validation parity but NOT applied — tenancy identity is adopt-only (the
|
||||
# demo's live entry must never be repointed at a different namespace).
|
||||
# read <id> <secret-dir>
|
||||
# R-39 leg (b): print the storage's token secret to STDOUT and nothing else.
|
||||
# The non-root agent WRITES this file through this wrapper but could never read it
|
||||
# back (/etc/pve/priv is 0700 root:www-data and there is no read verb), so its
|
||||
# 15-minute PBS verify loop was permanently blind to the one failure it exists to
|
||||
# catch — an `applied` tier authenticating 401. This verb is that missing read.
|
||||
# It is deliberately the narrowest thing that works: no network, no mutation, no
|
||||
# logging of the value, one file, prefix-asserted under the given secret dir.
|
||||
#
|
||||
# grant <id>
|
||||
# The Part-0-evidenced dual-grant: FelhomAgentStore on /storage/<id> to the agent
|
||||
# user AND token (privsep intersection). Datastore.Audit reads ride the base role.
|
||||
#===============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
die() { echo "felhom-pbs-apply: REFUSED: $*" >&2; exit 1; }
|
||||
|
||||
op="${1:-}"; id="${2:-}"
|
||||
[[ -n "$op" && -n "$id" ]] || die "usage: felhom-pbs-apply <create|reconcile|grant|read> <storage-id> ..."
|
||||
|
||||
# Storage id: PVE grammar, conservative. Also the ACL path component — no slashes possible.
|
||||
[[ "$id" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,27}$ ]] || die "bad storage id ($id)"
|
||||
|
||||
STORECFG=/etc/pve/storage.cfg
|
||||
PRIVDIR=/etc/pve/priv/storage
|
||||
|
||||
entry_exists() { grep -Eq "^pbs: ${id}\$" "$STORECFG"; }
|
||||
|
||||
val_server() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9.-]{0,62}$ ]] || die "bad server ($1)"; }
|
||||
val_datastore() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,31}$ ]] || die "bad datastore ($1)"; }
|
||||
val_ns() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]{0,31}$ ]] || die "bad namespace ($1)"; }
|
||||
val_tok() { [[ "$1" =~ ^[A-Za-z0-9_.-]+@[A-Za-z0-9]+![A-Za-z0-9_.-]+$ ]] || die "bad token id ($1)"; }
|
||||
val_fp() { [[ "$1" =~ ^([A-Fa-f0-9]{2}:){31}[A-Fa-f0-9]{2}$ ]] || die "bad fingerprint"; }
|
||||
val_sdir() {
|
||||
case "$1" in
|
||||
/etc/pve/priv/storage) : ;;
|
||||
/var/lib/felhom-agent/*) [[ "$1" != *..* ]] || die "secret-dir traversal" ;;
|
||||
*) die "secret-dir must be $PRIVDIR or under /var/lib/felhom-agent ($1)" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
read_secret() { # → SECRET (may be empty when optional)
|
||||
local s
|
||||
s=$(head -c 256 || true)
|
||||
s="${s%$'\n'}"; s="${s%$'\r'}"
|
||||
printf '%s' "$s"
|
||||
}
|
||||
|
||||
place_copies() { # secret-dir — the §4b WARN-fix placement (non-root agent can't read /etc/pve/priv)
|
||||
local sdir="$1"
|
||||
[[ "$sdir" == "$PRIVDIR" ]] && return 0
|
||||
install -d -o felhom-agent -g felhom-agent -m 0700 "$sdir"
|
||||
[[ -f "$PRIVDIR/$id.pw" ]] && install -o felhom-agent -g felhom-agent -m 0600 "$PRIVDIR/$id.pw" "$sdir/$id.pw"
|
||||
# K's copy: escrow-create stats PBSEncKeyPath(<secret-dir>/<id>.enc) — the spike §4 flag.
|
||||
[[ -f "$PRIVDIR/$id.enc" ]] && install -o root -g felhom-agent -m 0640 "$PRIVDIR/$id.enc" "$sdir/$id.enc"
|
||||
return 0
|
||||
}
|
||||
|
||||
case "$op" in
|
||||
create)
|
||||
[[ $# -eq 8 ]] || die "create needs 7 args: <id> <server> <datastore> <namespace> <token-id> <fingerprint> <secret-dir>"
|
||||
server="$3"; datastore="$4"; ns="$5"; tok="$6"; fp="$7"; sdir="$8"
|
||||
val_server "$server"; val_datastore "$datastore"; val_ns "$ns"; val_tok "$tok"; val_fp "$fp"; val_sdir "$sdir"
|
||||
entry_exists && die "entry $id already exists (reconcile is the re-apply path — set-only law)"
|
||||
SECRET=$(read_secret)
|
||||
[[ -n "$SECRET" ]] || die "create requires the token secret on stdin"
|
||||
pvesm add pbs "$id" \
|
||||
--server "$server" --datastore "$datastore" --namespace "$ns" \
|
||||
--username "$tok" --password "$SECRET" --fingerprint "$fp" \
|
||||
--content backup --encryption-key autogen >&2
|
||||
[[ -f "$PRIVDIR/$id.enc" ]] || die "pvesm add succeeded but K ($PRIVDIR/$id.enc) was not born"
|
||||
place_copies "$sdir"
|
||||
echo "felhom-pbs-apply: created $id (K born; encryption-key autogen)" >&2
|
||||
;;
|
||||
reconcile)
|
||||
[[ $# -eq 7 ]] || die "reconcile needs 6 args: <id> <server> <namespace> <token-id> <fingerprint> <secret-dir>"
|
||||
server="$3"; ns="$4"; tok="$5"; fp="$6"; sdir="$7"
|
||||
val_server "$server"; val_ns "$ns"; val_tok "$tok"; val_fp "$fp"; val_sdir "$sdir"
|
||||
entry_exists || die "entry $id does not exist (create is the fresh path)"
|
||||
SECRET=$(read_secret)
|
||||
# R-39 (2026-07-18): NEVER pass --server to `pvesm set`. PVE treats `server` as a CREATE-ONLY
|
||||
# parameter and rejects the ENTIRE call — "can't change value of fixed parameter 'server'" —
|
||||
# even when the value is byte-identical to the stored one. That made every reconcile exit 255,
|
||||
# so each hub-re-issued one-time secret was consumed-then-burned and the tier stayed pinned to
|
||||
# a revoked credential (401 forever). Proven live on the N100 demo host: `pvesm set <id>
|
||||
# --server <same> --fingerprint <same>` -> rejected; the same call without --server -> rc 0.
|
||||
# The server address is immutable by construction (relocating a PBS endpoint needs a fresh
|
||||
# create), so there is nothing here to reconcile. Guarded by
|
||||
# TestReconcileNeverPassesServerToPvesmSet.
|
||||
args=(--fingerprint "$fp")
|
||||
[[ -n "$SECRET" ]] && args+=(--password "$SECRET")
|
||||
pvesm set "$id" "${args[@]}" >&2
|
||||
place_copies "$sdir"
|
||||
echo "felhom-pbs-apply: reconciled $id (set-only; tenancy identity untouched)" >&2
|
||||
;;
|
||||
read)
|
||||
# R-39(b): the missing read path. Prints the secret to STDOUT, nothing else — no stderr note (it
|
||||
# would be the only verb whose success line could be confused with the value), no mutation.
|
||||
#
|
||||
# Traversal is refused three times over, because this is the one verb that EXFILTRATES a file and
|
||||
# its argv is attacker-shaped if the agent is ever compromised:
|
||||
# 1. `id` already matched ^[A-Za-z][A-Za-z0-9_.-]{0,27}$ at the top — it cannot start with a dot
|
||||
# and cannot contain a slash, so "../../etc/shadow" never reaches here;
|
||||
# 2. val_sdir pins the directory to PRIVDIR or under /var/lib/felhom-agent, rejecting "..";
|
||||
# 3. the RESOLVED path is prefix-asserted under that directory below, so even a future change to
|
||||
# either grammar cannot walk out.
|
||||
[[ $# -eq 3 ]] || die "read needs 2 args: <id> <secret-dir>"
|
||||
sdir="$3"
|
||||
val_sdir "$sdir"
|
||||
target="$sdir/$id.pw"
|
||||
# Belt: resolve and re-check the prefix (guards a symlinked <id>.pw pointing outside the dir).
|
||||
resolved=$(readlink -f -- "$target" 2>/dev/null || true)
|
||||
[[ -n "$resolved" ]] || die "secret file not found ($target)"
|
||||
case "$resolved" in
|
||||
"$sdir"/*) : ;;
|
||||
*) die "resolved secret path escapes the secret dir" ;;
|
||||
esac
|
||||
[[ -f "$resolved" ]] || die "secret file not found ($target)"
|
||||
cat -- "$resolved"
|
||||
;;
|
||||
grant)
|
||||
[[ $# -eq 2 ]] || die "grant takes only <id>"
|
||||
pveum acl modify "/storage/$id" --users felhom-agent@pve --roles FelhomAgentStore >&2
|
||||
pveum acl modify "/storage/$id" --tokens 'felhom-agent@pve!agent' --roles FelhomAgentStore >&2
|
||||
echo "felhom-pbs-apply: granted FelhomAgentStore on /storage/$id (user + token)" >&2
|
||||
;;
|
||||
*)
|
||||
die "unknown op ($op)"
|
||||
;;
|
||||
esac
|
||||
Executable
+546
@@ -0,0 +1,546 @@
|
||||
#!/usr/bin/python3
|
||||
"""felhom-priv-apply — the ROOT half of every file the agent writes into a root-read place (R-861, `03` §3.1).
|
||||
|
||||
Install as /usr/local/sbin/felhom-priv-apply (0755 root:root) — it rides the signed config bundle (R-840).
|
||||
|
||||
WHY IT EXISTS. Until agent v0.146.0 the agent's sudoers let it `install` a file it had written itself into a place a
|
||||
root program reads: a systemd .mount unit (a bind mount of an agent-owned directory over /etc/sudoers.d is a root
|
||||
shell), a dnsmasq drop-in (`dhcp-script=` runs as root), the WireGuard config (`PostUp=` runs as root) and the OOB
|
||||
sshd config (`AuthorizedKeysFile` + `StrictModes no`). A compromised agent PROCESS was therefore root on its host.
|
||||
Now the agent stages the file and this wrapper — root-owned, delivered only by an operator-signed bundle — checks
|
||||
the CONTENT against the exact grammar the agent's own renderers produce, and refuses anything else. The agent can no
|
||||
longer name the destination: each verb has a fixed source and a fixed (or strictly named) destination.
|
||||
|
||||
Verbs (each one sudoers line, exact-match pattern):
|
||||
unit <name> /var/lib/felhom-agent/units/<name> -> /etc/systemd/system/<name> (.mount | .automount)
|
||||
unit-forget <name> REMOVES /etc/systemd/system/<name> (R-1124): only a felhom by-UUID drive .mount unit that is
|
||||
disabled, not mounted, and whose drive is absent (its /dev/disk/by-uuid link is gone)
|
||||
dnsmasq <tmp> <name> /tmp/felhom-resolver-<digits>.conf -> /etc/dnsmasq.d/felhom-<...>.conf
|
||||
wg /var/lib/felhom-agent/wg/wg-felhom.conf -> /etc/wireguard/wg-felhom.conf (0600)
|
||||
sshd-config /var/lib/felhom-agent/felhom-sshd/sshd_config -> /etc/felhom-sshd/sshd_config
|
||||
sshd-key /var/lib/felhom-agent/felhom-sshd/authorized_keys.felhom-op -> /etc/felhom-sshd/authorized_keys/felhom-op
|
||||
controller-image <vmid> the ref on STDIN -> /etc/felhom-controller-image INSIDE guest <vmid> (R-861 (a) A1): only
|
||||
our registry + our repository + an x.y.z tag; the agent no longer has a `tee` grant
|
||||
--self-check prints "felhom-priv-apply ok verbs=..." (the bundle's self-check)
|
||||
|
||||
Exit codes: 0 installed (or already identical), 2 usage, 3 refused (content or source), 4 install failed.
|
||||
Every refusal is logged to the journal (tag felhom-priv-apply) with its rule; file CONTENT is never logged.
|
||||
Pinned by configs/test_felhom_priv_apply.py (one test per rule, red-proofs in the R-861 audit).
|
||||
"""
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
AGENT_USER = "felhom-agent"
|
||||
STATE = "/var/lib/felhom-agent"
|
||||
UNITS_SRC = STATE + "/units"
|
||||
UNIT_DIR = "/etc/systemd/system"
|
||||
DNSMASQ_DIR = "/etc/dnsmasq.d"
|
||||
WG_SRC, WG_DEST = STATE + "/wg/wg-felhom.conf", "/etc/wireguard/wg-felhom.conf"
|
||||
SSHD_SRC, SSHD_DEST = STATE + "/felhom-sshd/sshd_config", "/etc/felhom-sshd/sshd_config"
|
||||
KEY_SRC, KEY_DEST = STATE + "/felhom-sshd/authorized_keys.felhom-op", "/etc/felhom-sshd/authorized_keys/felhom-op"
|
||||
MAX_BYTES = 64 * 1024
|
||||
VERBS = ("unit", "unit-forget", "dnsmasq", "wg", "sshd-config", "sshd-key", "controller-image")
|
||||
# R-861 (a) A1 (`09` §3 decision 165): the SAME pattern as the agent's controllerImageRe (internal/localapi/
|
||||
# controllerswap.go) — a compromised agent cannot hand the guest's bootstrap any other image. Pinned by
|
||||
# configs/test_felhom_priv_apply.py ControllerImage.
|
||||
CONTROLLER_IMAGE_RE = re.compile(r"^gitea\.dooplex\.hu/admin/felhom-controller:[0-9]+\.[0-9]+\.[0-9]+$")
|
||||
CONTROLLER_IMAGE_FILE = "/etc/felhom-controller-image"
|
||||
CONTROLLER_IMAGE_MAX = 256
|
||||
VMID_RE = re.compile(r"^[0-9]{1,9}$")
|
||||
|
||||
UNIT_NAME_RE = re.compile(r"^mnt-[A-Za-z0-9_.\\-]+\.(mount|automount)$")
|
||||
DRIVE_UNIT_NAME_RE = re.compile(r"^mnt-[A-Za-z0-9_.\\-]+\.mount$")
|
||||
DRIVE_UNIT_MARKER = "Managed by felhom-agent"
|
||||
NET_UNIT_MARKER = "felhom network storage"
|
||||
WANTS_DIR = UNIT_DIR + "/multi-user.target.wants"
|
||||
BY_UUID_DIR = "/dev/disk/by-uuid"
|
||||
DNSMASQ_TMP_RE = re.compile(r"^/tmp/felhom-resolver-[0-9]+\.conf$")
|
||||
DNSMASQ_NAME_RE = re.compile(r"^felhom-[a-z0-9][a-z0-9._-]*\.conf$")
|
||||
SEG = r"[A-Za-z0-9_-][A-Za-z0-9_.-]*"
|
||||
WHERE_RE = re.compile(r"^/mnt/(felhom-drives/)?" + SEG + r"$")
|
||||
UUID_RE = re.compile(r"^[A-Fa-f0-9]{4,}(-[A-Fa-f0-9]+){0,4}$")
|
||||
HOST_RE = re.compile(r"^[A-Za-z0-9._:-]{1,255}$")
|
||||
NET_PATH_RE = re.compile(r"^[A-Za-z0-9._/@+-]{1,512}$")
|
||||
OPT_RE = re.compile(r"^[A-Za-z0-9_.:/@+-]+(=[A-Za-z0-9_.:/@+-]+)?$")
|
||||
LOCAL_TYPES = {"ext4", "xfs", "btrfs", "exfat", "vfat", "ntfs3", "ntfs"}
|
||||
NET_TYPES = {"nfs", "nfs4", "cifs"}
|
||||
# Options that turn a device mount into something else, or let set-uid/device files act on the host.
|
||||
FORBIDDEN_OPTS = {"bind", "rbind", "move", "rmove", "remount", "suid", "dev", "user", "users", "owner", "group",
|
||||
"x-mount.mkdir", "helper"}
|
||||
DESC_RE = re.compile(r"^[^\x00-\x1f\x7f]{0,200}$")
|
||||
WG_KEY_RE = re.compile(r"^[A-Za-z0-9+/]{42}[AEIMQUYcgkosw480]=$")
|
||||
KEY_LINE_RE = re.compile(r"^(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp(256|384|521)|sk-ssh-ed25519@openssh\.com) "
|
||||
r"[A-Za-z0-9+/]+={0,3}( [ -~]{0,200})?$")
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
def __init__(self, rule, reason):
|
||||
super().__init__(reason)
|
||||
self.rule, self.reason = rule, reason
|
||||
|
||||
|
||||
class Host:
|
||||
"""Every filesystem / process effect, so the tests can play the box in memory."""
|
||||
|
||||
def agent_uid(self):
|
||||
import pwd
|
||||
return pwd.getpwnam(AGENT_USER).pw_uid
|
||||
|
||||
def read_source(self, path):
|
||||
"""The staged file: a REGULAR file owned by the agent, never a symlink, at most MAX_BYTES."""
|
||||
try:
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
except OSError as e:
|
||||
raise Refused("P1", f"cannot open the staged file {path}: {e.strerror}")
|
||||
try:
|
||||
st = os.fstat(fd)
|
||||
if not stat.S_ISREG(st.st_mode):
|
||||
raise Refused("P1", f"{path} is not a regular file")
|
||||
if st.st_uid != self.agent_uid():
|
||||
raise Refused("P1", f"{path} is not owned by {AGENT_USER}")
|
||||
if st.st_size > MAX_BYTES:
|
||||
raise Refused("P1", f"{path} is larger than {MAX_BYTES} bytes")
|
||||
with os.fdopen(fd, "rb") as f:
|
||||
fd = -1
|
||||
return f.read(MAX_BYTES + 1)
|
||||
finally:
|
||||
if fd >= 0:
|
||||
os.close(fd)
|
||||
|
||||
def read_dest(self, path):
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
def install(self, dest, data, mode):
|
||||
"""Atomic, root-owned: a temp file beside the destination, fsync, rename."""
|
||||
d = os.path.dirname(dest)
|
||||
os.makedirs(d, mode=0o755, exist_ok=True)
|
||||
tmp = os.path.join(d, f".{os.path.basename(dest)}.felhom-new.{os.getpid()}")
|
||||
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(data)
|
||||
f.flush()
|
||||
os.fchown(f.fileno(), 0, 0)
|
||||
os.fchmod(f.fileno(), mode)
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp, dest)
|
||||
except BaseException:
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def read_stdin(self, limit):
|
||||
return sys.stdin.buffer.read(limit + 1)
|
||||
|
||||
# ---- unit-forget (R-1124) ----
|
||||
def read_installed(self, path):
|
||||
"""The installed unit: None when absent; Refused when it is not a regular file (a symlink is never followed)."""
|
||||
try:
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except OSError as e:
|
||||
raise Refused("F1", f"cannot open {path}: {e.strerror}")
|
||||
with os.fdopen(fd, "rb") as f:
|
||||
if not stat.S_ISREG(os.fstat(f.fileno()).st_mode):
|
||||
raise Refused("F1", f"{path} is not a regular file")
|
||||
return f.read(MAX_BYTES + 1)
|
||||
|
||||
def lexists(self, path):
|
||||
return os.path.lexists(path)
|
||||
|
||||
def mountpoints(self):
|
||||
"""Every mount point in /proc/self/mountinfo (field 5, octal escapes decoded)."""
|
||||
out = set()
|
||||
with open("/proc/self/mountinfo", "r", encoding="utf-8", errors="replace") as f:
|
||||
for line in f:
|
||||
parts = line.split(" ")
|
||||
if len(parts) > 4:
|
||||
out.add(re.sub(r"\\([0-7]{3})", lambda m: chr(int(m.group(1), 8)), parts[4]))
|
||||
return out
|
||||
|
||||
def remove(self, path):
|
||||
os.remove(path)
|
||||
|
||||
def write_guest_image(self, vmid, data):
|
||||
"""As root: `pct exec <vmid> -- tee <the fixed file>` with the checked ref on stdin (no shell)."""
|
||||
r = subprocess.run(["/usr/sbin/pct", "exec", str(vmid), "--", "tee", CONTROLLER_IMAGE_FILE],
|
||||
input=data, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, timeout=60)
|
||||
if r.returncode != 0:
|
||||
raise OSError(f"pct exec {vmid} tee exited {r.returncode}: {r.stderr.decode(errors='replace').strip()[:200]}")
|
||||
|
||||
def log(self, line):
|
||||
print(line, file=sys.stderr)
|
||||
try:
|
||||
subprocess.run(["logger", "-t", "felhom-priv-apply", "--", line], timeout=10, check=False)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
pass
|
||||
|
||||
|
||||
def systemd_escape_path(path):
|
||||
"""`systemd-escape --path`: strip the slashes at both ends, `/` -> `-`, every byte outside [A-Za-z0-9:_.] (and a
|
||||
leading `.`) -> `\\xNN`."""
|
||||
p = path.strip("/")
|
||||
out = []
|
||||
for i, ch in enumerate(p):
|
||||
if ch == "/":
|
||||
out.append("-")
|
||||
elif (ch.isascii() and (ch.isalnum() or ch in ":_.")) and not (i == 0 and ch == "."):
|
||||
out.append(ch)
|
||||
else:
|
||||
out.extend("\\x%02x" % b for b in ch.encode())
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def text_of(data, what):
|
||||
if len(data) > MAX_BYTES:
|
||||
raise Refused("P1", f"{what} is too large")
|
||||
try:
|
||||
text = data.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
raise Refused("P2", f"{what} is not UTF-8 text")
|
||||
if "\x00" in text or "\r" in text:
|
||||
raise Refused("P2", f"{what} carries a NUL or CR byte")
|
||||
return text
|
||||
|
||||
|
||||
def parse_ini(text, what):
|
||||
"""[Section] / Key=Value / comments / blank lines. A key outside a section or a repeated key is refused."""
|
||||
sections, cur = {}, None
|
||||
for n, raw in enumerate(text.split("\n"), 1):
|
||||
line = raw.strip()
|
||||
if line.endswith("\\"):
|
||||
# systemd joins a line ending in a backslash with the next one; this parser does not. Refused, so the two
|
||||
# can never read the same bytes differently (review 2026-10-05).
|
||||
raise Refused("U2", f"{what}: line {n} ends with a backslash (a continuation)")
|
||||
if not line or line.startswith("#") or line.startswith(";"):
|
||||
continue
|
||||
m = re.match(r"^\[([A-Za-z]+)\]$", line)
|
||||
if m:
|
||||
cur = m.group(1)
|
||||
if cur in sections:
|
||||
raise Refused("U2", f"{what}: section [{cur}] twice")
|
||||
sections[cur] = {}
|
||||
continue
|
||||
if cur is None or "=" not in line:
|
||||
raise Refused("U2", f"{what}: line {n} is not Key=Value inside a section")
|
||||
k, v = line.split("=", 1)
|
||||
k, v = k.strip(), v.strip()
|
||||
if k in sections[cur]:
|
||||
raise Refused("U2", f"{what}: {cur}.{k} given twice")
|
||||
sections[cur][k] = v
|
||||
return sections
|
||||
|
||||
|
||||
# ---------- the unit verb ----------
|
||||
def check_unit(name, text):
|
||||
if not UNIT_NAME_RE.match(name) or "/" in name:
|
||||
raise Refused("U1", f"unit name {name!r} is not mnt-<escaped path>.mount|.automount")
|
||||
kind = "automount" if name.endswith(".automount") else "mount"
|
||||
s = parse_ini(text, name)
|
||||
body = "Automount" if kind == "automount" else "Mount"
|
||||
# [Unit] holds ONLY what the renderers write: Description, and After=local-fs-pre.target on a local mount. A
|
||||
# Wants=/Requires=/Before= naming any unit would start it with the mount (Wants=reboot.target — found by review
|
||||
# 2026-10-05), so none of them is accepted.
|
||||
allowed = {"Unit": {"Description", "After"},
|
||||
body: {"Where", "TimeoutIdleSec"} if kind == "automount" else {"What", "Where", "Type", "Options"},
|
||||
"Install": {"WantedBy"}}
|
||||
for sec, keys in s.items():
|
||||
if sec not in allowed:
|
||||
raise Refused("U2", f"{name}: section [{sec}] is not allowed")
|
||||
bad = set(keys) - allowed[sec]
|
||||
if bad:
|
||||
raise Refused("U2", f"{name}: [{sec}] key(s) {sorted(bad)} not allowed")
|
||||
u = s.get("Unit", {})
|
||||
if not DESC_RE.match(u.get("Description", "")):
|
||||
raise Refused("U2", f"{name}: Description has control characters")
|
||||
if "After" in u and u["After"] != "local-fs-pre.target":
|
||||
raise Refused("U2", f"{name}: After= may only be local-fs-pre.target")
|
||||
inst = s.get("Install", {})
|
||||
if inst and inst.get("WantedBy") != "multi-user.target":
|
||||
raise Refused("U2", f"{name}: WantedBy must be multi-user.target")
|
||||
m = s.get(body)
|
||||
if not m or "Where" not in m:
|
||||
raise Refused("U3", f"{name}: no [{body}] Where=")
|
||||
where = m["Where"]
|
||||
if not WHERE_RE.match(where):
|
||||
raise Refused("U3", f"{name}: Where={where} is not /mnt/<name> or /mnt/felhom-drives/<name>")
|
||||
if systemd_escape_path(where) + "." + kind != name:
|
||||
raise Refused("U3", f"{name}: the unit name does not match Where={where}")
|
||||
if kind == "automount":
|
||||
t = m.get("TimeoutIdleSec", "")
|
||||
if t and not re.match(r"^[0-9]{1,6}$", t):
|
||||
raise Refused("U2", f"{name}: TimeoutIdleSec must be seconds")
|
||||
return
|
||||
what, typ = m.get("What", ""), m.get("Type", "")
|
||||
opts = [o for o in m.get("Options", "").split(",") if o]
|
||||
net = False
|
||||
mu = re.match(r"^/dev/disk/by-uuid/(.+)$", what)
|
||||
if mu:
|
||||
if not UUID_RE.match(mu.group(1)):
|
||||
raise Refused("U4", f"{name}: What= is not a filesystem UUID")
|
||||
if typ and typ not in LOCAL_TYPES:
|
||||
raise Refused("U4", f"{name}: Type={typ} is not a local filesystem")
|
||||
else:
|
||||
net = True
|
||||
if typ not in NET_TYPES:
|
||||
raise Refused("U4", f"{name}: What= is neither /dev/disk/by-uuid/<uuid> nor a network source with Type=nfs/nfs4/cifs")
|
||||
if typ == "cifs":
|
||||
mm = re.match(r"^//([^/]+)/(.+)$", what)
|
||||
else:
|
||||
mm = re.match(r"^([^/:][^:]*):(/.*)$", what)
|
||||
if not mm or not HOST_RE.match(mm.group(1)) or not NET_PATH_RE.match(mm.group(2)) or ".." in mm.group(2).split("/"):
|
||||
raise Refused("U4", f"{name}: What= is not a clean {typ} source")
|
||||
if not where.startswith("/mnt/felhom-drives/"):
|
||||
raise Refused("U3", f"{name}: a network share mounts only under /mnt/felhom-drives/")
|
||||
for o in opts:
|
||||
if not OPT_RE.match(o):
|
||||
raise Refused("U5", f"{name}: mount option {o!r} has characters a mount option never needs")
|
||||
if o.split("=", 1)[0].lower() in FORBIDDEN_OPTS or o.lower().startswith("x-mount."):
|
||||
raise Refused("U5", f"{name}: mount option {o.split('=', 1)[0]!r} is not allowed")
|
||||
if net and not {"nosuid", "nodev"} <= set(opts):
|
||||
# A network server is outside the box: a set-uid file on it must never run as root here.
|
||||
raise Refused("U5", f"{name}: a network share must carry nosuid,nodev")
|
||||
|
||||
|
||||
# ---------- dnsmasq ----------
|
||||
def _ip(v, v6=True):
|
||||
try:
|
||||
a = ipaddress.ip_address(v)
|
||||
except ValueError:
|
||||
return False
|
||||
return v6 or a.version == 4
|
||||
|
||||
|
||||
DOMAIN_RE = re.compile(r"^[A-Za-z0-9]([A-Za-z0-9-]{0,62})(\.[A-Za-z0-9]([A-Za-z0-9-]{0,62}))*$")
|
||||
|
||||
|
||||
def check_dnsmasq(text):
|
||||
for n, raw in enumerate(text.split("\n"), 1):
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if line in ("bind-interfaces", "no-resolv"):
|
||||
continue
|
||||
k, _, v = line.partition("=")
|
||||
if k == "listen-address" and _ip(v, v6=False):
|
||||
continue
|
||||
if k == "server" and (_ip(v) or (v.count("#") == 1 and _ip(v.split("#")[0]) and v.split("#")[1].isdigit())):
|
||||
continue
|
||||
m = re.match(r"^/([^/]+)/$", v)
|
||||
if k == "local" and m and DOMAIN_RE.match(m.group(1)):
|
||||
continue
|
||||
m = re.match(r"^/([^/]+)/([^/]+)$", v)
|
||||
if k == "address" and m and DOMAIN_RE.match(m.group(1)) and _ip(m.group(2), v6=False):
|
||||
continue
|
||||
raise Refused("D1", f"dnsmasq line {n} ({k or line[:20]!r}) is not one the resolver writes")
|
||||
|
||||
|
||||
# ---------- WireGuard ----------
|
||||
def check_wg(text):
|
||||
s = parse_ini(text, "wg-felhom.conf")
|
||||
if set(s) != {"Interface", "Peer"}:
|
||||
raise Refused("W1", "wg-felhom.conf must hold exactly [Interface] and [Peer]")
|
||||
i, p = s["Interface"], s["Peer"]
|
||||
if set(i) - {"PrivateKey", "Address", "MTU"} or set(p) - {"PublicKey", "Endpoint", "AllowedIPs", "PersistentKeepalive"}:
|
||||
raise Refused("W1", "wg-felhom.conf carries a key the agent never writes (PostUp/PreUp/... run as root)")
|
||||
if not WG_KEY_RE.match(i.get("PrivateKey", "")) or not WG_KEY_RE.match(p.get("PublicKey", "")):
|
||||
raise Refused("W2", "a WireGuard key is not 32 bytes of base64")
|
||||
try:
|
||||
a = ipaddress.ip_network(i.get("Address", ""), strict=False)
|
||||
if a.version != 4 or a.prefixlen != 32:
|
||||
raise ValueError
|
||||
if not (1280 <= int(i.get("MTU", "1280")) <= 1500):
|
||||
raise ValueError
|
||||
host, _, port = p.get("Endpoint", "").rpartition(":")
|
||||
if ipaddress.ip_address(host).version != 4 or not (1 <= int(port) <= 65535):
|
||||
raise ValueError
|
||||
for n in p.get("AllowedIPs", "").split(","):
|
||||
if ipaddress.ip_network(n.strip(), strict=True).prefixlen != 32:
|
||||
raise ValueError
|
||||
if not (0 <= int(p.get("PersistentKeepalive", "25")) <= 3600):
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise Refused("W2", "an Address/MTU/Endpoint/AllowedIPs/PersistentKeepalive value is not what the agent renders")
|
||||
|
||||
|
||||
# ---------- OOB sshd ----------
|
||||
def render_sshd(port):
|
||||
"""Byte-identical to felhomsshd.renderConfig (internal/felhomsshd/config.go) — pinned by a Go test."""
|
||||
return ("# felhom OOB sshd — agent-managed (H1); DO NOT EDIT\n"
|
||||
f"Port {port}\n"
|
||||
"ListenAddress 0.0.0.0\n"
|
||||
"ListenAddress ::\n"
|
||||
"HostKey /etc/felhom-sshd/ssh_host_ed25519_key\n"
|
||||
"PidFile /run/felhom-sshd.pid\n"
|
||||
"AuthorizedKeysFile /etc/felhom-sshd/authorized_keys/%u\n"
|
||||
"PasswordAuthentication no\n"
|
||||
"PermitRootLogin prohibit-password\n"
|
||||
"PubkeyAuthentication yes\n"
|
||||
"KbdInteractiveAuthentication no\n"
|
||||
"UsePAM yes\n"
|
||||
"AllowUsers root felhom-op\n"
|
||||
"X11Forwarding no\n"
|
||||
"Subsystem sftp internal-sftp\n")
|
||||
|
||||
|
||||
def check_sshd(text):
|
||||
m = re.search(r"^Port ([0-9]{1,5})$", text, re.M)
|
||||
if not m or not (1 <= int(m.group(1)) <= 65535) or int(m.group(1)) == 22:
|
||||
raise Refused("S1", "sshd_config has no Port (or claims :22, the household's sshd)")
|
||||
if text != render_sshd(int(m.group(1))):
|
||||
raise Refused("S1", "sshd_config differs from the one fixed template (only the Port may vary)")
|
||||
|
||||
|
||||
def check_key(text):
|
||||
lines = [l for l in text.split("\n") if l.strip()]
|
||||
if len(lines) > 1:
|
||||
raise Refused("S2", "felhom-op's authorized_keys holds more than one key")
|
||||
if lines and not KEY_LINE_RE.match(lines[0]):
|
||||
raise Refused("S2", "the key line is not a plain public key (no options such as command= or from=)")
|
||||
|
||||
|
||||
# ---------- main ----------
|
||||
def plan(argv):
|
||||
"""(verb, source, dest, mode, checker) for an argv, or Refused("A1")."""
|
||||
if not argv or argv[0] not in VERBS:
|
||||
raise Refused("A1", "usage: felhom-priv-apply unit <name> | dnsmasq <tmp> <name> | wg | sshd-config | sshd-key")
|
||||
v, rest = argv[0], argv[1:]
|
||||
if v == "unit" and len(rest) == 1:
|
||||
if not UNIT_NAME_RE.match(rest[0]):
|
||||
raise Refused("U1", f"unit name {rest[0]!r} is not mnt-<escaped path>.mount|.automount")
|
||||
return v, os.path.join(UNITS_SRC, rest[0]), os.path.join(UNIT_DIR, rest[0]), 0o644, lambda t: check_unit(rest[0], t)
|
||||
if v == "dnsmasq" and len(rest) == 2:
|
||||
if not DNSMASQ_TMP_RE.match(rest[0]) or not DNSMASQ_NAME_RE.match(rest[1]):
|
||||
raise Refused("D2", "dnsmasq wants /tmp/felhom-resolver-<digits>.conf and felhom-<name>.conf")
|
||||
return v, rest[0], os.path.join(DNSMASQ_DIR, rest[1]), 0o644, check_dnsmasq
|
||||
if v == "wg" and not rest:
|
||||
return v, WG_SRC, WG_DEST, 0o600, check_wg
|
||||
if v == "sshd-config" and not rest:
|
||||
return v, SSHD_SRC, SSHD_DEST, 0o644, check_sshd
|
||||
if v == "sshd-key" and not rest:
|
||||
return v, KEY_SRC, KEY_DEST, 0o644, check_key
|
||||
raise Refused("A1", f"wrong arguments for {v}")
|
||||
|
||||
|
||||
def controller_image(rest, host):
|
||||
"""R-861 (a) A1: read the ref on stdin, check it, write it INSIDE the guest as root."""
|
||||
try:
|
||||
if len(rest) != 1 or not VMID_RE.match(rest[0]):
|
||||
raise Refused("A1", "usage: felhom-priv-apply controller-image <vmid> (the ref on stdin)")
|
||||
raw = host.read_stdin(CONTROLLER_IMAGE_MAX)
|
||||
if len(raw) > CONTROLLER_IMAGE_MAX:
|
||||
raise Refused("I1", f"the image ref is longer than {CONTROLLER_IMAGE_MAX} bytes")
|
||||
try:
|
||||
text = raw.decode("ascii")
|
||||
except UnicodeDecodeError:
|
||||
raise Refused("I1", "the image ref is not ASCII")
|
||||
ref = text[:-1] if text.endswith("\n") else text
|
||||
if not CONTROLLER_IMAGE_RE.match(ref) or "\n" in ref:
|
||||
raise Refused("I1", "the image ref is not gitea.dooplex.hu/admin/felhom-controller:<x.y.z>")
|
||||
except Refused as e:
|
||||
host.log(f"felhom-priv-apply: REFUSED [{e.rule}] controller-image {' '.join(rest)[:40]}: {e.reason}")
|
||||
return 2 if e.rule == "A1" else 3
|
||||
vmid = int(rest[0])
|
||||
try:
|
||||
host.write_guest_image(vmid, (ref + "\n").encode())
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
host.log(f"felhom-priv-apply: FAILED controller-image {vmid}: {e}")
|
||||
return 4
|
||||
host.log(f"felhom-priv-apply: WROTE controller-image {vmid} {ref}")
|
||||
return 0
|
||||
|
||||
|
||||
def unit_forget(rest, host):
|
||||
"""R-1124: remove ONE felhom drive mount unit that the agent has already disabled, whose drive is gone. A drive that
|
||||
is still present or still mounted keeps its unit — this verb can only forget, never unmount or take a drive away."""
|
||||
try:
|
||||
if len(rest) != 1 or not DRIVE_UNIT_NAME_RE.match(rest[0]) or "/" in rest[0]:
|
||||
raise Refused("A1", "usage: felhom-priv-apply unit-forget mnt-<escaped path>.mount")
|
||||
name = rest[0]
|
||||
path = os.path.join(UNIT_DIR, name)
|
||||
data = host.read_installed(path)
|
||||
if data is None:
|
||||
host.log(f"felhom-priv-apply: ABSENT unit-forget {path} (nothing to remove)")
|
||||
return 0
|
||||
text = text_of(data, path)
|
||||
if DRIVE_UNIT_MARKER not in text or NET_UNIT_MARKER in text:
|
||||
raise Refused("F2", f"{name} is not a felhom drive unit")
|
||||
check_unit(name, text)
|
||||
s = parse_ini(text, name)
|
||||
what, where = s["Mount"].get("What", ""), s["Mount"]["Where"]
|
||||
mu = re.match(r"^/dev/disk/by-uuid/(.+)$", what)
|
||||
if not mu:
|
||||
raise Refused("F2", f"{name} is not a by-UUID drive unit")
|
||||
if host.lexists(os.path.join(WANTS_DIR, name)):
|
||||
raise Refused("F3", f"{name} is still enabled (disable it first)")
|
||||
if where in host.mountpoints():
|
||||
raise Refused("F4", f"{where} is mounted")
|
||||
if host.lexists(os.path.join(BY_UUID_DIR, mu.group(1))):
|
||||
raise Refused("F5", f"the drive of {name} is present (filesystem UUID {mu.group(1)})")
|
||||
except Refused as e:
|
||||
host.log(f"felhom-priv-apply: REFUSED [{e.rule}] unit-forget {' '.join(rest)[:120]}: {e.reason}")
|
||||
return 2 if e.rule == "A1" else 3
|
||||
try:
|
||||
host.remove(path)
|
||||
except OSError as e:
|
||||
host.log(f"felhom-priv-apply: FAILED unit-forget {path}: {e}")
|
||||
return 4
|
||||
host.log(f"felhom-priv-apply: REMOVED unit-forget {path} (where={where})")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv, host=None):
|
||||
host = host or Host()
|
||||
if argv == ["--self-check"]:
|
||||
print("felhom-priv-apply ok verbs=" + ",".join(VERBS))
|
||||
return 0
|
||||
if len(argv) >= 3 and argv[0] == "--check":
|
||||
# CHECK ONLY (tests and the Go contract tests): `--check <verb> [<name>] <file>` validates <file> as that
|
||||
# verb would and installs nothing. Not in sudoers. Prints OK or the rule; never the content.
|
||||
verb, file = argv[1], argv[-1]
|
||||
try:
|
||||
_, _, _, _, checker = plan([verb] + argv[2:-1] if verb != "dnsmasq" else
|
||||
[verb, "/tmp/felhom-resolver-1.conf", argv[2]])
|
||||
with open(file, "rb") as f:
|
||||
checker(text_of(f.read(), file))
|
||||
except Refused as e:
|
||||
print(f"REFUSED [{e.rule}] {e.reason}")
|
||||
return 3
|
||||
print("OK")
|
||||
return 0
|
||||
if argv and argv[0] == "controller-image":
|
||||
return controller_image(argv[1:], host)
|
||||
if argv and argv[0] == "unit-forget":
|
||||
return unit_forget(argv[1:], host)
|
||||
try:
|
||||
verb, src, dest, mode, checker = plan(argv)
|
||||
data = host.read_source(src)
|
||||
checker(text_of(data, src))
|
||||
except Refused as e:
|
||||
host.log(f"felhom-priv-apply: REFUSED [{e.rule}] {' '.join(argv)[:160]}: {e.reason}")
|
||||
return 2 if e.rule == "A1" else 3
|
||||
if host.read_dest(dest) == data:
|
||||
host.log(f"felhom-priv-apply: SAME {verb} {dest}")
|
||||
return 0
|
||||
try:
|
||||
host.install(dest, data, mode)
|
||||
except OSError as e:
|
||||
host.log(f"felhom-priv-apply: FAILED {verb} {dest}: {e}")
|
||||
return 4
|
||||
host.log(f"felhom-priv-apply: INSTALLED {verb} {dest} ({len(data)} bytes)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,15 @@
|
||||
# felhom privsep-dir persistence (TASK G1) — install as /etc/tmpfiles.d/felhom-privsep.conf.
|
||||
#
|
||||
# LAYER 1 (prevent): make OpenSSH's shared privilege-separation directory /run/sshd boot-persistent
|
||||
# and owned by NO systemd unit's lifecycle. This closes the exact incident cause from
|
||||
# SPIKE-felhom-sshd-2026-07-05 §8: a second sshd unit declaring `RuntimeDirectory=sshd` had systemd
|
||||
# REMOVE the shared /run/sshd on that unit's stop/failure, taking the stock sshd on :22 down with it
|
||||
# (sessions reset right after SSH2_MSG_KEXINIT). A tmpfiles.d entry recreates the dir at every boot
|
||||
# independently of any unit, so no unit's RuntimeDirectory cleanup can be the sole owner.
|
||||
#
|
||||
# systemd-tmpfiles is idempotent: `systemd-tmpfiles --create` re-run is a no-op on an existing,
|
||||
# correct dir (it only creates/fixes, never churns). Complemented at runtime by the
|
||||
# felhom-mgmt-watchdog timer (layer 2), which re-heals a dir removed AFTER boot.
|
||||
#
|
||||
# Type d = create the directory if absent (leaves an existing one, only fixing mode/owner).
|
||||
d /run/sshd 0755 root root -
|
||||
@@ -0,0 +1,7 @@
|
||||
# MANAGED BY FELHOM (the agent's config bundle) — R-982, operator ruling D4 (2026-10-22).
|
||||
# The Proxmox host's OWN sshd — the listener the household network reaches on port 22: root logs in with a KEY only.
|
||||
# The root password still works on the console (the break-glass password: felhom.eu runbooks/break-glass.md). The
|
||||
# operator's door (felhom-sshd, port 8822, /etc/felhom-sshd/sshd_config) is a separate daemon whose config includes
|
||||
# nothing from here, so it is not changed. Debian's sshd_config reads this folder BEFORE its own settings, and for
|
||||
# sshd the first value wins; the bundle checks the effective value with `sshd -T` after the reload.
|
||||
PermitRootLogin prohibit-password
|
||||
@@ -0,0 +1,147 @@
|
||||
#!/bin/sh
|
||||
# felhom-selfupdate-guarded — the ROOT half of the agent's A/B self-update (TASK D1).
|
||||
#
|
||||
# Install as /usr/local/sbin/felhom-selfupdate-guarded (0755 root:root). The non-root agent invokes
|
||||
# `apply`/`commit` via `sudo -n` (FELHOM_SELFUPDATE alias); `rollback` is ALSO the ExecStart of
|
||||
# felhom-agent-rollback.service — the OnFailure= target that auto-reverts a crash-looping update.
|
||||
#
|
||||
# Design provenance: SPIKE-agent-selfupdate-2026-07-05 (its SF-findings are cited inline). The core
|
||||
# principle: the thing that performs rollback is never the thing being updated — this wrapper +
|
||||
# systemd change almost never; the agent binary is what flips.
|
||||
#
|
||||
# Trust model: the agent verifies the download against the OPERATOR-SIGNED sha before staging; this
|
||||
# wrapper RE-verifies the same sha as root (defense in depth — the sudoers arg glob is coarse, the
|
||||
# sha check here is the real gate). Path confinement: apply only ever reads from the agent's own
|
||||
# staging dir and only ever writes the fixed live path + its siblings. NO env-overridable paths —
|
||||
# path-fixedness IS the security property (a test-mode override would be a root escalation hole).
|
||||
#
|
||||
# Verbs:
|
||||
# apply <staged> <sha256> stage-verify → .prev → atomic flip → pending marker → detached restart
|
||||
# commit clear the pending marker (idempotent; .prev retained as a manual net)
|
||||
# rollback pending-guarded revert to .prev + restart (no pending → exit 0 no-op)
|
||||
set -u
|
||||
|
||||
BIN=/usr/local/bin/felhom-agent
|
||||
PREV=$BIN.prev
|
||||
STAGING=/var/lib/felhom-agent/selfupdate
|
||||
# R-861 (agent v0.146.1): `apply` takes ONLY the root-owned copy felhom-os-apply writes after it has verified the
|
||||
# operator's signature and hashed exactly those bytes (mode agent_update). The agent cannot call `apply` any more (it
|
||||
# left the sudoers), and the agent's own staging dir is no longer accepted: a file in a directory the agent owns can be
|
||||
# swapped between this script's sha check and its copy.
|
||||
ROOT_STAGING=/var/lib/felhom-os-apply/agent-update
|
||||
PENDING=$STAGING/pending.json
|
||||
UNIT=felhom-agent.service
|
||||
|
||||
# Every refusal/decision goes to stderr AND the journal (strict rule 10).
|
||||
log() { echo "felhom-selfupdate-guarded: $*" >&2; logger -t felhom-selfupdate-guarded -- "$*" 2>/dev/null || true; }
|
||||
|
||||
case "${1:-}" in
|
||||
|
||||
apply)
|
||||
staged=${2:-}; want=${3:-}
|
||||
# [SF-7] entry sweep: a kill between staging-copy and mv leaves an orphaned temp — harmless,
|
||||
# but sweep it so temps can never accumulate.
|
||||
rm -f "$BIN".new.*
|
||||
|
||||
if [ -z "$staged" ] || [ -z "$want" ]; then
|
||||
log "refusing apply: usage: apply <staged> <sha256>"
|
||||
exit 2
|
||||
fi
|
||||
# Root-side path confinement: the staged binary MUST be felhom-os-apply's root-owned copy (R-861).
|
||||
case "$staged" in
|
||||
"$ROOT_STAGING"/*) ;;
|
||||
*) log "refusing apply: staged path outside $ROOT_STAGING: $staged"; exit 1 ;;
|
||||
esac
|
||||
if [ -L "$staged" ] || [ "$(stat -c %u "$staged" 2>/dev/null)" != "0" ]; then
|
||||
log "refusing apply: $staged is a symlink or not root-owned"; exit 1
|
||||
fi
|
||||
case "$staged" in
|
||||
*..*) log "refusing apply: staged path contains '..'"; exit 1 ;;
|
||||
esac
|
||||
[ -f "$staged" ] || { log "refusing apply: staged file missing: $staged"; exit 1; }
|
||||
# The sha must be 64 lowercase hex chars — anything else is refused before any hashing.
|
||||
case "$want" in
|
||||
*[!0-9a-f]*) log "refusing apply: sha256 is not lowercase hex"; exit 1 ;;
|
||||
esac
|
||||
[ "${#want}" -eq 64 ] || { log "refusing apply: sha256 must be 64 hex chars (got ${#want})"; exit 1; }
|
||||
|
||||
# [SF-7] sha-verify FIRST — before .prev, before any mutation (spike S3a companion ordering).
|
||||
got=$(sha256sum "$staged" | awk '{print $1}')
|
||||
if [ "$got" != "$want" ]; then
|
||||
log "refusing apply: sha mismatch (got=$got want=$want)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Same-fs assert (§8): the atomic-rename guarantee only holds within one filesystem.
|
||||
if [ "$(stat -c %d "$staged")" != "$(stat -c %d /usr/local/bin)" ]; then
|
||||
log "refusing apply: staging and /usr/local/bin are on different filesystems — atomic rename impossible"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
old_ver=$("$BIN" --version 2>/dev/null | awk '{print $2}')
|
||||
[ -n "$old_ver" ] || old_ver=unknown
|
||||
# The staged filename is felhom-agent-<version> (executor contract) — version without executing.
|
||||
new_ver=$(basename "$staged"); new_ver=${new_ver#felhom-agent-}
|
||||
|
||||
cp -p "$BIN" "$PREV" || { log "apply failed: cannot snapshot current binary to .prev"; exit 1; }
|
||||
# Stage a root-owned 0755 copy next to the live path, then ATOMIC same-fs rename.
|
||||
if ! cp "$staged" "$BIN.new.$$" || ! chmod 0755 "$BIN.new.$$" || ! chown root:root "$BIN.new.$$"; then
|
||||
rm -f "$BIN.new.$$"; log "apply failed: staging copy"; exit 1
|
||||
fi
|
||||
mv "$BIN.new.$$" "$BIN" || { rm -f "$BIN.new.$$"; log "apply failed: atomic rename"; exit 1; }
|
||||
|
||||
# Pending marker: written AFTER the flip — its existence means "an update is awaiting commit",
|
||||
# which is exactly the rollback unit's trigger condition.
|
||||
printf '{"old_version":"%s","new_version":"%s","sha256":"%s","applied_at":"%s"}\n' \
|
||||
"$old_ver" "$new_ver" "$want" "$(date -Is)" > "$PENDING" \
|
||||
|| { log "apply failed: cannot write pending marker"; exit 1; }
|
||||
|
||||
# [SF-4/5] deliberate restarts consume start-limit budget — clear the counter first.
|
||||
systemctl reset-failed "$UNIT" 2>/dev/null || true
|
||||
# [SF-6] the spike's S2b winner, verbatim: detached transient timer OUTSIDE the agent's cgroup,
|
||||
# so the sudo/agent caller survives to log the handoff and the restart cannot be torn down
|
||||
# by its own requester dying.
|
||||
systemd-run --on-active=2s --timer-property=AccuracySec=100ms systemctl restart "$UNIT" \
|
||||
|| { log "apply: flip done but detached restart scheduling FAILED — restart $UNIT manually"; exit 1; }
|
||||
log "applied $new_ver (prev $old_ver, sha $want); detached restart scheduled"
|
||||
;;
|
||||
|
||||
commit)
|
||||
if [ ! -f "$PENDING" ]; then
|
||||
log "commit: no pending — no-op"
|
||||
exit 0
|
||||
fi
|
||||
# .prev is deliberately RETAINED (spike S3d) — a manual safety net until the next apply.
|
||||
rm -f "$PENDING" || { log "commit failed: cannot remove pending marker"; exit 1; }
|
||||
log "committed (pending cleared, .prev retained)"
|
||||
;;
|
||||
|
||||
rollback)
|
||||
# [SF-1] On systemd 257 OnFailure= fires on EVERY crash, so this verb runs MANY times per
|
||||
# incident — the pending-guard makes every fire after the first a harmless no-op, and makes a
|
||||
# crash with NO update in flight touch nothing at all (spike S1d/S3e).
|
||||
if [ ! -f "$PENDING" ]; then
|
||||
log "rollback: no pending update — no-op"
|
||||
exit 0
|
||||
fi
|
||||
[ -f "$PREV" ] || { log "rollback FAILED: pending exists but no .prev binary"; exit 1; }
|
||||
rm -f "$BIN".new.*
|
||||
if ! cp "$PREV" "$BIN.new.$$" || ! chmod 0755 "$BIN.new.$$" || ! chown root:root "$BIN.new.$$"; then
|
||||
rm -f "$BIN.new.$$"; log "rollback FAILED: staging copy"; exit 1
|
||||
fi
|
||||
mv "$BIN.new.$$" "$BIN" || { rm -f "$BIN.new.$$"; log "rollback FAILED: atomic rename"; exit 1; }
|
||||
# Clear pending BEFORE the restart: once the binary is reverted, later OnFailure fires must
|
||||
# no-op (the guard above) instead of re-copying .prev forever.
|
||||
rm -f "$PENDING"
|
||||
# [SF-4/5] the crash burst has been eating the start-limit budget — reset before starting.
|
||||
systemctl reset-failed "$UNIT" 2>/dev/null || true
|
||||
# Direct restart is correct HERE: this caller is the rollback oneshot, OUTSIDE the agent cgroup.
|
||||
systemctl restart "$UNIT" || { log "rollback: binary reverted but restart FAILED"; exit 1; }
|
||||
log "rolled back to previous binary and restarted $UNIT"
|
||||
;;
|
||||
|
||||
*)
|
||||
log "usage: felhom-selfupdate-guarded apply <staged> <sha256> | commit | rollback"
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Felhom stable drive parent (shared bind for live drive hot-swap)
|
||||
After=local-fs.target
|
||||
Before=pve-guests.service
|
||||
ConditionPathExists=/usr/local/sbin/felhom-shared-parent.sh
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=yes
|
||||
ExecStart=/usr/local/sbin/felhom-shared-parent.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=pve-guests.service multi-user.target
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
# felhom stable drive parent: a SHARED bind so the agent can swap backing drives underneath it and the
|
||||
# guest sees the change live (no restart). MUST run before pve-guests so the guest's parent bind inherits
|
||||
# the shared peer group (slave). Installed + enabled by felhom-agent. Idempotent.
|
||||
set -e
|
||||
mkdir -p /mnt/felhom-drives
|
||||
# Isolate + share ONLY when first creating the self-bind (a fresh boot). The self-bind inherits the root
|
||||
# mount's shared peer group, so make-private detaches it (else binds under it DOUBLE via the root peer),
|
||||
# then make-shared gives it its own group whose only slave is the guest's parent bind. Re-running this on
|
||||
# an existing parent would churn the peer-group id and orphan the guest's slave — so guard on mountpoint.
|
||||
if ! mountpoint -q /mnt/felhom-drives; then
|
||||
mount --bind /mnt/felhom-drives /mnt/felhom-drives
|
||||
mount --make-private /mnt/felhom-drives
|
||||
mount --make-shared /mnt/felhom-drives
|
||||
fi
|
||||
@@ -0,0 +1,28 @@
|
||||
# felhom-sshd.service (TASK H1) — install as /etc/systemd/system/felhom-sshd.service.
|
||||
#
|
||||
# The dedicated OOB sshd instance: a SECOND sshd on a claimed non-22 port, COEXISTING with the
|
||||
# customer's/stock sshd on :22 (never touched). Config is AGENT-RENDERED at /etc/felhom-sshd/sshd_config
|
||||
# (Port from the agent's claim); the agent reloads on change (never restart-on-change [SF-2]).
|
||||
#
|
||||
# CRITICAL [SF-1]: this unit MUST NOT declare `RuntimeDirectory=` — that directive (value `sshd`)
|
||||
# removed the SHARED /run/sshd and broke the stock sshd live (SPIKE-felhom-sshd §8). /run/sshd is
|
||||
# guaranteed by G1 (tmpfiles + watchdog); the ExecStartPre mkdir is a harmless extra guarantee. The
|
||||
# host-install RuntimeDirectory guard will REJECT this unit if the directive is ever added.
|
||||
[Unit]
|
||||
Description=Felhom OOB sshd (dedicated instance, agent-managed; H1)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
ExecStartPre=-/usr/bin/mkdir -p /run/sshd
|
||||
ExecStartPre=/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config
|
||||
ExecStart=/usr/sbin/sshd -D -f /etc/felhom-sshd/sshd_config
|
||||
# Config-change path: validate then HUP — a bad reload is REFUSED and the running daemon SURVIVES
|
||||
# [SF-2]. The agent always writes→sshd -t→reload; it never `restart`s on a config change.
|
||||
ExecReload=/usr/sbin/sshd -t -f /etc/felhom-sshd/sshd_config
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
KillMode=process
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+74
@@ -0,0 +1,74 @@
|
||||
#!/bin/bash
|
||||
# felhom-thin-check — list the thin device ids a thin pool holds next to the ones LVM knows (R-1012). READ-ONLY: it
|
||||
# names an orphan, it never removes one. Installed as /usr/local/sbin/felhom-thin-check (config bundle).
|
||||
#
|
||||
# felhom-thin-check <vg> <pool>
|
||||
#
|
||||
# A full host root while vzdump takes its snapshot can leave a thin snapshot inside the pool that LVM no longer knows
|
||||
# (demo-hp 2026-10-14: `thin_ls` device 4, LVM only 1 and 2); from then on every snapshot backup of that guest fails
|
||||
# („Creation of new snapshot 4 of device 2 failed") and nothing said why. The repair (`dmsetup message … delete <id>`)
|
||||
# stays a HUMAN step, decided by the operator — this helper only reads.
|
||||
#
|
||||
# Reading the live pool's device list needs a metadata snapshot: the kernel's documented way to read metadata while
|
||||
# the pool is in use (`reserve_metadata_snap`, `thin_ls --metadata-snap`, `release_metadata_snap`). It changes no data
|
||||
# and no mapping; it is held for the length of one `thin_ls -o DEV` (well under a second on these pools). Rules:
|
||||
# 1. the arguments are an LVM name pair; <vg>/<pool> is a thin pool (lv_attr starts with `t`);
|
||||
# 2. only one run at a time (a lock file); a metadata snapshot that someone ELSE holds is never touched — the run
|
||||
# refuses (exit 3) and does not release it;
|
||||
# 3. LVM's list is read before AND after the pool's, so a volume created in between is not reported as an orphan
|
||||
# (the caller treats an id as an orphan only when it is in the pool and in neither LVM list).
|
||||
# Output, one record per line: lvm-before <id> <lv> | pool <id> | lvm-after <id> <lv> | ok
|
||||
# Exit: 0 read, 1 refused (arguments / not a thin pool), 3 a metadata snapshot is already held or cannot be reserved,
|
||||
# 4 the pool could not be read. Pinned by configs/test_felhom_thin_check.py.
|
||||
set -u
|
||||
LVS=/usr/sbin/lvs
|
||||
DMSETUP=/usr/sbin/dmsetup
|
||||
THIN_LS=/usr/sbin/thin_ls
|
||||
LOCK=/run/lock/felhom-thin-check.lock
|
||||
NAME_RE='^[A-Za-z0-9_+][A-Za-z0-9_.+-]{0,63}$'
|
||||
|
||||
die() { echo "felhom-thin-check: REFUSED: $1" >&2; exit "${2:-1}"; } # $1 the reason, $2 the exit code
|
||||
|
||||
lvm_ids() { # $1 tag → "<tag> <id> <lv>" for every thin volume of this pool
|
||||
"$LVS" --noheadings --separator , -o lv_name,thin_id --select "pool_lv=$POOL" -- "$VG" 2>/dev/null |
|
||||
awk -F, -v tag="$1" '{ gsub(/ /, ""); if ($2 ~ /^[0-9]+$/) print tag, $2, $1 }'
|
||||
}
|
||||
|
||||
release() { "$DMSETUP" message "$TPOOL" 0 release_metadata_snap >/dev/null 2>&1; }
|
||||
|
||||
main() {
|
||||
[[ $# -eq 2 ]] || die "usage: felhom-thin-check <vg> <pool>"
|
||||
VG=$1 POOL=$2
|
||||
local attr held ids
|
||||
[[ "$VG" =~ $NAME_RE && "$POOL" =~ $NAME_RE ]] || die "not an LVM name pair"
|
||||
|
||||
attr=$("$LVS" --noheadings -o lv_attr -- "$VG/$POOL" 2>/dev/null | tr -d ' ')
|
||||
[[ "$attr" == t* ]] || die "$VG/$POOL is not a thin pool"
|
||||
|
||||
DM="${VG//-/--}-${POOL//-/--}"
|
||||
TPOOL="$DM-tpool"
|
||||
TMETA="/dev/mapper/${DM}_tmeta"
|
||||
|
||||
exec 9>"$LOCK" || die "cannot open the lock file" 4
|
||||
flock -w 30 9 || die "another check holds the lock" 3
|
||||
|
||||
# Field 7 of the thin-pool status is the held metadata root: "-" when no metadata snapshot exists.
|
||||
held=$("$DMSETUP" status "$TPOOL" 2>/dev/null | awk '$3 == "thin-pool" { print $7 }')
|
||||
[[ -n "$held" ]] || die "cannot read the status of $TPOOL" 4
|
||||
[[ "$held" == "-" ]] || die "a metadata snapshot of $TPOOL is already held (by someone else) — not touched" 3
|
||||
|
||||
lvm_ids lvm-before
|
||||
"$DMSETUP" message "$TPOOL" 0 reserve_metadata_snap >/dev/null 2>&1 || die "could not reserve a metadata snapshot of $TPOOL" 3
|
||||
trap release EXIT
|
||||
trap 'release; trap - EXIT; exit 4' INT TERM HUP
|
||||
if ! ids=$("$THIN_LS" --metadata-snap --no-headers -o DEV "$TMETA" 2>/dev/null); then
|
||||
die "thin_ls could not read $TMETA" 4
|
||||
fi
|
||||
release
|
||||
trap - EXIT INT TERM HUP
|
||||
awk '$1 ~ /^[0-9]+$/ { print "pool", $1 }' <<<"$ids"
|
||||
lvm_ids lvm-after
|
||||
echo ok
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@"; fi
|
||||
Executable
+99
@@ -0,0 +1,99 @@
|
||||
#!/bin/bash
|
||||
# felhom-vzdump-partial-clean — remove ONE dead, half-written vzdump backup (R-1006). The only delete the agent's
|
||||
# sudoers allows for backup files. Installed as /usr/local/sbin/felhom-vzdump-partial-clean (config bundle).
|
||||
#
|
||||
# felhom-vzdump-partial-clean <storage> <vmid> <name>
|
||||
#
|
||||
# A power cut during the night backup leaves `vzdump-lxc-<vmid>-<date>.tar.dat` (+ its `.tmp/` directory) on the
|
||||
# storage; nothing removes it, every later backup is then skipped „not enough space", and a full host root breaks
|
||||
# LVM's own metadata writes (power-cut-fixes-2026-10-14 C4). This helper removes such a leftover — and NOTHING else.
|
||||
# It trusts nothing the agent says; every rule is checked here, as root, and any doubt REFUSES (exit 1):
|
||||
# 1. the arguments: a storage id, a vmid, and a name that is exactly
|
||||
# vzdump-(lxc|qemu)-<THAT vmid>-YYYY_MM_DD-HH_MM_SS.(tar.dat|vma.dat|tmp) — never a finished archive
|
||||
# (.tar.zst, .vma.zst, …), never a .log/.notes, never another vmid;
|
||||
# 2. the storage: its section in /etc/pve/storage.cfg is `dir: <storage>` with a `path`; the target is
|
||||
# <path>/dump/<name>, not a symlink, and its real directory IS <path>/dump;
|
||||
# 3. not in use: no vzdump runs on the host at all (a task `…:vzdump:…` or a `vzdump` command — a job for several
|
||||
# guests carries no vmid in its task name), and no
|
||||
# process holds the target (or, for the .tmp directory, anything under it) open or as its working directory
|
||||
# (C2: after a killed worker, tar and zstd kept writing into the .tar.dat for 4 min 40 s, none named „vzdump");
|
||||
# 4. dead: last modified before this boot (`btime` in /proc/stat), or untouched for at least QUIET_S seconds.
|
||||
# Prints one line on success: „removed <name> <bytes> B (<why>)". Pinned by configs/test_felhom_vzdump_partial_clean.py.
|
||||
set -u
|
||||
STORAGE_CFG=/etc/pve/storage.cfg
|
||||
PROC=/proc
|
||||
QUIET_S=120
|
||||
|
||||
die() { echo "felhom-vzdump-partial-clean: REFUSED: $*" >&2; exit 1; }
|
||||
|
||||
storage_path() { # $1 storage id → prints the dir storage's path, or nothing
|
||||
awk -v want="$1" '
|
||||
/^[a-z]+:[ \t]/ { split($0, h, /:[ \t]+/); type = h[1]; id = h[2]; sub(/[ \t]+$/, "", id); in_s = (id == want); next }
|
||||
in_s && /^[ \t]+path[ \t]/ { if (type == "dir") { sub(/^[ \t]+path[ \t]+/, ""); sub(/[ \t]+$/, ""); print; exit } }
|
||||
' "$STORAGE_CFG"
|
||||
}
|
||||
|
||||
storage_type() {
|
||||
awk -v want="$1" '/^[a-z]+:[ \t]/ { split($0, h, /:[ \t]+/); id = h[2]; sub(/[ \t]+$/, "", id); if (id == want) { print h[1]; exit } }' "$STORAGE_CFG"
|
||||
}
|
||||
|
||||
boot_time() { awk '$1 == "btime" { print $2 }' "$PROC/stat"; }
|
||||
|
||||
newest_mtime() { # a file's mtime, or the newest mtime of anything in a directory (the dir itself included)
|
||||
if [[ -d "$1" ]]; then find "$1" -xdev -printf '%T@\n' 2>/dev/null | sort -n | tail -1 | cut -d. -f1
|
||||
else stat -c %Y "$1"; fi
|
||||
}
|
||||
|
||||
busy_reason() { # $1 vmid, $2 target → prints why it is in use, or nothing
|
||||
local vmid=$1 target=$2 p link cmd
|
||||
for p in "$PROC"/[0-9]*; do
|
||||
[[ -r "$p/cmdline" ]] || continue
|
||||
cmd=$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null)
|
||||
# ANY vzdump on the host refuses: a job for several guests runs as `task UPID:…:vzdump::…` (no vmid in it), and
|
||||
# its tar may be between this guest's files. Waiting costs one backup run; deleting a live one costs the backup.
|
||||
if [[ "$cmd" == *":vzdump:"* ]] || [[ "$cmd" =~ (^|/)vzdump([[:space:]]|$) ]]; then
|
||||
echo "a vzdump runs on this host (pid ${p##*/})"; return
|
||||
fi
|
||||
for link in "$p/cwd" "$p"/fd/*; do
|
||||
local to
|
||||
to=$(readlink "$link" 2>/dev/null) || continue
|
||||
if [[ "$to" == "$target" || "$to" == "$target/"* ]]; then
|
||||
echo "pid ${p##*/} has it open"; return
|
||||
fi
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
local storage=${1:-} vmid=${2:-} name=${3:-}
|
||||
[[ $# -eq 3 ]] || die "usage: felhom-vzdump-partial-clean <storage> <vmid> <name>"
|
||||
[[ "$storage" =~ ^[A-Za-z][A-Za-z0-9_-]{0,63}$ ]] || die "bad storage id"
|
||||
[[ "$vmid" =~ ^[1-9][0-9]{2,8}$ ]] || die "bad vmid"
|
||||
[[ "$name" =~ ^vzdump-(lxc|qemu)-${vmid}-[0-9]{4}_[0-9]{2}_[0-9]{2}-[0-9]{2}_[0-9]{2}_[0-9]{2}\.(tar\.dat|vma\.dat|tmp)$ ]] \
|
||||
|| die "name is not a half-written vzdump of guest $vmid: $name"
|
||||
[[ "$(storage_type "$storage")" == "dir" ]] || die "storage $storage is not a dir storage"
|
||||
local path; path=$(storage_path "$storage")
|
||||
[[ "$path" == /* ]] || die "storage $storage has no path"
|
||||
local dump="$path/dump" target="$path/dump/$name"
|
||||
[[ -e "$target" ]] || die "no such entry: $target"
|
||||
[[ ! -L "$target" ]] || die "a symlink: $target"
|
||||
[[ "$(realpath -e "$dump")" == "$(realpath -e "$(dirname "$target")")" ]] || die "not directly in $dump"
|
||||
if [[ "$name" == *.tmp ]]; then [[ -d "$target" ]] || die ".tmp is not a directory"
|
||||
else [[ -f "$target" ]] || die "not a regular file"; fi
|
||||
local busy; busy=$(busy_reason "$vmid" "$target")
|
||||
[[ -z "$busy" ]] || die "in use: $busy"
|
||||
local bt mt now why
|
||||
bt=$(boot_time); mt=$(newest_mtime "$target"); now=$(date +%s)
|
||||
[[ "$bt" =~ ^[0-9]+$ && "$mt" =~ ^[0-9]+$ ]] || die "cannot read the boot time or the modification time"
|
||||
if (( mt < bt )); then why="written before this boot"
|
||||
elif (( now - mt >= QUIET_S )); then why="untouched for $(( now - mt )) s, no vzdump for $vmid, nothing holds it"
|
||||
else die "modified $(( now - mt )) s ago — may still be written"; fi
|
||||
local size; size=$(du -sb "$target" 2>/dev/null | cut -f1)
|
||||
if [[ -d "$target" ]]; then rm -rf --one-file-system -- "$target"; else rm -f -- "$target"; fi
|
||||
[[ ! -e "$target" ]] || die "remove failed: $target"
|
||||
echo "removed $name ${size:-?} B ($why)"
|
||||
}
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
main "$@"
|
||||
fi
|
||||
@@ -0,0 +1,973 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for the config bundle (R-840, `11` §5.4.2): felhom-os-apply's `bundle` mode and `--install-bundle`, and
|
||||
scripts/build-config-bundle.py. An in-memory host plays the files; nothing real is written or run. Each refusal has a
|
||||
test; each test names the rule it pins. Red-proof: `audits/r840-config-bundle-2026-10-04/partB/redproof.txt`.
|
||||
|
||||
Run: python3 configs/test_felhom_config_bundle.py (also run by internal/osupdate's Go test)
|
||||
"""
|
||||
import sys
|
||||
sys.dont_write_bytecode = True # importing the builder must not leave scripts/__pycache__ behind
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import stat as statmod
|
||||
import unittest
|
||||
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
REPO = HERE.parent
|
||||
_loader = importlib.machinery.SourceFileLoader("osapply", os.environ.get("OSAPPLY_UNDER_TEST", str(HERE / "felhom-os-apply")))
|
||||
_spec = importlib.util.spec_from_loader("osapply", _loader)
|
||||
osapply = importlib.util.module_from_spec(_spec)
|
||||
_loader.exec_module(osapply)
|
||||
_bl = importlib.machinery.SourceFileLoader("bundlebuild", str(REPO / "scripts" / "build-config-bundle.py"))
|
||||
_bs = importlib.util.spec_from_loader("bundlebuild", _bl)
|
||||
builder = importlib.util.module_from_spec(_bs)
|
||||
_bl.exec_module(builder)
|
||||
|
||||
PLAN = "/var/lib/felhom-agent/os/plan-b1.json"
|
||||
BUNDLE = "/var/lib/felhom-agent/os/bundle-0.143.0.json"
|
||||
HOST = "demo-hp-bb76ea"
|
||||
INSTALLER = REPO.parent / "felhom.eu" / "scripts" / "felhom-host-install.sh"
|
||||
|
||||
|
||||
class St:
|
||||
def __init__(self, mode, uid):
|
||||
self.st_mode, self.st_uid, self.st_size = mode, uid, 100
|
||||
|
||||
|
||||
class Box:
|
||||
"""An in-memory host. files: path -> bytes; modes/uids per path; dirs: a set."""
|
||||
|
||||
def __init__(self, bundle_bytes, signed, oob=False, signers=True, install_mode="appliance"):
|
||||
self.files, self.modes, self.uids = {}, {}, {}
|
||||
if install_mode is not None: # the installer's ROOT-OWNED record (D43 = A reads it for the sshd drop-in)
|
||||
self.put(osapply.INSTALL_STATE, json.dumps({"completed": [], "mode": install_mode}).encode(), 0o644)
|
||||
self.dirs = {osapply.OOB_DIR} if oob else set()
|
||||
self.put(osapply.TRUST_FILE, json.dumps({"host_id": HOST, "ring0_slow_lane": False}).encode(), 0o644)
|
||||
if signers:
|
||||
self.put(osapply.TRUST_SIGNERS, b'felhom-op-1 namespaces="felhom-op-v1" ssh-ed25519 AAAA felhom-op-1\n', 0o644)
|
||||
self.put("/proc/sys/kernel/panic", b"0\n", 0o644)
|
||||
self.plan = {"release_id": "bundle-0.143.0", "layer": "host", "mode": "bundle", "bundle": BUNDLE, "signed": signed}
|
||||
self.put(PLAN, json.dumps(self.plan).encode(), 0o600, uid=999)
|
||||
self.put(BUNDLE, bundle_bytes, 0o600, uid=999)
|
||||
self.sig_rc, self.nonces, self.clock = 0, {}, 1791115200.0 # 2026-10-04T12:00:00Z
|
||||
self.calls, self.logs, self.writes = [], [], []
|
||||
self.visudo_fail = False # the WHOLE sudoers (`visudo -c`) after install
|
||||
self.sudo_l = " (root) NOPASSWD: /usr/local/sbin/felhom-os-apply --plan /var/lib/felhom-agent/os/plan-*.json\n"
|
||||
self.guard = {"armed": True, "kernel_panic": 10}
|
||||
|
||||
def put(self, p, data, mode, uid=0):
|
||||
self.files[p], self.modes[p], self.uids[p] = data, mode, uid
|
||||
|
||||
# Runner interface
|
||||
def now(self):
|
||||
return self.clock
|
||||
|
||||
def log(self, line):
|
||||
self.logs.append(line)
|
||||
|
||||
def agent_uid(self):
|
||||
return 999
|
||||
|
||||
def verify_sig(self, signers, key_id, ns, blob, sig):
|
||||
self.verified = (signers, key_id, ns)
|
||||
return self.sig_rc
|
||||
|
||||
def read_nonces(self):
|
||||
return dict(self.nonces)
|
||||
|
||||
def write_nonces(self, d):
|
||||
self.nonces = dict(d)
|
||||
|
||||
def read_file(self, p):
|
||||
if p not in self.files:
|
||||
raise OSError("no such file")
|
||||
return self.files[p].decode()
|
||||
|
||||
def read_bytes(self, p):
|
||||
if p not in self.files:
|
||||
raise OSError("no such file")
|
||||
return self.files[p]
|
||||
|
||||
def read_staged_once(self, p, owner_uid, limit):
|
||||
if p not in self.files:
|
||||
raise OSError("no such file")
|
||||
if self.uids[p] != owner_uid:
|
||||
raise osapply.Refused("R19", f"{p} is not a regular file owned by felhom-agent")
|
||||
self.staged_reads = getattr(self, "staged_reads", 0) + 1
|
||||
return self.files[p]
|
||||
|
||||
def stat(self, p):
|
||||
if p not in self.files:
|
||||
raise OSError("no such file")
|
||||
return St(statmod.S_IFREG | self.modes[p], self.uids[p])
|
||||
|
||||
def lexists(self, p):
|
||||
return p in self.files
|
||||
|
||||
def isdir(self, p):
|
||||
return p in self.dirs
|
||||
|
||||
def put_file(self, p, data, mode):
|
||||
self.writes.append(p)
|
||||
self.put(p, data, mode)
|
||||
|
||||
def remove(self, p):
|
||||
self.writes.append("rm " + p)
|
||||
del self.files[p]
|
||||
|
||||
def list_dir(self, p):
|
||||
return sorted({k[len(p) + 1:].split("/")[0] for k in self.files if k.startswith(p + "/")})
|
||||
|
||||
def rmtree(self, p):
|
||||
for k in [k for k in self.files if k.startswith(p + "/")]:
|
||||
del self.files[k]
|
||||
|
||||
def check_content(self, kind, data):
|
||||
return (1, f"{kind}: syntax error") if b"BROKEN-SYNTAX" in data else (0, "")
|
||||
|
||||
def host(self, argv, timeout=600, stdin=None):
|
||||
self.calls.append(argv)
|
||||
if argv[:2] == ["visudo", "-c"]:
|
||||
return (1, "", "parse error") if self.visudo_fail else (0, "ok", "")
|
||||
if argv[:2] == ["sudo", "-n"]:
|
||||
return 0, self.sudo_l, ""
|
||||
if argv[-2:] == ["/usr/local/sbin/felhom-priv-apply", "--self-check"]:
|
||||
body = self.files.get("/usr/local/sbin/felhom-priv-apply", b"")
|
||||
return (0, "felhom-priv-apply ok verbs=unit\n", "") if b"VERBS" in body else (1, "", "boom")
|
||||
if argv[-1] == "--self-check":
|
||||
body = self.files.get("/usr/local/sbin/felhom-os-apply", b"")
|
||||
return (0, "felhom-os-apply ok bundle-format=1 files=22\n", "") if b"BUNDLE_OP" in body else (1, "", "boom")
|
||||
if argv[-1] == "/usr/local/sbin/felhom-selfupdate-guarded":
|
||||
return 2, "", "felhom-selfupdate-guarded: usage: ...\n"
|
||||
if argv[-1] == "status" and argv[0].endswith("felhom-crash-guard"):
|
||||
return 0, json.dumps(self.guard), ""
|
||||
if argv[:3] == ["systemctl", "enable", "--now"] and "felhom-crash-guard.service" in argv:
|
||||
self.put("/proc/sys/kernel/panic", f"{self.guard['kernel_panic']}\n".encode(), 0o644)
|
||||
return 0, "", ""
|
||||
|
||||
|
||||
def signed_job(sha, version="0.143.0", host=HOST, op="agent_config_update", nonce="b1",
|
||||
issued="2026-10-04T11:50:00Z", expires="2026-10-04T12:30:00Z"):
|
||||
blob = json.dumps({"expires_at": expires, "issued_at": issued, "key_id": "felhom-op-1", "nonce": nonce, "op": op,
|
||||
"params": {"agent_version": version, "bundle_sha256": sha},
|
||||
"target": {"guest_id": "", "host_id": host}}, sort_keys=True).encode()
|
||||
return {"blob_b64": base64.b64encode(blob).decode(), "sig": "-----BEGIN SSH SIGNATURE-----\nx\n-----END SSH SIGNATURE-----\n"}
|
||||
|
||||
|
||||
def real_bundle(version="0.143.0"):
|
||||
data = builder.build(version)
|
||||
return data, hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def edited_bundle(edit):
|
||||
"""The real bundle, with edit(files_list) applied and every sha recomputed — a SIGNED bundle with bad content."""
|
||||
b = json.loads(builder.build("0.143.0"))
|
||||
edit(b["files"])
|
||||
for e in b["files"]:
|
||||
e["sha256"] = hashlib.sha256(base64.b64decode(e["content_b64"])).hexdigest()
|
||||
data = (json.dumps(b, indent=1, sort_keys=True) + "\n").encode()
|
||||
return data, hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def replace_content(files, path, fn):
|
||||
for e in files:
|
||||
if e["path"] == path:
|
||||
e["content_b64"] = base64.b64encode(fn(base64.b64decode(e["content_b64"]))).decode()
|
||||
|
||||
|
||||
def run(box, argv=None, environ=None):
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
rc = osapply.main(argv or ["felhom-os-apply", "--plan", PLAN], runner=box, environ=environ or {})
|
||||
line = [l for l in buf.getvalue().splitlines() if l.startswith("OSAPPLY-REPORT ")][-1]
|
||||
return rc, json.loads(line[len("OSAPPLY-REPORT "):])
|
||||
|
||||
|
||||
def box_files(box):
|
||||
return {p: box.files[p] for p in box.files if p in osapply.BUNDLE_DESTS}
|
||||
|
||||
|
||||
class Install(unittest.TestCase):
|
||||
def test_fresh_box_gets_every_file_and_a_record(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
b = rep["bundle"]
|
||||
# every path but the four OOB ones (no belt on this box)
|
||||
self.assertEqual(len(b["written"]), len(osapply.BUNDLE_FILES) - 4, b)
|
||||
self.assertEqual(len(b["skipped"]), 4)
|
||||
self.assertEqual(box.files["/etc/sudoers.d/felhom-agent"], (REPO / "configs" / "felhom-agent.sudoers").read_bytes())
|
||||
self.assertEqual(box.modes["/etc/sudoers.d/felhom-agent"], 0o440)
|
||||
rec = json.loads(box.files[osapply.BUNDLE_RECORD])
|
||||
self.assertEqual((rec["agent_version"], rec["bundle_sha256"], rec["authority"]), ("0.143.0", sha, "signed"))
|
||||
self.assertIn("b1", box.nonces, "the job is consumed")
|
||||
self.assertEqual(b["self_check"]["crash_guard"], {"armed": True, "kernel_panic": 10})
|
||||
self.assertIn(["systemctl", "daemon-reload"], box.calls)
|
||||
|
||||
def test_sudoers_is_written_after_every_wrapper(self):
|
||||
"""Order: a referenced wrapper is in place before the sudoers line that allows it."""
|
||||
data, sha = edited_bundle(lambda f: f.reverse()) # the bundle lists the sudoers FIRST; the wrapper must reorder
|
||||
box = Box(data, signed_job(sha))
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
w = [p for p in box.writes if p in osapply.BUNDLE_DESTS]
|
||||
self.assertEqual(w[-1], "/etc/sudoers.d/felhom-agent")
|
||||
self.assertLess(w.index("/usr/local/sbin/felhom-os-apply"), w.index("/etc/sudoers.d/felhom-agent"))
|
||||
|
||||
def test_identical_box_writes_nothing(self):
|
||||
"""The demo boxes' case: hand-copied files equal to the release → 0 written, all 'same'."""
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
for dest, src, mode, _, policy in osapply.BUNDLE_FILES:
|
||||
if policy != "oob":
|
||||
box.put(dest, (REPO / "configs" / src).read_bytes(), mode)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(rep["bundle"]["written"], [])
|
||||
self.assertEqual(rep["bundle"]["same"], len(osapply.BUNDLE_FILES) - 5) # 4 oob skipped + crash-guard.conf kept
|
||||
self.assertEqual(rep["bundle"]["kept"], ["/etc/felhom/crash-guard.conf"])
|
||||
|
||||
def test_a_wrong_mode_is_rewritten(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.put("/etc/sudoers.d/felhom-agent", (REPO / "configs" / "felhom-agent.sudoers").read_bytes(), 0o644)
|
||||
rc, rep = run(box)
|
||||
self.assertIn("/etc/sudoers.d/felhom-agent", rep["bundle"]["written"])
|
||||
self.assertEqual(box.modes["/etc/sudoers.d/felhom-agent"], 0o440)
|
||||
|
||||
def test_tuned_crash_guard_conf_is_kept(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.put("/etc/felhom/crash-guard.conf", b"LIMIT=5\n", 0o644)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(box.files["/etc/felhom/crash-guard.conf"], b"LIMIT=5\n")
|
||||
|
||||
def test_oob_files_only_on_a_box_with_the_belt(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha), oob=True)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertIn("/etc/sudoers.d/felhom-op", rep["bundle"]["written"])
|
||||
self.assertEqual(rep["bundle"]["skipped"], [])
|
||||
|
||||
def test_previous_copies_are_kept(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.put("/usr/local/sbin/felhom-pbs-apply", b"#!/bin/bash\necho old\n", 0o755)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(box.files[rep["bundle"]["prev_dir"] + "/usr/local/sbin/felhom-pbs-apply"], b"#!/bin/bash\necho old\n")
|
||||
|
||||
|
||||
class Refusals(unittest.TestCase):
|
||||
"""Each: refused, and NOTHING on the box changed."""
|
||||
|
||||
def refused(self, box, code):
|
||||
before = dict(box_files(box))
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 2, rep)
|
||||
self.assertEqual(rep["refused"]["code"], code, rep)
|
||||
self.assertEqual(box_files(box), before, "a refusal changed a file")
|
||||
self.assertNotIn(osapply.BUNDLE_RECORD, box.files)
|
||||
return rep
|
||||
|
||||
def test_wrong_sha_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job("0" * 64))
|
||||
self.refused(box, "R18")
|
||||
self.assertEqual(box.nonces, {}, "a wrong sha must not burn the job")
|
||||
|
||||
def test_bad_signature_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.sig_rc = 255
|
||||
self.refused(box, "R3")
|
||||
|
||||
def test_other_op_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
self.refused(Box(data, signed_job(sha, op="agent_update")), "R3")
|
||||
|
||||
def test_other_host_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
self.refused(Box(data, signed_job(sha, host="demo-felhom-8363b5")), "R3")
|
||||
|
||||
def test_expired_job_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
self.refused(Box(data, signed_job(sha, expires="2026-10-04T11:55:00Z")), "R3")
|
||||
|
||||
def test_replayed_job_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.nonces = {"b1": box.clock + 600}
|
||||
self.refused(box, "R3")
|
||||
|
||||
def test_version_mismatch_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
self.refused(Box(data, signed_job(sha, version="0.142.1")), "R18")
|
||||
|
||||
def test_sudoers_failing_visudo_is_refused(self):
|
||||
data, sha = edited_bundle(lambda f: replace_content(f, "/etc/sudoers.d/felhom-agent", lambda c: c + b"BROKEN-SYNTAX\n"))
|
||||
self.refused(Box(data, signed_job(sha)), "R18")
|
||||
|
||||
def test_sudoers_dropping_the_route_is_refused(self):
|
||||
data, sha = edited_bundle(lambda f: replace_content(f, "/etc/sudoers.d/felhom-agent",
|
||||
lambda c: c.replace(b"/usr/local/sbin/felhom-os-apply --plan", b"/bin/true --plan")))
|
||||
self.refused(Box(data, signed_job(sha)), "R18")
|
||||
|
||||
def test_wrapper_without_bundle_mode_is_refused(self):
|
||||
data, sha = edited_bundle(lambda f: replace_content(f, "/usr/local/sbin/felhom-os-apply",
|
||||
lambda c: c.replace(b'BUNDLE_OP = "agent_config_update"', b'BUNDLE_OPX = 1')))
|
||||
self.refused(Box(data, signed_job(sha)), "R18")
|
||||
|
||||
def test_python_syntax_error_is_refused(self):
|
||||
data, sha = edited_bundle(lambda f: replace_content(f, "/usr/local/sbin/felhom-crash-guard", lambda c: c + b"\ndef (\n"))
|
||||
self.refused(Box(data, signed_job(sha)), "R18")
|
||||
|
||||
def test_unit_with_runtime_directory_is_refused(self):
|
||||
data, sha = edited_bundle(lambda f: replace_content(f, "/etc/systemd/system/felhom-mgmt-watchdog.service",
|
||||
lambda c: c + b"RuntimeDirectory=sshd\n"))
|
||||
self.refused(Box(data, signed_job(sha)), "R18")
|
||||
|
||||
def test_agent_unit_not_as_the_agent_user_is_refused(self):
|
||||
data, sha = edited_bundle(lambda f: replace_content(f, "/etc/systemd/system/felhom-agent.service",
|
||||
lambda c: c.replace(b"User=felhom-agent", b"User=root")))
|
||||
self.refused(Box(data, signed_job(sha)), "R18")
|
||||
|
||||
def test_a_bundle_that_changes_a_signer_is_refused(self):
|
||||
"""R17: the trust root is not a bundle's to change — not even a signed one."""
|
||||
def add(f):
|
||||
f.append({"path": osapply.TRUST_SIGNERS, "content_b64": base64.b64encode(b"evil-key\n").decode()})
|
||||
data, sha = edited_bundle(add)
|
||||
box = Box(data, signed_job(sha))
|
||||
self.refused(box, "R17")
|
||||
self.assertIn(b"felhom-op-1", box.files[osapply.TRUST_SIGNERS])
|
||||
|
||||
def test_a_path_outside_the_table_is_refused(self):
|
||||
def add(f):
|
||||
f.append({"path": "/etc/shadow", "content_b64": base64.b64encode(b"root::0:0\n").decode()})
|
||||
data, sha = edited_bundle(add)
|
||||
self.refused(Box(data, signed_job(sha)), "R16")
|
||||
|
||||
def test_content_not_matching_its_sha_is_refused(self):
|
||||
b = json.loads(builder.build("0.143.0"))
|
||||
b["files"][0]["content_b64"] = base64.b64encode(b"#!/bin/bash\nexit 0\n").decode()
|
||||
data = json.dumps(b).encode()
|
||||
self.refused(Box(data, signed_job(hashlib.sha256(data).hexdigest())), "R18")
|
||||
|
||||
def test_bundle_outside_the_plan_dir_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.plan["bundle"] = "/tmp/bundle-0.143.0.json"
|
||||
box.files[PLAN] = json.dumps(box.plan).encode()
|
||||
self.refused(box, "R1")
|
||||
|
||||
def test_bundle_not_owned_by_the_agent_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.uids[BUNDLE] = 0
|
||||
self.refused(box, "R1")
|
||||
|
||||
def test_no_trust_file_is_refused(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
del box.files[osapply.TRUST_FILE]
|
||||
self.refused(box, "R3")
|
||||
|
||||
|
||||
class SelfCheckUndo(unittest.TestCase):
|
||||
def assert_restored(self, box, before, rep):
|
||||
self.assertTrue(rep["bundle"]["rolled_back"], rep)
|
||||
self.assertEqual(box_files(box), before, "the previous files must be back, byte for byte")
|
||||
self.assertNotIn(osapply.BUNDLE_RECORD, box.files)
|
||||
|
||||
def with_old_files(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.put("/usr/local/sbin/felhom-pbs-apply", b"#!/bin/bash\necho old\n", 0o755)
|
||||
box.put("/etc/sudoers.d/felhom-agent", b"# old sudoers\n", 0o440)
|
||||
return box
|
||||
|
||||
def test_route_missing_after_install_puts_everything_back(self):
|
||||
box = self.with_old_files()
|
||||
before = dict(box_files(box))
|
||||
box.sudo_l = " (root) NOPASSWD: /bin/true\n"
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 3, rep)
|
||||
self.assert_restored(box, before, rep)
|
||||
self.assertEqual(box.calls[-1], ["visudo", "-c"], "the undo re-checks the whole sudoers")
|
||||
|
||||
def test_visudo_failing_after_install_puts_everything_back(self):
|
||||
box = self.with_old_files()
|
||||
before = dict(box_files(box))
|
||||
box.visudo_fail = True
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 3, rep)
|
||||
self.assert_restored(box, before, rep)
|
||||
|
||||
def test_crash_guard_disagreeing_with_kernel_panic_puts_everything_back(self):
|
||||
box = self.with_old_files()
|
||||
before = dict(box_files(box))
|
||||
box.guard = {"armed": True, "kernel_panic": 10}
|
||||
box.host_orig = box.host
|
||||
|
||||
def host(argv, timeout=600, stdin=None):
|
||||
if argv[:3] == ["systemctl", "enable", "--now"]:
|
||||
box.calls.append(argv)
|
||||
return 0, "", "" # the unit "started" but kernel.panic stayed 0
|
||||
return box.host_orig(argv, timeout, stdin)
|
||||
box.host = host
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 3, rep)
|
||||
self.assert_restored(box, before, rep)
|
||||
|
||||
|
||||
|
||||
class RootKeyOnly(unittest.TestCase):
|
||||
"""R-982, operator ruling D4 (2026-10-22): the bundle gives the host's own sshd a root-key-only drop-in, tests the
|
||||
whole config and reloads sshd — only when the file was written; a drop-in that says anything but
|
||||
`prohibit-password` is refused before anything is written.
|
||||
RED-PROOF: with the after_install block removed, test_fresh_box_takes_the_dropin_and_reloads fails (no reload);
|
||||
with the content rule removed, test_a_dropin_that_says_yes_or_no_is_refused fails — seen failing 2026-10-22."""
|
||||
DEST = "/etc/ssh/sshd_config.d/10-felhom-root-key-only.conf"
|
||||
|
||||
def test_fresh_box_takes_the_dropin_and_reloads(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertIn(b"PermitRootLogin prohibit-password", box.files[self.DEST])
|
||||
self.assertEqual(box.modes[self.DEST], 0o644)
|
||||
self.assertIn(["/usr/sbin/sshd", "-t"], box.calls)
|
||||
self.assertIn(["systemctl", "try-reload-or-restart", "ssh"], box.calls)
|
||||
self.assertLess(box.calls.index(["/usr/sbin/sshd", "-t"]), box.calls.index(["systemctl", "try-reload-or-restart", "ssh"]))
|
||||
self.assertFalse(any("felhom-sshd" in " ".join(c) for c in box.calls), "the operator's door must not be touched")
|
||||
|
||||
def test_an_unchanged_dropin_is_not_reloaded(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.put(self.DEST, (REPO / "configs" / "felhom-root-ssh-key-only.conf").read_bytes(), 0o644)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertNotIn(["systemctl", "try-reload-or-restart", "ssh"], box.calls)
|
||||
|
||||
def test_a_dropin_that_says_yes_or_no_is_refused(self):
|
||||
for bad in (b"PermitRootLogin yes\n", b"PermitRootLogin no\n", b"# nothing\n",
|
||||
b"PermitRootLogin prohibit-password\nPermitRootLogin yes\n"):
|
||||
def edit(files, bad=bad):
|
||||
for e in files:
|
||||
if e["path"] == self.DEST:
|
||||
e["content_b64"] = base64.b64encode(bad).decode()
|
||||
data, sha = edited_bundle(edit)
|
||||
box = Box(data, signed_job(sha))
|
||||
rc, rep = run(box)
|
||||
self.assertNotEqual(rc, 0, bad)
|
||||
self.assertNotIn(self.DEST, box.files, bad)
|
||||
|
||||
# Review 2026-10-22 night. A2: the effective value is read back with `sshd -T`; a drop-in that is not in force is
|
||||
# said in the log and the report (not refused — a refusal would freeze the bundle route). A3: a bundle undone
|
||||
# after the reload reloads sshd again with the old state. RED-PROOF: with the sshd -T check removed,
|
||||
# test_a_dropin_not_in_force_is_reported fails; with the undo reload removed, test_an_undone_bundle_reloads_sshd
|
||||
# fails — seen failing 2026-10-22 night.
|
||||
def _sshd_T(self, box, value):
|
||||
orig = box.host
|
||||
def host(argv, timeout=600, stdin=None):
|
||||
if argv[:2] == ["/usr/sbin/sshd", "-T"]:
|
||||
box.calls.append(argv)
|
||||
return 0, f"port 22\npermitrootlogin {value}\npasswordauthentication yes\n", ""
|
||||
return orig(argv, timeout, stdin)
|
||||
box.host = host
|
||||
|
||||
def test_the_effective_value_is_checked(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
self._sshd_T(box, "without-password")
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertTrue(any(c[:2] == ["/usr/sbin/sshd", "-T"] for c in box.calls))
|
||||
self.assertNotIn("sshd_dropin_not_in_force", rep["bundle"])
|
||||
|
||||
def test_a_dropin_not_in_force_is_reported(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
self._sshd_T(box, "yes") # e.g. no Include of sshd_config.d
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertTrue(rep["bundle"].get("sshd_dropin_not_in_force"), rep)
|
||||
self.assertTrue(any("NOT in force" in l for l in box.logs), box.logs)
|
||||
|
||||
def test_an_undone_bundle_reloads_sshd(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.visudo_fail = True # the self-check fails AFTER the install → undo
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 3, rep)
|
||||
self.assertNotIn(self.DEST, box.files)
|
||||
reloads = [i for i, c in enumerate(box.calls) if c == ["systemctl", "try-reload-or-restart", "ssh"]]
|
||||
self.assertEqual(len(reloads), 2, box.calls) # after_install, then the undo
|
||||
|
||||
class ByoKeepsItsSSH(unittest.TestCase):
|
||||
"""D43 = A (operator 2026-10-24, R-1257): the root-key-only drop-in goes ONLY to a box whose root-owned install record
|
||||
says appliance; on a BYO host nothing is written and a copy WE wrote is removed; a box with no record is left as it
|
||||
is. X3: the in-force check asks sshd about each of the host's own LAN addresses. X4: a failing `sshd -t` puts back
|
||||
only the SSH step. RED-PROOF (2026-10-24, felhom-agent f5c3d77 + these tests): with the drop-in's policy back to
|
||||
`replace`, test_a_byo_host_gets_no_dropin and test_our_copy_comes_off_a_byo_host fail; with the probe back to
|
||||
192.0.2.1 only, test_a_match_block_for_the_lan_is_not_in_force fails; with `sshd -t` back to `must`,
|
||||
test_a_failing_sshd_t_undoes_only_the_ssh_step fails (rc 3, the whole bundle undone)."""
|
||||
DEST = "/etc/ssh/sshd_config.d/10-felhom-root-key-only.conf"
|
||||
OURS = (REPO / "configs" / "felhom-root-ssh-key-only.conf").read_bytes()
|
||||
|
||||
def test_a_byo_host_gets_no_dropin(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha), install_mode="byo")
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertNotIn(self.DEST, box.files)
|
||||
self.assertNotIn(["/usr/sbin/sshd", "-t"], box.calls)
|
||||
self.assertNotIn(["systemctl", "try-reload-or-restart", "ssh"], box.calls)
|
||||
self.assertIn(self.DEST, rep["bundle"]["skipped"])
|
||||
rec = json.loads(box.files[osapply.BUNDLE_RECORD])
|
||||
self.assertEqual(rec["files"][self.DEST], "skipped")
|
||||
|
||||
def test_our_copy_comes_off_a_byo_host(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha), install_mode="byo")
|
||||
box.put(self.DEST, self.OURS, 0o644) # an earlier bundle wrote it (before D43)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertNotIn(self.DEST, box.files)
|
||||
self.assertEqual(rep["bundle"]["removed"], [self.DEST])
|
||||
self.assertTrue(any(k.endswith(self.DEST) and k.startswith(osapply.BUNDLE_PREV_DIR) for k in box.files),
|
||||
"the previous copy must be kept for the undo")
|
||||
self.assertLess(box.calls.index(["/usr/sbin/sshd", "-t"]), box.calls.index(["systemctl", "try-reload-or-restart", "ssh"]))
|
||||
self.assertEqual(json.loads(box.files[osapply.BUNDLE_RECORD])["files"][self.DEST], "removed")
|
||||
|
||||
def test_the_owners_own_file_stays_on_a_byo_host(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha), install_mode="byo")
|
||||
box.put(self.DEST, b"# my own\nPermitRootLogin yes\n", 0o644)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(box.files[self.DEST], b"# my own\nPermitRootLogin yes\n")
|
||||
self.assertNotIn(["systemctl", "try-reload-or-restart", "ssh"], box.calls)
|
||||
|
||||
def test_a_box_with_no_record_is_left_as_it_is(self):
|
||||
for setup in ("none", "agent-writable"):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha), install_mode=None if setup == "none" else "byo")
|
||||
if setup == "agent-writable":
|
||||
box.modes[osapply.INSTALL_STATE] = 0o666 # proves nothing
|
||||
box.put(self.DEST, self.OURS, 0o644)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(box.files.get(self.DEST), self.OURS, setup)
|
||||
self.assertIn(self.DEST, rep["bundle"]["skipped"], setup)
|
||||
|
||||
def _fake(self, box, addrs=("192.168.0.50",), match=None, sshd_t_rc=0):
|
||||
orig = box.host
|
||||
def host(argv, timeout=600, stdin=None):
|
||||
if argv[:3] == ["ip", "-4", "-o"]:
|
||||
box.calls.append(argv)
|
||||
return 0, "".join(f"2: vmbr0 inet {a}/24 brd x scope global vmbr0\\ valid_lft forever\n" for a in addrs), ""
|
||||
if argv[:2] == ["/usr/sbin/sshd", "-T"]:
|
||||
box.calls.append(argv)
|
||||
val = "yes" if match and f"addr={match}" in argv[-1] else "without-password"
|
||||
return 0, f"port 22\npermitrootlogin {val}\n", ""
|
||||
if argv == ["/usr/sbin/sshd", "-t"]:
|
||||
box.calls.append(argv)
|
||||
return (sshd_t_rc, "", "/etc/ssh/sshd_config line 9: Bad configuration option") if sshd_t_rc else (0, "", "")
|
||||
return orig(argv, timeout, stdin)
|
||||
box.host = host
|
||||
|
||||
# R-1285: the policy pass the agent runs right after every bundle install, with the wrapper NOW on disk. No
|
||||
# signature (it only removes a file we wrote). RED-PROOF: make policy_pass return before the loop → our copy stays.
|
||||
POLICY_PLAN = {"release_id": "bundle-policy-0.166.0", "layer": "host", "mode": "bundle-policy"}
|
||||
|
||||
def test_the_policy_pass_removes_our_copy_from_a_byo_host(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, None, install_mode="byo")
|
||||
box.put(PLAN, json.dumps(self.POLICY_PLAN).encode(), 0o600, uid=999)
|
||||
box.put(self.DEST, self.OURS, 0o644) # the old wrapper's bundle run left it
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertNotIn(self.DEST, box.files)
|
||||
self.assertEqual(rep["bundle_policy"], {"install_mode": "byo", "removed": [self.DEST]}, rep)
|
||||
self.assertTrue(any(k.endswith(self.DEST) and k.startswith(osapply.BUNDLE_PREV_DIR) for k in box.files))
|
||||
self.assertLess(box.calls.index(["/usr/sbin/sshd", "-t"]), box.calls.index(["systemctl", "try-reload-or-restart", "ssh"]))
|
||||
|
||||
def test_the_policy_pass_leaves_an_appliance_and_the_owners_file(self):
|
||||
for mode, content in (("appliance", None), ("byo", b"# my own\nPermitRootLogin yes\n")):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, None, install_mode=mode)
|
||||
box.put(PLAN, json.dumps(self.POLICY_PLAN).encode(), 0o600, uid=999)
|
||||
body = content if content is not None else self.OURS
|
||||
box.put(self.DEST, body, 0o644)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(box.files.get(self.DEST), body, mode)
|
||||
self.assertEqual(rep["bundle_policy"]["removed"], [], mode)
|
||||
self.assertNotIn(["systemctl", "try-reload-or-restart", "ssh"], box.calls, mode)
|
||||
|
||||
def test_the_policy_pass_puts_our_copy_back_when_sshd_t_fails(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, None, install_mode="byo")
|
||||
box.put(PLAN, json.dumps(self.POLICY_PLAN).encode(), 0o600, uid=999)
|
||||
box.put(self.DEST, self.OURS, 0o644)
|
||||
self._fake(box, sshd_t_rc=1)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(box.files.get(self.DEST), self.OURS)
|
||||
self.assertEqual(rep["bundle_policy"]["removed"], [])
|
||||
self.assertNotIn(["systemctl", "try-reload-or-restart", "ssh"], box.calls)
|
||||
|
||||
def test_a_match_block_for_the_lan_is_not_in_force(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
self._fake(box, addrs=("192.168.0.50", "10.77.0.4"), match="192.168.0.50")
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
probed = [c[-1] for c in box.calls if c[:2] == ["/usr/sbin/sshd", "-T"]]
|
||||
self.assertTrue(any("addr=192.168.0.50" in p for p in probed) and any("addr=10.77.0.4" in p for p in probed), probed)
|
||||
self.assertFalse(any("addr=192.0.2.1" in p for p in probed), probed)
|
||||
self.assertTrue(rep["bundle"].get("sshd_dropin_not_in_force"), rep)
|
||||
self.assertTrue(any("NOT in force for 192.168.0.50" in l for l in box.logs), box.logs)
|
||||
|
||||
def test_the_lan_check_passes_when_every_address_is_key_only(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
self._fake(box, addrs=("192.168.0.50", "10.77.0.4"))
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertNotIn("sshd_dropin_not_in_force", rep["bundle"])
|
||||
|
||||
def test_a_failing_sshd_t_undoes_only_the_ssh_step(self):
|
||||
for before in (None, b"# MANAGED BY FELHOM (the agent's config bundle) \xe2\x80\x94 R-982 old\nPermitRootLogin prohibit-password\n"):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
if before is not None:
|
||||
box.put(self.DEST, before, 0o644)
|
||||
self._fake(box, sshd_t_rc=255)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep) # the bundle stays
|
||||
self.assertEqual(box.files.get(self.DEST), before, "only the SSH step is put back")
|
||||
self.assertIn("/usr/local/sbin/felhom-os-apply", box.files, "the rest of the bundle stays")
|
||||
self.assertIn("sshd -t failed", rep["bundle"].get("sshd_step_undone", ""), rep)
|
||||
self.assertNotIn(["systemctl", "try-reload-or-restart", "ssh"], box.calls, "sshd never ran the refused state")
|
||||
want = "absent" if before is None else hashlib.sha256(before).hexdigest()
|
||||
self.assertEqual(json.loads(box.files[osapply.BUNDLE_RECORD])["files"][self.DEST], want)
|
||||
|
||||
|
||||
class TrustBootstrap(unittest.TestCase):
|
||||
def test_missing_signers_verifies_against_the_pinned_key_and_creates_it(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha), signers=False)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(box.verified[0], osapply.PINNED_SIGNERS, "verified against the pinned key, nothing else")
|
||||
self.assertTrue(rep["bundle"]["signers_created"])
|
||||
self.assertEqual(box.files[osapply.TRUST_SIGNERS], osapply.pinned_signers_line().encode())
|
||||
self.assertEqual(box.modes[osapply.TRUST_SIGNERS], 0o644)
|
||||
|
||||
def test_missing_signers_and_a_job_the_pinned_key_did_not_sign(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha), signers=False)
|
||||
box.sig_rc = 255
|
||||
rc, rep = run(box)
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R3"))
|
||||
self.assertNotIn(osapply.TRUST_SIGNERS, box.files)
|
||||
|
||||
def test_present_signers_are_never_touched(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.put(osapply.TRUST_SIGNERS, b'felhom-op-2 namespaces="felhom-op-v1" ssh-ed25519 BBBB felhom-op-2\n', 0o644)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(box.verified[0], osapply.TRUST_SIGNERS)
|
||||
self.assertFalse(rep["bundle"]["signers_created"])
|
||||
self.assertIn(b"felhom-op-2", box.files[osapply.TRUST_SIGNERS])
|
||||
|
||||
def test_agent_writable_signers_are_refused(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
box.uids[osapply.TRUST_SIGNERS] = 999
|
||||
rc, rep = run(box)
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R3"))
|
||||
|
||||
def test_pinned_operator_key_equals_the_installers(self):
|
||||
"""The bootstrap key is exactly the one felhom-host-install.sh pins (OPERATOR_KEY_OPERATIONAL_*)."""
|
||||
text = INSTALLER.read_text()
|
||||
kid = re.search(r'^OPERATOR_KEY_OPERATIONAL_ID="([^"]+)"', text, re.M).group(1)
|
||||
line = re.search(r'^OPERATOR_KEY_OPERATIONAL_LINE="([^"]+)"', text, re.M).group(1)
|
||||
self.assertEqual((osapply.PINNED_OPERATOR_KEY_ID, osapply.PINNED_OPERATOR_KEY_LINE), (kid, line))
|
||||
# and the file format is the installer's printf, byte for byte
|
||||
self.assertIn("printf '%s namespaces=\"felhom-op-v1\" %s\\n'", text)
|
||||
|
||||
|
||||
class InstallerEntry(unittest.TestCase):
|
||||
def test_installer_installs_without_a_signature(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, None)
|
||||
box.put("/root/bundle.json", data, 0o600)
|
||||
rc, rep = run(box, ["felhom-os-apply", "--install-bundle", "/root/bundle.json", "--sha256", sha])
|
||||
self.assertEqual(rc, 0, rep)
|
||||
self.assertEqual(json.loads(box.files[osapply.BUNDLE_RECORD])["authority"], "installer")
|
||||
|
||||
def test_installer_entry_checks_the_sha(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, None)
|
||||
box.put("/root/bundle.json", data, 0o600)
|
||||
rc, rep = run(box, ["felhom-os-apply", "--install-bundle", "/root/bundle.json", "--sha256", "1" * 64])
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R18"))
|
||||
|
||||
def test_installer_entry_is_refused_through_sudo(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, None)
|
||||
box.put("/root/bundle.json", data, 0o600)
|
||||
rc, rep = run(box, ["felhom-os-apply", "--install-bundle", "/root/bundle.json", "--sha256", sha], {"SUDO_UID": "999"})
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R1"))
|
||||
self.assertNotIn(osapply.BUNDLE_RECORD, box.files)
|
||||
|
||||
def test_self_check_answers(self):
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
rc = osapply.main(["felhom-os-apply", "--self-check"], runner=Box(b"", None))
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertIn("bundle-format=1", buf.getvalue())
|
||||
|
||||
|
||||
class Facts(unittest.TestCase):
|
||||
def test_state_reports_drift_against_the_record(self):
|
||||
data, sha = real_bundle()
|
||||
box = Box(data, signed_job(sha))
|
||||
run(box)
|
||||
box.put("/usr/local/sbin/felhom-pbs-apply", b"#!/bin/bash\necho by hand\n", 0o755)
|
||||
a = osapply.Apply(box, PLAN)
|
||||
st = osapply.Bundle(a).state()
|
||||
self.assertEqual(st["version"], "0.143.0")
|
||||
self.assertEqual(st["drift"], ["/usr/local/sbin/felhom-pbs-apply"])
|
||||
self.assertTrue(st["signers_present"])
|
||||
|
||||
def test_state_without_a_record_says_none(self):
|
||||
box = Box(b"", None)
|
||||
st = osapply.Bundle(osapply.Apply(box, PLAN)).state()
|
||||
self.assertEqual(st["version"], "none")
|
||||
self.assertNotIn("drift", st)
|
||||
self.assertEqual(st["live"]["/etc/sudoers.d/felhom-agent"], "absent")
|
||||
|
||||
|
||||
class Builder(unittest.TestCase):
|
||||
def test_reproducible(self):
|
||||
self.assertEqual(builder.build("0.143.0"), builder.build("0.143.0"))
|
||||
|
||||
def test_every_source_exists_and_every_dest_is_unique(self):
|
||||
dests = [e[0] for e in osapply.BUNDLE_FILES]
|
||||
self.assertEqual(len(dests), len(set(dests)))
|
||||
for _, src, *_ in osapply.BUNDLE_FILES:
|
||||
self.assertTrue((REPO / "configs" / src).is_file(), src)
|
||||
|
||||
def test_every_root_file_the_installer_writes_is_in_the_bundle(self):
|
||||
"""One source of truth: a felhom root-owned path the installer names must be a bundle path, a trust file, or a
|
||||
path the AGENT itself writes at run time (named here, with why). Scope is a regex over the WHOLE installer."""
|
||||
text = INSTALLER.read_text()
|
||||
found = set(re.findall(r"(/usr/local/sbin/felhom-[a-z-]+|/etc/systemd/system/felhom-[a-z.-]+|"
|
||||
r"/etc/sudoers\.d/felhom-[a-z-]+|/etc/felhom-oob\.nft|/etc/tmpfiles\.d/felhom-[a-z.-]+|"
|
||||
r"/etc/felhom/[a-z.-]+)", text))
|
||||
agent_writes = {"/usr/local/sbin/felhom-shared-parent", "/etc/systemd/system/felhom-shared-parent.service"}
|
||||
trust = {osapply.TRUST_FILE, osapply.TRUST_SIGNERS, osapply.TRUST_SIGNERS + ".tmp", osapply.BUNDLE_RECORD}
|
||||
# Written by the appliance ISO's first boot (felhom.eu scripts/iso/felhom-bootstrap.sh), never by the installer:
|
||||
# since installer 1.32.0 (R-275) the uninstall only NAMES them under KEPT.
|
||||
iso_writes = {"/etc/felhom/.bootstrap-done", "/etc/felhom/appliance-pairing-code"}
|
||||
missing = sorted(p for p in found if p not in osapply.BUNDLE_DESTS and p not in agent_writes | trust | iso_writes)
|
||||
self.assertEqual(missing, [], "the installer writes these root files, but the bundle does not carry them")
|
||||
# the limits drop-in is named through $AGENT_UNIT in the installer
|
||||
self.assertIn("/etc/systemd/system/felhom-agent.service.d/felhom-agent-limits.conf", osapply.BUNDLE_DESTS)
|
||||
|
||||
|
||||
|
||||
# ---------- R-861 (agent v0.146.0): the agent binary only by an operator-signed agent_update, checked as root ----------
|
||||
STAGED = "/var/lib/felhom-agent/selfupdate/felhom-agent-0.146.0"
|
||||
NEW_BIN = b"\x7fELF the new agent"
|
||||
|
||||
|
||||
def update_job(sha, version="0.146.0", op="agent_update", nonce="u1", host=HOST):
|
||||
blob = json.dumps({"expires_at": "2026-10-04T12:30:00Z", "issued_at": "2026-10-04T11:50:00Z", "key_id": "felhom-op-1",
|
||||
"nonce": nonce, "op": op, "params": {"sha256": sha, "version": version},
|
||||
"target": {"guest_id": "", "host_id": host}}, sort_keys=True).encode()
|
||||
return {"blob_b64": base64.b64encode(blob).decode(), "sig": "-----BEGIN SSH SIGNATURE-----\nx\n-----END SSH SIGNATURE-----\n"}
|
||||
|
||||
|
||||
def update_box(job, staged=STAGED, content=NEW_BIN):
|
||||
box = Box(b"{}", None)
|
||||
box.put(PLAN, json.dumps({"release_id": "agent-0.146.0", "layer": "host", "mode": "agent_update",
|
||||
"signed": job, "staged": staged}).encode(), 0o600, uid=999)
|
||||
box.put(STAGED, content, 0o755, uid=999)
|
||||
return box
|
||||
|
||||
|
||||
def wrapper_calls(box):
|
||||
return [c for c in box.calls if c and c[0] == osapply.SELFUPDATE_WRAPPER and c[1:2] == ["apply"]]
|
||||
|
||||
|
||||
class AgentUpdate(unittest.TestCase):
|
||||
"""RED-PROOF: make agent_update skip verify_signed → test_a_bad_signature_never_reaches_the_wrapper fails."""
|
||||
|
||||
def test_signed_update_flips_and_burns_the_nonce(self):
|
||||
sha = hashlib.sha256(NEW_BIN).hexdigest()
|
||||
box = update_box(update_job(sha))
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 0, rep)
|
||||
root_copy = osapply.SELFUPDATE_ROOT_DIR + "/felhom-agent-0.146.0"
|
||||
# the wrapper gets the ROOT-OWNED copy of the bytes that were hashed — never the agent's path (review 2026-10-05)
|
||||
self.assertEqual(wrapper_calls(box), [[osapply.SELFUPDATE_WRAPPER, "apply", root_copy, sha]])
|
||||
self.assertIn(root_copy, box.writes)
|
||||
self.assertNotIn(root_copy, box.files, "the root copy is removed after the flip")
|
||||
self.assertEqual(box.staged_reads, 1, "the agent's file is read exactly once")
|
||||
self.assertIn("u1", box.nonces)
|
||||
self.assertEqual(rep["agent_update"]["version"], "0.146.0")
|
||||
|
||||
def test_a_staged_file_the_agent_does_not_own_is_refused(self):
|
||||
sha = hashlib.sha256(NEW_BIN).hexdigest()
|
||||
box = update_box(update_job(sha))
|
||||
box.uids[STAGED] = 0
|
||||
rc, rep = run(box)
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R19"), rep)
|
||||
self.assertEqual(wrapper_calls(box), [])
|
||||
|
||||
def test_a_bad_signature_never_reaches_the_wrapper(self):
|
||||
sha = hashlib.sha256(NEW_BIN).hexdigest()
|
||||
box = update_box(update_job(sha))
|
||||
box.sig_rc = 1
|
||||
rc, rep = run(box)
|
||||
self.assertTrue(rep.get("refused"), f"a job whose signature does not verify was NOT refused: {rep}")
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R3"), rep)
|
||||
self.assertEqual(wrapper_calls(box), [])
|
||||
|
||||
def test_a_staged_binary_the_signature_does_not_pin_is_refused(self):
|
||||
sha = hashlib.sha256(NEW_BIN).hexdigest()
|
||||
box = update_box(update_job(sha), content=b"\x7fELF something the agent put there")
|
||||
rc, rep = run(box)
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R19"), rep)
|
||||
self.assertEqual(wrapper_calls(box), [])
|
||||
self.assertNotIn("u1", box.nonces, "a refused job keeps its nonce (the operator fixes the file, not the key)")
|
||||
|
||||
def test_another_staging_path_is_refused(self):
|
||||
sha = hashlib.sha256(NEW_BIN).hexdigest()
|
||||
box = update_box(update_job(sha), staged="/tmp/felhom-agent-0.146.0")
|
||||
rc, rep = run(box)
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R19"), rep)
|
||||
self.assertEqual(wrapper_calls(box), [])
|
||||
|
||||
def test_a_bundle_job_is_not_an_agent_update(self):
|
||||
sha = hashlib.sha256(NEW_BIN).hexdigest()
|
||||
box = update_box(update_job(sha, op="agent_config_update"))
|
||||
rc, rep = run(box)
|
||||
self.assertEqual((rc, rep["refused"]["code"]), (2, "R3"), rep)
|
||||
|
||||
def test_another_hosts_job_and_a_replay_are_refused(self):
|
||||
sha = hashlib.sha256(NEW_BIN).hexdigest()
|
||||
box = update_box(update_job(sha, host="tester-1-d70be4"))
|
||||
self.assertEqual(run(box)[1]["refused"]["code"], "R3")
|
||||
box2 = update_box(update_job(sha))
|
||||
box2.nonces["u1"] = 1999999999
|
||||
self.assertEqual(run(box2)[1]["refused"]["code"], "R3")
|
||||
self.assertEqual(wrapper_calls(box) + wrapper_calls(box2), [])
|
||||
|
||||
def test_a_failed_flip_keeps_the_nonce(self):
|
||||
sha = hashlib.sha256(NEW_BIN).hexdigest()
|
||||
box = update_box(update_job(sha))
|
||||
orig = box.host
|
||||
box.host = lambda argv, timeout=600, stdin=None: (1, "", "refusing apply: sha mismatch") if argv[:1] == [osapply.SELFUPDATE_WRAPPER] else orig(argv, timeout, stdin)
|
||||
rc, rep = run(box)
|
||||
self.assertEqual(rc, 3, rep)
|
||||
self.assertNotIn("u1", box.nonces)
|
||||
|
||||
|
||||
|
||||
class SelfupdateWrapperConfinement(unittest.TestCase):
|
||||
"""R-861 (v0.146.1): the A/B wrapper takes only felhom-os-apply's root-owned copy, never the agent's staging dir
|
||||
(a file there can be swapped between the wrapper's sha check and its copy). The path check runs before anything
|
||||
is touched, so the real script can be run here unprivileged.
|
||||
RED-PROOF: point ROOT_STAGING back at /var/lib/felhom-agent/selfupdate → this fails (the path is accepted and the
|
||||
script goes on to `staged file missing`)."""
|
||||
|
||||
def test_the_agents_staging_dir_is_refused(self):
|
||||
import subprocess
|
||||
sha = "0" * 64
|
||||
p = subprocess.run(["sh", str(HERE / "felhom-selfupdate-guarded"), "apply",
|
||||
"/var/lib/felhom-agent/selfupdate/felhom-agent-0.146.1", sha], capture_output=True, text=True)
|
||||
self.assertEqual(p.returncode, 1, p.stderr)
|
||||
self.assertIn("outside /var/lib/felhom-os-apply/agent-update", p.stderr)
|
||||
|
||||
|
||||
_sb = importlib.machinery.SourceFileLoader("stepbuild", str(REPO / "scripts" / "build-step-bundle.py"))
|
||||
_ss = importlib.util.spec_from_loader("stepbuild", _sb)
|
||||
stepbuild = importlib.util.module_from_spec(_ss)
|
||||
_sb.exec_module(stepbuild)
|
||||
NEW_IN_0146 = {"/usr/local/sbin/felhom-priv-apply", "/var/lib/vz/snippets/felhom-guest-hook.sh",
|
||||
"/usr/local/sbin/felhom-shared-parent.sh", "/etc/systemd/system/felhom-shared-parent.service"}
|
||||
|
||||
|
||||
class StepBundle(unittest.TestCase):
|
||||
"""R-880 (agent v0.146.1): an INSTALLED wrapper checks an incoming bundle's paths against its OWN table (R16), so a
|
||||
release that adds paths needs a step bundle: the boxes' current bundle with only felhom-os-apply replaced.
|
||||
RED-PROOF: deliver the full bundle to the old table → R16 (test_the_full_bundle_is_refused_by_an_old_table)."""
|
||||
|
||||
def old_world(self):
|
||||
"""The base bundle an older wrapper (no R-861 paths) installed, and that wrapper's table."""
|
||||
full = json.loads(builder.build("0.145.0"))
|
||||
full["files"] = [e for e in full["files"] if e["path"] not in NEW_IN_0146]
|
||||
old_wrapper = b'# the v0.145.0 wrapper stands in here\nBUNDLE_OP = "agent_config_update"\n'
|
||||
for e in full["files"]:
|
||||
if e["path"] == "/usr/local/sbin/felhom-os-apply":
|
||||
e["content_b64"], e["sha256"] = base64.b64encode(old_wrapper).decode(), hashlib.sha256(old_wrapper).hexdigest()
|
||||
base = (json.dumps(full, indent=1, sort_keys=True) + "\n").encode()
|
||||
old_dests = {k: v for k, v in osapply.BUNDLE_DESTS.items() if k not in NEW_IN_0146}
|
||||
return base, old_dests
|
||||
|
||||
def parse_with_table(self, data, dests, version):
|
||||
saved = osapply.BUNDLE_DESTS
|
||||
osapply.BUNDLE_DESTS = dests
|
||||
try:
|
||||
return osapply.Bundle(osapply.Apply(Box(b"{}", None), "")).parse(data, hashlib.sha256(data).hexdigest(), version)
|
||||
finally:
|
||||
osapply.BUNDLE_DESTS = saved
|
||||
|
||||
def test_the_full_bundle_is_refused_by_an_old_table(self):
|
||||
_, old_dests = self.old_world()
|
||||
full = builder.build("0.146.1")
|
||||
with self.assertRaises(osapply.Refused) as cm:
|
||||
self.parse_with_table(full, old_dests, "0.146.1")
|
||||
self.assertEqual(cm.exception.code, "R16")
|
||||
|
||||
def test_the_step_bundle_is_accepted_by_the_old_table_and_changes_only_the_wrapper(self):
|
||||
base, old_dests = self.old_world()
|
||||
new_wrapper = (HERE / "felhom-os-apply").read_bytes()
|
||||
step = stepbuild.build_step(base, "0.146.1-step1", new_wrapper)
|
||||
ver, files = self.parse_with_table(step, old_dests, "0.146.1-step1")
|
||||
self.assertEqual(ver, "0.146.1-step1")
|
||||
b, s_ = json.loads(base), json.loads(step)
|
||||
self.assertEqual(sorted(e["path"] for e in b["files"]), sorted(e["path"] for e in s_["files"]), "the paths must not change")
|
||||
changed = [e["path"] for e, f in zip(sorted(b["files"], key=lambda x: x["path"]), sorted(s_["files"], key=lambda x: x["path"]))
|
||||
if e != f]
|
||||
self.assertEqual(changed, ["/usr/local/sbin/felhom-os-apply"], "exactly the wrapper changes")
|
||||
installed = dict((d, c) for d, c, *_ in files)
|
||||
self.assertEqual(installed["/usr/local/sbin/felhom-os-apply"], new_wrapper)
|
||||
# and the NEW wrapper (now installed) knows every path the release's full bundle names
|
||||
self.assertTrue({e["path"] for e in json.loads(builder.build("0.146.1"))["files"]} <= set(osapply.BUNDLE_DESTS))
|
||||
|
||||
def test_a_step_version_must_carry_a_suffix(self):
|
||||
base, _ = self.old_world()
|
||||
with self.assertRaises(SystemExit):
|
||||
stepbuild.build_step(base, "0.146.1", b"x")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/python3
|
||||
"""Tests for felhom-crash-guard (`11` §5.9). Temp dirs only; nothing real is touched. Red-proof seam: CRASHGUARD_UNDER_TEST."""
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
_loader = importlib.machinery.SourceFileLoader("crashguard", os.environ.get("CRASHGUARD_UNDER_TEST", str(HERE / "felhom-crash-guard")))
|
||||
_spec = importlib.util.spec_from_loader("crashguard", _loader)
|
||||
cg = importlib.util.module_from_spec(_spec)
|
||||
_loader.exec_module(cg)
|
||||
|
||||
T0 = 1791115200.0 # 2026-10-04T12:00:00Z
|
||||
|
||||
|
||||
class FakeEnv(cg.Env):
|
||||
def __init__(self, d):
|
||||
super().__init__(conf=os.path.join(d, "conf"), state_dir=os.path.join(d, "state"),
|
||||
panic_path=os.path.join(d, "panic"), uptime_path=os.path.join(d, "uptime"),
|
||||
boot_id_path=os.path.join(d, "bootid"))
|
||||
self.t = T0
|
||||
self.logs = []
|
||||
open(self.panic_path, "w").write("0\n")
|
||||
open(self.uptime_path, "w").write("20.00 10.00\n")
|
||||
|
||||
def now(self):
|
||||
return self.t
|
||||
|
||||
def log(self, line):
|
||||
self.logs.append(line)
|
||||
|
||||
def panic(self):
|
||||
return int(open(self.panic_path).read())
|
||||
|
||||
def state(self):
|
||||
return json.load(open(os.path.join(self.state_dir, "state.json")))
|
||||
|
||||
|
||||
class Guard(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.d = tempfile.TemporaryDirectory()
|
||||
self.e = FakeEnv(self.d.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.d.cleanup()
|
||||
|
||||
def crash_boot(self, minutes_later):
|
||||
self.e.t += minutes_later * 60
|
||||
cg.main(["x", "boot"], self.e) # no clean-stop before it: an unclean stop
|
||||
|
||||
def clean_reboot(self, minutes_later):
|
||||
cg.main(["x", "clean-stop"], self.e)
|
||||
self.e.t += minutes_later * 60
|
||||
cg.main(["x", "boot"], self.e)
|
||||
|
||||
def test_first_boot_is_not_a_crash_and_arms(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
s = self.e.state()
|
||||
self.assertFalse(s["last_boot_unclean"])
|
||||
self.assertEqual(self.e.panic(), 10)
|
||||
self.assertTrue(s["armed"])
|
||||
|
||||
def test_clean_reboots_never_count(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
for _ in range(5):
|
||||
self.clean_reboot(1)
|
||||
s = self.e.state()
|
||||
self.assertEqual(s["unclean_boots_in_window"], 0)
|
||||
self.assertEqual(self.e.panic(), 10)
|
||||
|
||||
def test_third_crash_in_an_hour_leaves_the_box_off(self):
|
||||
# operator's words: "if it crashes 3 times within one hour, it stays off" — after crash 2 the guard trips,
|
||||
# so crash 3 (kernel.panic = 0) does not restart the box.
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.assertEqual(self.e.panic(), 10, "one crash: still restarts")
|
||||
self.crash_boot(5)
|
||||
s = self.e.state()
|
||||
self.assertTrue(s["tripped"], s)
|
||||
self.assertEqual(self.e.panic(), 0, "after the 2nd crash boot the 3rd crash must leave the box off")
|
||||
self.assertIn("2 unclean boots within 60 minutes", s["tripped_reason"])
|
||||
|
||||
def test_crashes_spread_over_more_than_the_window_do_not_trip(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.crash_boot(61)
|
||||
self.assertFalse(self.e.state()["tripped"])
|
||||
self.assertEqual(self.e.panic(), 10)
|
||||
|
||||
def test_tripped_stays_tripped_across_boots(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.crash_boot(5)
|
||||
self.clean_reboot(30) # the operator switched it on; even a clean boot keeps the trip
|
||||
self.assertTrue(self.e.state()["tripped"])
|
||||
self.assertEqual(self.e.panic(), 0)
|
||||
|
||||
def test_rearms_after_24h_of_normal_running(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.crash_boot(5)
|
||||
self.e.t += 23 * 3600
|
||||
cg.main(["x", "check"], self.e)
|
||||
self.assertTrue(self.e.state()["tripped"], "not before 24 h")
|
||||
self.e.t += 3600
|
||||
cg.main(["x", "check"], self.e)
|
||||
s = self.e.state()
|
||||
self.assertFalse(s["tripped"])
|
||||
self.assertEqual(self.e.panic(), 10)
|
||||
self.assertIn("timer", s["rearmed_by"])
|
||||
|
||||
def test_operator_rearm_starts_a_fresh_window(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.crash_boot(5)
|
||||
self.crash_boot(5)
|
||||
self.e.t += 60
|
||||
cg.main(["x", "rearm"], self.e)
|
||||
s = self.e.state()
|
||||
self.assertFalse(s["tripped"])
|
||||
self.assertEqual(s["rearmed_by"], "operator")
|
||||
self.assertEqual(s["unclean_boots_24h"], 2, "the history stays")
|
||||
self.crash_boot(5)
|
||||
self.assertFalse(self.e.state()["tripped"], "one crash after a re-arm must not trip at once")
|
||||
|
||||
def test_config_numbers_are_read(self):
|
||||
open(self.e.conf, "w").write("LIMIT=2\nPANIC_SECONDS=30\n")
|
||||
cg.main(["x", "boot"], self.e)
|
||||
self.assertEqual(self.e.panic(), 30)
|
||||
self.crash_boot(1)
|
||||
self.assertTrue(self.e.state()["tripped"], "LIMIT=2: the first crash boot trips")
|
||||
|
||||
def test_state_is_world_readable_for_the_agent(self):
|
||||
cg.main(["x", "boot"], self.e)
|
||||
mode = os.stat(os.path.join(self.e.state_dir, "state.json")).st_mode & 0o777
|
||||
self.assertEqual(mode, 0o644)
|
||||
|
||||
|
||||
class KernelStepCannotLeaveTheBoxOff(unittest.TestCase):
|
||||
"""R-836 / `11` §5.11 Part B 4: a kernel step's planned reboot, one crash and one self-revert cannot add up to the
|
||||
box staying off. The wrapper starts a step only when the guard is armed with NO unclean boot in its window
|
||||
(felhom-os-apply Kernel.check_guard, R21); the planned reboot and the self-revert are orderly (`systemctl reboot` —
|
||||
the clean-stop marker); so the step adds at most ONE unclean boot, and the box stays off only after the LIMIT-th
|
||||
(3rd) within the hour. Red-proof: audits/kernel-lane-2026-10-07/A/redproof.txt."""
|
||||
|
||||
def setUp(self):
|
||||
self.d = tempfile.TemporaryDirectory()
|
||||
self.e = FakeEnv(self.d.name)
|
||||
cg.main(["x", "boot"], self.e)
|
||||
s = self.e.state()
|
||||
self.assertTrue(s["armed"])
|
||||
self.assertEqual(s["unclean_boots_in_window"], 0, "the wrapper's precondition (R21)")
|
||||
|
||||
def tearDown(self):
|
||||
self.d.cleanup()
|
||||
|
||||
def planned(self, minutes):
|
||||
cg.main(["x", "clean-stop"], self.e)
|
||||
self.e.t += minutes * 60
|
||||
cg.main(["x", "boot"], self.e)
|
||||
|
||||
def crash(self, minutes):
|
||||
self.e.t += minutes * 60
|
||||
cg.main(["x", "boot"], self.e)
|
||||
|
||||
def test_planned_reboot_one_crash_one_self_revert(self):
|
||||
self.planned(2) # the step's one-shot reboot (orderly)
|
||||
self.crash(3) # the new kernel crashes after the guard ran; the box restarts (panic=10)
|
||||
self.planned(2) # the self-revert (orderly)
|
||||
s = self.e.state()
|
||||
self.assertFalse(s["tripped"], s)
|
||||
self.assertTrue(s["armed"])
|
||||
self.assertEqual(self.e.panic(), 10, "the box still restarts after a crash")
|
||||
self.assertEqual(s["unclean_boots_in_window"], 1, "the step added exactly one unclean boot")
|
||||
|
||||
def test_a_panic_before_userspace_is_not_even_counted(self):
|
||||
# the one-shot kernel panics before the guard's unit runs (measured: rdinit= and init= missing): the planned
|
||||
# reboot's clean-stop marker is still there when the old kernel boots, so this boot counts as clean.
|
||||
cg.main(["x", "clean-stop"], self.e)
|
||||
self.e.t += 120 # the panicking boot: no userspace, the guard never ran
|
||||
cg.main(["x", "boot"], self.e)
|
||||
s = self.e.state()
|
||||
self.assertEqual(s["unclean_boots_in_window"], 0, s)
|
||||
self.assertEqual(self.e.panic(), 10)
|
||||
|
||||
def test_the_box_stays_off_only_after_two_more_crashes_than_the_step_makes(self):
|
||||
self.planned(2)
|
||||
self.crash(3) # the step's one crash
|
||||
self.planned(2) # the self-revert
|
||||
self.crash(5) # an UNRELATED crash within the hour: the guard trips (the 3rd would leave it off)
|
||||
s = self.e.state()
|
||||
self.assertTrue(s["tripped"])
|
||||
self.assertEqual(s["unclean_boots_in_window"], 2, "two unclean boots: one from the step, one not")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for configs/felhom-internal-place (felhom.eu architecture/16 §5, §7 — sizes ruled 2026-10-16).
|
||||
|
||||
The helper is run for real under bash; `lvs`, `lvcreate`, `blkid`, `mkfs.ext4` and `stat` are PATH stubs that read and
|
||||
write one JSON state file, so nothing on this machine is touched. Every refusal has a positive control: the same setup
|
||||
without the one condition goes through — so a test cannot pass because the helper refuses everything.
|
||||
Run: python3 configs/test_felhom_internal_place.py (also run by the Go suite, internal/internalplace)
|
||||
"""
|
||||
import json, os, pathlib, stat, subprocess, tempfile, unittest
|
||||
|
||||
HELPER = pathlib.Path(__file__).resolve().parent / "felhom-internal-place"
|
||||
GIB = 1 << 30
|
||||
|
||||
STUB_LVS = r'''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
st = json.load(open(os.environ["IP_STATE"]))
|
||||
if st.get("lvs_fail_listing") and "lv_name,pool_lv,lv_size" in sys.argv: sys.exit(5)
|
||||
args = [a for a in sys.argv[1:] if not a.startswith("-") and a not in ("b",)]
|
||||
fields = sys.argv[sys.argv.index("-o") + 1].split(",")
|
||||
target = args[-1]
|
||||
listing = target == st["vg"]
|
||||
for lv in st["lvs"]:
|
||||
full = st["vg"] + "/" + lv["name"]
|
||||
if listing and st.get("hide_pool_in_listing") and lv.get("pool", "") == "":
|
||||
continue # a listing that silently lost the pool row (the pool's own size stays readable)
|
||||
if listing or target == full:
|
||||
row = {"lv_name": lv["name"], "pool_lv": lv.get("pool", ""), "lv_size": str(lv["size"])}
|
||||
print(" " + " ".join(row[f] for f in fields))
|
||||
'''
|
||||
STUB_LVCREATE = r'''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
p = os.environ["IP_STATE"]; st = json.load(open(p))
|
||||
a = sys.argv[1:]
|
||||
size = int(a[a.index("-V") + 1].rstrip("G")) * (1 << 30)
|
||||
vg, pool = a[a.index("-T") + 1].split("/")
|
||||
st["lvs"].append({"name": a[a.index("-n") + 1], "pool": pool, "size": size, "active": True})
|
||||
st.setdefault("calls", []).append("lvcreate " + " ".join(a))
|
||||
if st.get("dirty_new"): st["fs"]["felhom-belso"] = {"TYPE": "", "RAW": "junk"}
|
||||
json.dump(st, open(p, "w"))
|
||||
'''
|
||||
STUB_BLKID = r'''#!/usr/bin/env python3
|
||||
# like blkid -p: exit 0 when something is found, 2 when nothing is; st["blkid_rc"] forces an error exit
|
||||
import json, os, sys
|
||||
st = json.load(open(os.environ["IP_STATE"]))
|
||||
if st.get("blkid_rc"): sys.exit(st["blkid_rc"])
|
||||
fs = st["fs"].get(os.path.basename(sys.argv[-1]), {})
|
||||
if "-s" in sys.argv:
|
||||
v = fs.get(sys.argv[sys.argv.index("-s") + 1], "")
|
||||
if v: print(v)
|
||||
sys.exit(0 if v else 2)
|
||||
vals = [fs.get(k, "") for k in ("TYPE", "LABEL", "UUID", "RAW") if fs.get(k)]
|
||||
if vals: print("\n".join(vals))
|
||||
sys.exit(0 if vals else 2)
|
||||
'''
|
||||
STUB_MKFS = r'''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
p = os.environ["IP_STATE"]; st = json.load(open(p))
|
||||
dev = os.path.basename(sys.argv[-1])
|
||||
st["fs"][dev] = {"TYPE": "ext4", "LABEL": sys.argv[sys.argv.index("-L") + 1], "UUID": "0b6e4c5a-1d2e-4f3a-9b8c-7d6e5f4a3b2c"}
|
||||
st.setdefault("calls", []).append("mkfs.ext4 " + " ".join(sys.argv[1:]))
|
||||
json.dump(st, open(p, "w"))
|
||||
'''
|
||||
STUB_STAT = r'''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
st = json.load(open(os.environ["IP_STATE"]))
|
||||
name = os.path.basename(sys.argv[-1])
|
||||
if any(lv["name"] == name and lv.get("active", True) for lv in st["lvs"]): print("block special file")
|
||||
else: sys.exit(1)
|
||||
'''
|
||||
|
||||
CFG = "dir: local\n\tpath /var/lib/vz\n\tcontent iso,backup\n\nlvmthin: local-lvm\n\tthinpool data\n\tvgname pve\n\tcontent rootdir,images\n"
|
||||
|
||||
|
||||
class Box:
|
||||
def __init__(self, pool_gib, lvs=(), fs=None, cfg=CFG, **extra):
|
||||
self.t = tempfile.TemporaryDirectory()
|
||||
r = pathlib.Path(self.t.name)
|
||||
self.bin = r / "bin"; self.bin.mkdir()
|
||||
for name, body in (("lvs", STUB_LVS), ("lvcreate", STUB_LVCREATE), ("blkid", STUB_BLKID),
|
||||
("mkfs.ext4", STUB_MKFS), ("stat", STUB_STAT)):
|
||||
f = self.bin / name; f.write_text(body); f.chmod(0o755)
|
||||
self.cfg = r / "storage.cfg"; self.cfg.write_text(cfg)
|
||||
self.state = r / "state.json"
|
||||
st = {"vg": "pve", "lvs": [{"name": "data", "pool": "", "size": pool_gib * GIB}] + [dict(l) for l in lvs],
|
||||
"fs": fs or {}}
|
||||
st.update(extra)
|
||||
self.state.write_text(json.dumps(st))
|
||||
|
||||
def run(self, *args):
|
||||
env = {"PATH": f"{self.bin}:/usr/bin:/bin", "IP_STATE": str(self.state), "FELHOM_STORAGE_CFG": str(self.cfg)}
|
||||
p = subprocess.run(["bash", str(HELPER), *args], env=env, capture_output=True, text=True)
|
||||
return p.returncode, p.stdout.strip(), p.stderr.strip()
|
||||
|
||||
def st(self):
|
||||
return json.loads(self.state.read_text())
|
||||
|
||||
def calls(self):
|
||||
return self.st().get("calls", [])
|
||||
|
||||
def close(self):
|
||||
self.t.cleanup()
|
||||
|
||||
|
||||
def thin(name, gib, pool="data"):
|
||||
return {"name": name, "pool": pool, "size": gib * GIB}
|
||||
|
||||
|
||||
class Sizes(unittest.TestCase):
|
||||
"""The operator's table (SPACE-CHECK §4, ruled 2026-10-16), as the helper computes it on a fresh box."""
|
||||
|
||||
def plan(self, pool, planned, lvs=()):
|
||||
b = Box(pool, lvs)
|
||||
try:
|
||||
rc, out, err = b.run("plan", str(planned))
|
||||
self.assertEqual(rc, 0, err)
|
||||
self.assertEqual(b.calls(), [], "plan must change nothing")
|
||||
return dict(kv.split("=") for kv in out.split()[1:])
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_the_ruled_table(self):
|
||||
# disk → pool (GiB), data volume (planned = 32 + mp0), expected internal
|
||||
for pool, mp0, reserve, internal in ((140, 50, 16, 42), (349, 100, 34, 183), (800, 130, 48, 590), (1720, 130, 48, 1510)):
|
||||
got = self.plan(pool, 32 + mp0)
|
||||
self.assertEqual((int(got["reserve"]), int(got["internal"])), (reserve, internal), (pool, got))
|
||||
|
||||
def test_promised_volumes_count_but_vzdump_snapshots_and_other_pools_do_not(self):
|
||||
got = self.plan(349, 0, [thin("vm-9201-disk-0", 32), thin("vm-9201-disk-1", 250),
|
||||
thin("snap_vm-9201-disk-1_vzdump", 250), thin("vm-500-disk-0", 40, pool="other")])
|
||||
self.assertEqual((int(got["promised"]), int(got["internal"])), (282, 33), got)
|
||||
|
||||
|
||||
class Create(unittest.TestCase):
|
||||
def test_a_fresh_box_gets_the_place(self):
|
||||
b = Box(349)
|
||||
try:
|
||||
rc, out, err = b.run("create", "132")
|
||||
self.assertEqual(rc, 0, err)
|
||||
self.assertRegex(out, r"^created uuid=[0-9a-f-]{36} gib=183$")
|
||||
self.assertEqual(len([c for c in b.calls() if c.startswith("lvcreate")]), 1)
|
||||
self.assertIn("-V 183G -T pve/data -n felhom-belso", b.calls()[0])
|
||||
self.assertIn("-L felhom-belso", b.calls()[1])
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_too_little_space_makes_nothing(self):
|
||||
b = Box(53) # demo-hp's 128 GB disk
|
||||
try:
|
||||
rc, out, err = b.run("create", str(32 + 50))
|
||||
self.assertEqual(rc, 0, err)
|
||||
self.assertTrue(out.startswith("skipped "), out)
|
||||
self.assertEqual(b.calls(), [])
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_an_existing_place_is_found_never_formatted(self):
|
||||
fs = {"felhom-belso": {"TYPE": "ext4", "LABEL": "felhom-belso", "UUID": "11111111-2222-3333-4444-555555555555"}}
|
||||
b = Box(349, [thin("felhom-belso", 183)], fs)
|
||||
try:
|
||||
rc, out, err = b.run("create", "0")
|
||||
self.assertEqual(rc, 0, err)
|
||||
self.assertEqual(out, "existing uuid=11111111-2222-3333-4444-555555555555 gib=183")
|
||||
self.assertEqual(b.calls(), [], "an existing place must never be created or formatted again")
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_a_foreign_filesystem_on_the_name_is_refused(self):
|
||||
# positive control: test_an_existing_place_is_found_never_formatted (same LV, Felhom's label) goes through
|
||||
for fs in ({"TYPE": "xfs", "LABEL": "felhom-belso", "UUID": "x"}, {"TYPE": "ext4", "LABEL": "films", "UUID": "y"}):
|
||||
b = Box(349, [thin("felhom-belso", 183)], {"felhom-belso": fs})
|
||||
try:
|
||||
rc, out, err = b.run("create", "0")
|
||||
self.assertEqual(rc, 1, (fs, out))
|
||||
self.assertIn("REFUSED", err)
|
||||
self.assertEqual(b.calls(), [], fs)
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_a_crash_between_lvcreate_and_mkfs_is_finished(self):
|
||||
b = Box(349, [thin("felhom-belso", 183)], {})
|
||||
try:
|
||||
rc, out, err = b.run("create", "0")
|
||||
self.assertEqual(rc, 0, err)
|
||||
self.assertRegex(out, r"^formatted uuid=")
|
||||
self.assertEqual([c.split()[0] for c in b.calls()], ["mkfs.ext4"])
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_an_unnamed_signature_is_refused(self):
|
||||
b = Box(349, [thin("felhom-belso", 183)], {"felhom-belso": {"RAW": "something"}})
|
||||
try:
|
||||
rc, out, err = b.run("create", "0")
|
||||
self.assertEqual(rc, 1, out)
|
||||
self.assertEqual(b.calls(), [])
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_a_new_volume_that_is_not_empty_is_not_formatted(self):
|
||||
b = Box(349, dirty_new=True)
|
||||
try:
|
||||
rc, out, err = b.run("create", "132")
|
||||
self.assertEqual(rc, 1, out)
|
||||
self.assertNotIn("mkfs", " ".join(b.calls()))
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_bad_arguments_and_no_local_lvm_are_refused(self):
|
||||
b = Box(349, cfg="dir: local\n\tpath /var/lib/vz\n")
|
||||
try:
|
||||
self.assertEqual(b.run("create", "0")[0], 1)
|
||||
self.assertEqual(b.calls(), [])
|
||||
finally:
|
||||
b.close()
|
||||
b = Box(349)
|
||||
try:
|
||||
for args in (("create", "-1"), ("create", "1x"), ("wipe", "0"), ("create",)):
|
||||
self.assertEqual(b.run(*args)[0], 1, args)
|
||||
self.assertEqual(b.calls(), [])
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
|
||||
class FailsClosed(unittest.TestCase):
|
||||
"""A read that fails must never read as „empty" or „nothing promised". Positive control: Create (same setups)."""
|
||||
|
||||
def test_a_failed_volume_listing_creates_nothing(self):
|
||||
b = Box(349, lvs_fail_listing=True)
|
||||
try:
|
||||
rc, out, err = b.run("create", "132")
|
||||
self.assertEqual(rc, 1, out)
|
||||
self.assertEqual(b.calls(), [])
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_a_listing_without_the_pool_creates_nothing(self):
|
||||
b = Box(349, [thin("vm-9201-disk-1", 250)], hide_pool_in_listing=True)
|
||||
try:
|
||||
self.assertEqual(b.run("create", "132")[0], 1)
|
||||
self.assertEqual(b.calls(), [])
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
def test_a_blkid_error_never_reads_as_empty(self):
|
||||
for setup in ({"lvs": [thin("felhom-belso", 183)], "fs": {}}, {}):
|
||||
b = Box(349, **({"lvs": setup["lvs"], "fs": setup["fs"]} if setup else {}), blkid_rc=4)
|
||||
try:
|
||||
rc, out, err = b.run("create", "0" if setup else "132")
|
||||
self.assertEqual(rc, 1, (setup, out))
|
||||
self.assertNotIn("mkfs", " ".join(b.calls()), setup)
|
||||
finally:
|
||||
b.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for configs/felhom-mkfs-guarded.sh — the ONLY mkfs path the agent's sudoers permits.
|
||||
|
||||
First tests of this wrapper (agent v0.158.0), written for partitions as places (felhom.eu architecture/16 §6). Every
|
||||
mount of every node of the disk outside /mnt/felhom-drives refuses — INCLUDING a sibling that is Felhom's own drive
|
||||
(its raw /mnt/<name> mount): the agent can make that corroborating mount itself, so the root wrapper must not trust
|
||||
it (security review of 912f7c1). An unlistable disk refuses.
|
||||
|
||||
HOW IT RUNS ANYWHERE: the wrapper is copied with five literal host paths replaced (each replacement is asserted, so a
|
||||
drifted wrapper fails here instead of silently testing nothing): /proc/mounts → a fake mount table, the block-device
|
||||
test → true (no real /dev nodes here), the pvs candidates → a stub, /sys/block → a fake tree, the mkfs binaries → a
|
||||
recorder. `lsblk` is a PATH stub answering the three calls the wrapper makes from one JSON disk description.
|
||||
Every refusal has a positive control (the same disk without the one condition is formatted).
|
||||
Run: python3 configs/test_felhom_mkfs_guarded.py (also run by the Go suite, internal/storage)
|
||||
"""
|
||||
import json, os, pathlib, subprocess, tempfile, unittest
|
||||
|
||||
WRAPPER = pathlib.Path(__file__).resolve().parent / "felhom-mkfs-guarded.sh"
|
||||
|
||||
STUB_LSBLK = r'''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
st = json.load(open(os.environ["MG_STATE"]))
|
||||
a = sys.argv[1:]; dev = a[-1]
|
||||
nodes = st["nodes"] # [{"name": "/dev/sdb", "pk": "", "fstype": ""}, ...]
|
||||
if a[0] == "-ndo" and a[1] == "PKNAME":
|
||||
for n in nodes:
|
||||
if n["name"] == dev and n.get("pk"): print(n["pk"])
|
||||
elif a[0] == "-nrpo" and a[1] == "NAME":
|
||||
if not st.get("lsblk_empty"):
|
||||
for n in nodes: print(n["name"])
|
||||
elif a[0] == "-nro" and a[1] == "FSTYPE":
|
||||
for n in nodes: print(n.get("fstype", ""))
|
||||
elif a[0] == "-nro" and a[1] == "MOUNTPOINT": # the pre-v0.158.0 wrapper's call: one mountpoint per node, like lsblk
|
||||
mt = [l.split()[:2] for l in open(os.environ["MG_MOUNTS"]) if l.strip()]
|
||||
for n in nodes:
|
||||
mps = [m for d, m in mt if d == n["name"]]
|
||||
print(mps[0] if mps else "")
|
||||
'''
|
||||
STUB_MKFS = '#!/bin/sh\necho "$0 $*" >> "$MG_CALLS"\n'
|
||||
STUB_PVS = '#!/bin/sh\nprintf "%s\\n" $MG_PVS\n'
|
||||
|
||||
|
||||
class Disk:
|
||||
def __init__(self, nodes, mounts, ro="0", pvs="", lsblk_empty=False):
|
||||
self.t = tempfile.TemporaryDirectory()
|
||||
r = pathlib.Path(self.t.name)
|
||||
self.bin = r / "bin"; self.bin.mkdir()
|
||||
(self.bin / "lsblk").write_text(STUB_LSBLK); (self.bin / "lsblk").chmod(0o755)
|
||||
for n in ("mkfs.ext4", "mkfs.xfs"):
|
||||
(self.bin / n).write_text(STUB_MKFS); (self.bin / n).chmod(0o755)
|
||||
(self.bin / "pvs").write_text(STUB_PVS); (self.bin / "pvs").chmod(0o755)
|
||||
self.state = r / "state.json"
|
||||
self.state.write_text(json.dumps({"nodes": nodes, "lsblk_empty": lsblk_empty}))
|
||||
self.mounts = r / "mounts"
|
||||
self.mounts.write_text("".join(f"{d} {m} ext4 rw 0 0\n" for d, m in mounts))
|
||||
self.sys = r / "sys"
|
||||
for n in nodes:
|
||||
if not n.get("pk"):
|
||||
d = self.sys / os.path.basename(n["name"]); d.mkdir(parents=True); (d / "ro").write_text(ro + "\n")
|
||||
self.calls = r / "calls"
|
||||
self.pvs = pvs
|
||||
src = WRAPPER.read_text()
|
||||
subs = [
|
||||
("/proc/mounts", str(self.mounts)),
|
||||
('[[ -b "$dev" ]]', "true"),
|
||||
("for c in /usr/sbin/pvs /sbin/pvs; do", f"for c in {self.bin}/pvs; do"),
|
||||
('rof="/sys/block/', f'rof="{self.sys}/'),
|
||||
("exec /usr/sbin/mkfs.ext4", f"exec {self.bin}/mkfs.ext4"),
|
||||
("exec /usr/sbin/mkfs.xfs ", f"exec {self.bin}/mkfs.xfs "),
|
||||
]
|
||||
for a, b in subs:
|
||||
assert a in src, f"the wrapper no longer contains {a!r} — update this test's substitutions"
|
||||
src = src.replace(a, b)
|
||||
self.copy = r / "mkfs-guarded"; self.copy.write_text(src)
|
||||
|
||||
def run(self, dev, fstype="ext4"):
|
||||
env = {"PATH": f"{self.bin}:/usr/bin:/bin", "MG_STATE": str(self.state), "MG_CALLS": str(self.calls),
|
||||
"MG_MOUNTS": str(self.mounts),
|
||||
"MG_PVS": self.pvs}
|
||||
p = subprocess.run(["bash", str(self.copy), dev, fstype], env=env, capture_output=True, text=True)
|
||||
made = self.calls.read_text().split("\n")[0] if self.calls.exists() else ""
|
||||
return p.returncode, p.stderr.strip(), made
|
||||
|
||||
def close(self):
|
||||
self.t.cleanup()
|
||||
|
||||
|
||||
def disk_sdb(sdb1_fstype="ext4", sdb2_fstype=""):
|
||||
return [{"name": "/dev/sdb", "pk": "", "fstype": ""},
|
||||
{"name": "/dev/sdb1", "pk": "sdb", "fstype": sdb1_fstype},
|
||||
{"name": "/dev/sdb2", "pk": "sdb", "fstype": sdb2_fstype}]
|
||||
|
||||
|
||||
ROOT = ("/dev/sda2", "/")
|
||||
OWN_SDB1 = [("/dev/sdb1", "/mnt/hdd_1"), ("/dev/sdb1", "/mnt/felhom-drives/hdd_1")]
|
||||
|
||||
|
||||
class Partitions(unittest.TestCase):
|
||||
def check(self, mounts, dev="/dev/sdb2", nodes=None, want_ok=True, **kw):
|
||||
d = Disk(nodes or disk_sdb(), [ROOT] + mounts, **kw)
|
||||
try:
|
||||
rc, err, made = d.run(dev)
|
||||
if want_ok:
|
||||
self.assertEqual(rc, 0, err)
|
||||
self.assertTrue(made.endswith(f"mkfs.ext4 -F {dev}"), made)
|
||||
else:
|
||||
self.assertEqual(rc, 1, (mounts, made))
|
||||
self.assertIn("REFUSED", err)
|
||||
self.assertEqual(made, "", "a refused format must not run mkfs")
|
||||
return err
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
def test_a_free_part_on_an_unmounted_disk_is_formatted(self):
|
||||
self.check([]) # positive control for every refusal below
|
||||
|
||||
def test_the_free_part_next_to_felhoms_own_drive_is_refused(self):
|
||||
# The agent could make the „corroborating" managed mount itself: the root wrapper does not trust it.
|
||||
self.check(OWN_SDB1, want_ok=False)
|
||||
|
||||
def test_a_sibling_mounted_raw_only_is_refused(self):
|
||||
self.check([("/dev/sdb1", "/mnt/hdd_1")], want_ok=False)
|
||||
|
||||
def test_a_sibling_mounted_elsewhere_is_refused_even_with_a_managed_mount(self):
|
||||
for elsewhere in ("/srv/data", "/media/usb", "/mnt/a/b", "/home/x"):
|
||||
self.check([("/dev/sdb1", elsewhere), ("/dev/sdb1", "/mnt/felhom-drives/hdd_1")], want_ok=False)
|
||||
|
||||
def test_the_target_itself_mounted_raw_is_refused(self):
|
||||
self.check([("/dev/sdb2", "/mnt/films"), ("/dev/sdb2", "/mnt/felhom-drives/films")], want_ok=False)
|
||||
|
||||
def test_the_target_mounted_only_under_felhom_drives_is_formatted(self):
|
||||
self.check([("/dev/sdb2", "/mnt/felhom-drives/films")])
|
||||
|
||||
def test_an_unlistable_disk_is_refused(self):
|
||||
self.check([], want_ok=False, lsblk_empty=True)
|
||||
|
||||
def test_the_catastrophic_cases_still_refuse(self):
|
||||
self.check([("/dev/sdb1", "/")], want_ok=False) # system disk
|
||||
self.check([], nodes=disk_sdb(sdb1_fstype="LVM2_member"), want_ok=False) # a member signature
|
||||
self.check([], pvs="/dev/sdb1", want_ok=False) # an LVM PV (pvs)
|
||||
self.check([], ro="1", want_ok=False) # read-only
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,442 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for felhom-priv-apply (R-861, `03` §3.1). An in-memory host plays the files; nothing real is written or run.
|
||||
Each refusal rule has a test that feeds it the ATTACK it exists for; each accepted shape is a file the agent really
|
||||
renders (the Go contract tests feed the live renderers too). Red-proof: audits/hub-safety-2026-10-05/partF/.
|
||||
|
||||
Run: python3 configs/test_felhom_priv_apply.py (also run by internal/privapply's Go test)
|
||||
"""
|
||||
import sys
|
||||
sys.dont_write_bytecode = True
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import os
|
||||
import pathlib
|
||||
import unittest
|
||||
|
||||
HERE = pathlib.Path(__file__).resolve().parent
|
||||
_loader = importlib.machinery.SourceFileLoader("privapply", os.environ.get("PRIVAPPLY_UNDER_TEST", str(HERE / "felhom-priv-apply")))
|
||||
_spec = importlib.util.spec_from_loader("privapply", _loader)
|
||||
pa = importlib.util.module_from_spec(_spec)
|
||||
_loader.exec_module(pa)
|
||||
|
||||
AGENT_UID = 999
|
||||
|
||||
|
||||
class FakeHost:
|
||||
def __init__(self):
|
||||
self.src, self.dest, self.logs, self.writes = {}, {}, [], []
|
||||
self.owner, self.kind = {}, {}
|
||||
|
||||
def stage(self, path, text, uid=AGENT_UID, kind="file"):
|
||||
self.src[path] = text.encode() if isinstance(text, str) else text
|
||||
self.owner[path], self.kind[path] = uid, kind
|
||||
|
||||
def agent_uid(self):
|
||||
return AGENT_UID
|
||||
|
||||
def read_source(self, path):
|
||||
# mirrors Host.read_source's refusals: absent, not a regular file (a symlink is refused by O_NOFOLLOW), owner, size
|
||||
if path not in self.src:
|
||||
raise pa.Refused("P1", f"cannot open the staged file {path}")
|
||||
if self.kind[path] != "file":
|
||||
raise pa.Refused("P1", f"{path} is not a regular file")
|
||||
if self.owner[path] != AGENT_UID:
|
||||
raise pa.Refused("P1", f"{path} is not owned by felhom-agent")
|
||||
if len(self.src[path]) > pa.MAX_BYTES:
|
||||
raise pa.Refused("P1", f"{path} is larger than {pa.MAX_BYTES} bytes")
|
||||
return self.src[path]
|
||||
|
||||
def read_dest(self, path):
|
||||
return self.dest.get(path)
|
||||
|
||||
def install(self, dest, data, mode):
|
||||
self.writes.append((dest, mode))
|
||||
self.dest[dest] = data
|
||||
|
||||
def log(self, line):
|
||||
self.logs.append(line)
|
||||
|
||||
# controller-image (R-861 (a) A1): the ref arrives on stdin and is written INSIDE the guest by root.
|
||||
stdin = b""
|
||||
guest_writes = None
|
||||
|
||||
def read_stdin(self, limit):
|
||||
return self.stdin[:limit + 1]
|
||||
|
||||
def write_guest_image(self, vmid, data):
|
||||
if self.guest_writes is None:
|
||||
self.guest_writes = []
|
||||
self.guest_writes.append((vmid, data))
|
||||
|
||||
|
||||
LOCAL_UNIT = """# Managed by felhom-agent — do not edit by hand.
|
||||
[Unit]
|
||||
Description=Felhom storage mount 91d2dc2d-2d28-4929-9bdd-3e11fa2f41ae
|
||||
After=local-fs-pre.target
|
||||
|
||||
[Mount]
|
||||
What=/dev/disk/by-uuid/91d2dc2d-2d28-4929-9bdd-3e11fa2f41ae
|
||||
Where=/mnt/hdd_1
|
||||
Type=ext4
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
"""
|
||||
NET_MOUNT = """# felhom network storage — do not edit by hand.
|
||||
[Unit]
|
||||
Description=Felhom network storage media (nfs)
|
||||
|
||||
[Mount]
|
||||
What=nas.lan:/volume1/media
|
||||
Where=/mnt/felhom-drives/media
|
||||
Type=nfs4
|
||||
Options=vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0,nosuid,nodev
|
||||
"""
|
||||
NET_AUTOMOUNT = """# felhom network storage — do not edit by hand.
|
||||
[Unit]
|
||||
Description=Felhom network storage automount media (nfs)
|
||||
|
||||
[Automount]
|
||||
Where=/mnt/felhom-drives/media
|
||||
TimeoutIdleSec=600
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
"""
|
||||
NET_NAME = "mnt-felhom\\x2ddrives-media.mount"
|
||||
DNS_BASE = """# felhom split-horizon resolver — host base config (agent-managed; DO NOT EDIT)
|
||||
bind-interfaces
|
||||
listen-address=192.168.0.104
|
||||
listen-address=127.0.0.1
|
||||
no-resolv
|
||||
server=1.1.1.1
|
||||
server=9.9.9.9
|
||||
"""
|
||||
DNS_GUEST = """# felhom split-horizon DNS — customer demo-hp (agent-managed; DO NOT EDIT)
|
||||
local=/enkisfelhom.hu/
|
||||
address=/enkisfelhom.hu/192.168.0.138
|
||||
"""
|
||||
K = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" # base64 of 32 bytes
|
||||
WG = f"""# felhom offsite tunnel — agent-managed (S3); DO NOT EDIT
|
||||
[Interface]
|
||||
PrivateKey = {K}
|
||||
Address = 10.77.0.3/32
|
||||
MTU = 1280
|
||||
|
||||
[Peer]
|
||||
PublicKey = {K}
|
||||
Endpoint = 49.12.1.2:51820
|
||||
AllowedIPs = 10.77.0.1/32, 10.77.0.250/32
|
||||
PersistentKeepalive = 25
|
||||
"""
|
||||
KEYLINE = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL8z0qCNgA3x2xxAB0Qj5ro8waFjGZ8Ta/sWB63tlLw+ felhom-op-1\n"
|
||||
|
||||
|
||||
def run(host, *argv):
|
||||
return pa.main(list(argv), host=host)
|
||||
|
||||
|
||||
class Accepts(unittest.TestCase):
|
||||
"""What the agent really writes is accepted and installed, root-owned, at the fixed destination."""
|
||||
|
||||
def test_local_unit_installed(self):
|
||||
h = FakeHost()
|
||||
h.stage("/var/lib/felhom-agent/units/mnt-hdd_1.mount", LOCAL_UNIT)
|
||||
self.assertEqual(run(h, "unit", "mnt-hdd_1.mount"), 0)
|
||||
self.assertEqual(h.writes, [("/etc/systemd/system/mnt-hdd_1.mount", 0o644)])
|
||||
|
||||
def test_local_unit_without_type(self): # the N100's unit has no Type= (autodetect)
|
||||
h = FakeHost()
|
||||
h.stage("/var/lib/felhom-agent/units/mnt-hdd_1.mount", LOCAL_UNIT.replace("Type=ext4\n", ""))
|
||||
self.assertEqual(run(h, "unit", "mnt-hdd_1.mount"), 0)
|
||||
|
||||
def test_network_pair(self):
|
||||
h = FakeHost()
|
||||
h.stage("/var/lib/felhom-agent/units/" + NET_NAME, NET_MOUNT)
|
||||
h.stage("/var/lib/felhom-agent/units/mnt-felhom\\x2ddrives-media.automount", NET_AUTOMOUNT)
|
||||
self.assertEqual(run(h, "unit", NET_NAME), 0)
|
||||
self.assertEqual(run(h, "unit", "mnt-felhom\\x2ddrives-media.automount"), 0)
|
||||
|
||||
def test_identical_is_not_rewritten(self):
|
||||
h = FakeHost()
|
||||
h.stage("/var/lib/felhom-agent/units/mnt-hdd_1.mount", LOCAL_UNIT)
|
||||
h.dest["/etc/systemd/system/mnt-hdd_1.mount"] = LOCAL_UNIT.encode()
|
||||
self.assertEqual(run(h, "unit", "mnt-hdd_1.mount"), 0)
|
||||
self.assertEqual(h.writes, [])
|
||||
|
||||
def test_dnsmasq_both_dropins(self):
|
||||
h = FakeHost()
|
||||
h.stage("/tmp/felhom-resolver-123.conf", DNS_BASE)
|
||||
self.assertEqual(run(h, "dnsmasq", "/tmp/felhom-resolver-123.conf", "felhom-resolver-base.conf"), 0)
|
||||
h.stage("/tmp/felhom-resolver-124.conf", DNS_GUEST)
|
||||
self.assertEqual(run(h, "dnsmasq", "/tmp/felhom-resolver-124.conf", "felhom-demo-hp.conf"), 0)
|
||||
self.assertIn(("/etc/dnsmasq.d/felhom-demo-hp.conf", 0o644), h.writes)
|
||||
|
||||
def test_wg_installed_0600(self):
|
||||
h = FakeHost()
|
||||
h.stage(pa.WG_SRC, WG)
|
||||
self.assertEqual(run(h, "wg"), 0)
|
||||
self.assertEqual(h.writes, [(pa.WG_DEST, 0o600)])
|
||||
|
||||
def test_sshd_template_and_key(self):
|
||||
h = FakeHost()
|
||||
h.stage(pa.SSHD_SRC, pa.render_sshd(8822))
|
||||
h.stage(pa.KEY_SRC, KEYLINE)
|
||||
self.assertEqual(run(h, "sshd-config"), 0)
|
||||
self.assertEqual(run(h, "sshd-key"), 0)
|
||||
h.stage(pa.KEY_SRC, "") # clearing the operator login is allowed
|
||||
self.assertEqual(run(h, "sshd-key"), 0)
|
||||
|
||||
def test_escape_matches_systemd(self):
|
||||
self.assertEqual(pa.systemd_escape_path("/mnt/felhom-drives/media"), "mnt-felhom\\x2ddrives-media")
|
||||
self.assertEqual(pa.systemd_escape_path("/mnt/hdd_1"), "mnt-hdd_1")
|
||||
self.assertEqual(pa.systemd_escape_path("/mnt/.x"), "mnt-.x") # a dot is escaped only at the very start
|
||||
|
||||
|
||||
class Refuses(unittest.TestCase):
|
||||
"""Each rule, with the attack it exists for. Nothing is written on a refusal."""
|
||||
|
||||
def refused(self, h, argv, rule):
|
||||
rc = run(h, *argv)
|
||||
self.assertIn(rc, (2, 3), f"{argv} was accepted")
|
||||
self.assertEqual(h.writes, [], f"{argv} wrote something")
|
||||
self.assertTrue(any(f"[{rule}]" in l for l in h.logs), f"{argv}: rule {rule} not logged: {h.logs}")
|
||||
|
||||
def unit(self, text, name="mnt-hdd_1.mount"):
|
||||
h = FakeHost()
|
||||
h.stage("/var/lib/felhom-agent/units/" + name, text)
|
||||
return h, ["unit", name]
|
||||
|
||||
def test_U1_name_outside_mnt(self):
|
||||
h = FakeHost()
|
||||
self.refused(h, ["unit", "etc-sudoers.d.mount"], "U1")
|
||||
self.refused(h, ["unit", "../../etc/x.mount"], "U1")
|
||||
self.refused(h, ["unit", "mnt-x.service"], "U1")
|
||||
|
||||
def test_U2_service_section(self):
|
||||
self.refused(*self.unit(LOCAL_UNIT + "\n[Service]\nExecStart=/bin/sh -c id\n"), "U2")
|
||||
|
||||
def test_U2_wants_starts_another_unit(self): # review 2026-10-05: Wants=reboot.target would reboot the host
|
||||
for extra in ("Wants=reboot.target", "Requires=felhom-agent-rollback.service", "Before=pve-guests.service"):
|
||||
self.refused(*self.unit(LOCAL_UNIT.replace("After=local-fs-pre.target", "After=local-fs-pre.target\n" + extra)), "U2")
|
||||
self.refused(*self.unit(LOCAL_UNIT.replace("After=local-fs-pre.target", "After=poweroff.target")), "U2")
|
||||
|
||||
def test_U2_continuation_line(self):
|
||||
t = LOCAL_UNIT.replace("Description=Felhom storage mount 91d2dc2d-2d28-4929-9bdd-3e11fa2f41ae",
|
||||
"Description=Felhom storage mount \\")
|
||||
self.refused(*self.unit(t), "U2")
|
||||
self.refused(*self.unit(LOCAL_UNIT.replace("# Managed by felhom-agent", "# comment \\\n# Managed by felhom-agent")), "U2")
|
||||
|
||||
def test_U2_unknown_key(self):
|
||||
self.refused(*self.unit(LOCAL_UNIT.replace("Type=ext4", "Type=ext4\nDirectoryMode=0777")), "U2")
|
||||
|
||||
def test_U3_bind_over_sudoers_dir(self):
|
||||
# the R-861 attack: mount an agent-owned directory over /etc/sudoers.d
|
||||
t = LOCAL_UNIT.replace("Where=/mnt/hdd_1", "Where=/etc/sudoers.d")
|
||||
self.refused(*self.unit(t, "mnt-hdd_1.mount"), "U3")
|
||||
|
||||
def test_U3_name_must_match_where(self):
|
||||
self.refused(*self.unit(LOCAL_UNIT.replace("Where=/mnt/hdd_1", "Where=/mnt/other")), "U3")
|
||||
|
||||
def test_U3_traversal_in_where(self):
|
||||
# the name passes U1 and equals the escaped Where — ONLY the Where rule stops a mount at /mnt/../etc = /etc
|
||||
self.assertEqual(pa.systemd_escape_path("/mnt/../etc") + ".mount", "mnt-..-etc.mount")
|
||||
self.refused(*self.unit(LOCAL_UNIT.replace("Where=/mnt/hdd_1", "Where=/mnt/../etc"), "mnt-..-etc.mount"), "U3")
|
||||
|
||||
def test_U4_what_is_an_agent_directory(self):
|
||||
t = LOCAL_UNIT.replace("What=/dev/disk/by-uuid/91d2dc2d-2d28-4929-9bdd-3e11fa2f41ae", "What=/var/lib/felhom-agent/evil")
|
||||
self.refused(*self.unit(t), "U4")
|
||||
|
||||
def test_U4_tmpfs(self):
|
||||
self.refused(*self.unit(LOCAL_UNIT.replace("Type=ext4", "Type=tmpfs")), "U4")
|
||||
|
||||
def test_U5_bind_option(self):
|
||||
self.refused(*self.unit(LOCAL_UNIT.replace("Type=ext4", "Type=ext4\nOptions=bind")), "U5")
|
||||
|
||||
def test_U5_network_without_nosuid(self):
|
||||
t = NET_MOUNT.replace(",nosuid,nodev", "")
|
||||
self.refused(*self.unit(t, NET_NAME), "U5")
|
||||
|
||||
def test_U5_suid_option(self):
|
||||
self.refused(*self.unit(LOCAL_UNIT.replace("Type=ext4", "Type=ext4\nOptions=suid,dev")), "U5")
|
||||
|
||||
def test_U3_network_outside_drives(self):
|
||||
t = NET_MOUNT.replace("/mnt/felhom-drives/media", "/mnt/media")
|
||||
self.refused(*self.unit(t, "mnt-media.mount"), "U3")
|
||||
|
||||
def test_D1_dhcp_script(self): # runs as root
|
||||
h = FakeHost()
|
||||
h.stage("/tmp/felhom-resolver-1.conf", DNS_BASE + "dhcp-script=/var/lib/felhom-agent/x.sh\n")
|
||||
self.refused(h, ["dnsmasq", "/tmp/felhom-resolver-1.conf", "felhom-x.conf"], "D1")
|
||||
|
||||
def test_D1_conf_dir_and_log_file(self):
|
||||
for extra in ("conf-dir=/var/lib/felhom-agent\n", "log-facility=/etc/sudoers.d/x\n", "user=root\n"):
|
||||
h = FakeHost()
|
||||
h.stage("/tmp/felhom-resolver-1.conf", DNS_GUEST + extra)
|
||||
self.refused(h, ["dnsmasq", "/tmp/felhom-resolver-1.conf", "felhom-x.conf"], "D1")
|
||||
|
||||
def test_D2_paths(self):
|
||||
h = FakeHost()
|
||||
self.refused(h, ["dnsmasq", "/etc/shadow", "felhom-x.conf"], "D2")
|
||||
self.refused(h, ["dnsmasq", "/tmp/felhom-resolver-1.conf", "../sudoers.d/x.conf"], "D2")
|
||||
|
||||
def test_W1_postup(self): # wg-quick runs PostUp as root
|
||||
h = FakeHost()
|
||||
h.stage(pa.WG_SRC, WG.replace("MTU = 1280", "MTU = 1280\nPostUp = /bin/sh -c id"))
|
||||
self.refused(h, ["wg"], "W1")
|
||||
|
||||
def test_W2_values(self):
|
||||
h = FakeHost()
|
||||
h.stage(pa.WG_SRC, WG.replace("AllowedIPs = 10.77.0.1/32, 10.77.0.250/32", "AllowedIPs = 0.0.0.0/0"))
|
||||
self.refused(h, ["wg"], "W2")
|
||||
|
||||
def test_S1_sshd_strictmodes(self): # an AuthorizedKeysFile the agent owns + StrictModes no = root login
|
||||
h = FakeHost()
|
||||
h.stage(pa.SSHD_SRC, pa.render_sshd(8822) + "StrictModes no\n")
|
||||
self.refused(h, ["sshd-config"], "S1")
|
||||
h2 = FakeHost()
|
||||
h2.stage(pa.SSHD_SRC, pa.render_sshd(8822).replace("/etc/felhom-sshd/authorized_keys/%u", "/var/lib/felhom-agent/k"))
|
||||
self.refused(h2, ["sshd-config"], "S1")
|
||||
|
||||
def test_S1_port_22(self):
|
||||
h = FakeHost()
|
||||
h.stage(pa.SSHD_SRC, pa.render_sshd(22))
|
||||
self.refused(h, ["sshd-config"], "S1")
|
||||
|
||||
def test_S2_key_options_and_two_keys(self):
|
||||
h = FakeHost()
|
||||
h.stage(pa.KEY_SRC, 'command="/bin/sh" ' + KEYLINE)
|
||||
self.refused(h, ["sshd-key"], "S2")
|
||||
h2 = FakeHost()
|
||||
h2.stage(pa.KEY_SRC, KEYLINE + KEYLINE)
|
||||
self.refused(h2, ["sshd-key"], "S2")
|
||||
|
||||
def test_P1_symlink_owner_size(self):
|
||||
h = FakeHost()
|
||||
h.stage(pa.WG_SRC, WG, kind="symlink")
|
||||
self.refused(h, ["wg"], "P1")
|
||||
h2 = FakeHost()
|
||||
h2.stage(pa.WG_SRC, WG, uid=0)
|
||||
self.refused(h2, ["wg"], "P1")
|
||||
h3 = FakeHost()
|
||||
h3.stage(pa.WG_SRC, "#" * (pa.MAX_BYTES + 1))
|
||||
self.refused(h3, ["wg"], "P1")
|
||||
|
||||
def test_A1_usage(self):
|
||||
h = FakeHost()
|
||||
self.refused(h, ["install", "/etc/shadow"], "A1")
|
||||
self.refused(h, ["wg", "/etc/shadow"], "A1")
|
||||
|
||||
|
||||
class ControllerImage(unittest.TestCase):
|
||||
"""R-861 (a) A1 (`09` §3 decision 165): the agent can no longer `tee` any image ref into the guest. The root verb
|
||||
reads the ref on stdin, requires our registry + our repository + an x.y.z tag, and writes the guest file itself.
|
||||
RED-PROOF: on the pre-A1 wrapper `controller-image` is not a verb (A1 usage, rc 2) — the accepted case fails."""
|
||||
|
||||
def go(self, ref, *argv):
|
||||
h = FakeHost()
|
||||
h.stdin = ref.encode() if isinstance(ref, str) else ref
|
||||
return h, run(h, *(argv or ("controller-image", "9201")))
|
||||
|
||||
def test_our_controller_ref_is_written_in_the_guest(self):
|
||||
h, rc = self.go("gitea.dooplex.hu/admin/felhom-controller:0.301.0\n")
|
||||
self.assertEqual(rc, 0, h.logs)
|
||||
self.assertEqual(h.guest_writes, [(9201, b"gitea.dooplex.hu/admin/felhom-controller:0.301.0\n")])
|
||||
|
||||
def test_a_foreign_image_is_refused(self):
|
||||
for ref in ("docker.io/library/alpine:latest\n", "alpine\n",
|
||||
"gitea.dooplex.hu/admin/felhom-controller:latest\n",
|
||||
"gitea.dooplex.hu/admin/other:0.1.0\n",
|
||||
"evil.example/admin/felhom-controller:0.301.0\n",
|
||||
"gitea.dooplex.hu/admin/felhom-controller:0.301.0\nalpine\n",
|
||||
"gitea.dooplex.hu/admin/felhom-controller:0.301.0 x\n",
|
||||
"", "\n"):
|
||||
h, rc = self.go(ref)
|
||||
self.assertEqual(rc, 3, f"{ref!r} was accepted")
|
||||
self.assertFalse(h.guest_writes, f"{ref!r} wrote the guest file")
|
||||
self.assertTrue(any("[I1]" in l for l in h.logs), h.logs)
|
||||
|
||||
def test_oversize_stdin_is_refused(self):
|
||||
h, rc = self.go("gitea.dooplex.hu/admin/felhom-controller:0.301.0" + " " * 300)
|
||||
self.assertEqual(rc, 3)
|
||||
self.assertFalse(h.guest_writes)
|
||||
|
||||
def test_vmid_must_be_numeric(self):
|
||||
for argv in (("controller-image", "9201;id"), ("controller-image", "-1"), ("controller-image",),
|
||||
("controller-image", "9201", "9202")):
|
||||
h, rc = self.go("gitea.dooplex.hu/admin/felhom-controller:0.301.0\n", *argv)
|
||||
self.assertIn(rc, (2, 3), argv)
|
||||
self.assertFalse(h.guest_writes, argv)
|
||||
|
||||
def test_self_check_names_the_verb(self):
|
||||
import io, contextlib
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
pa.main(["--self-check"])
|
||||
self.assertIn("controller-image", buf.getvalue())
|
||||
|
||||
|
||||
class UnitForget(unittest.TestCase):
|
||||
"""R-1124: the agent forgets a decommissioned drive whose disk is gone — the root half removes its unit file, and
|
||||
ONLY a felhom by-UUID drive unit that is disabled, not mounted, and whose /dev/disk/by-uuid link is absent.
|
||||
RED-PROOF: on the pre-R-1124 wrapper `unit-forget` is not a verb (A1 usage, rc 2) — the removal case fails."""
|
||||
NAME = "mnt-hdd_1.mount"
|
||||
PATH = "/etc/systemd/system/mnt-hdd_1.mount"
|
||||
UUID_LINK = "/dev/disk/by-uuid/91d2dc2d-2d28-4929-9bdd-3e11fa2f41ae"
|
||||
|
||||
def host(self, text=LOCAL_UNIT, kind="file", links=(), mounted=()):
|
||||
h = FakeHost()
|
||||
h.installed = {} if text is None else {self.PATH: (kind, text.encode())}
|
||||
h.links, h.mounted, h.removed = set(links), set(mounted), []
|
||||
h.read_installed = lambda p: (None if p not in h.installed else
|
||||
(_ for _ in ()).throw(pa.Refused("F1", f"{p} is not a regular file"))
|
||||
if h.installed[p][0] != "file" else h.installed[p][1])
|
||||
h.lexists = lambda p: p in h.links
|
||||
h.mountpoints = lambda: set(h.mounted)
|
||||
h.remove = lambda p: h.removed.append(p)
|
||||
return h
|
||||
|
||||
def test_an_absent_disabled_unmounted_drive_unit_is_removed(self):
|
||||
h = self.host()
|
||||
self.assertEqual(run(h, "unit-forget", self.NAME), 0, h.logs)
|
||||
self.assertEqual(h.removed, [self.PATH])
|
||||
|
||||
def test_already_gone_is_fine(self):
|
||||
h = self.host(text=None)
|
||||
self.assertEqual(run(h, "unit-forget", self.NAME), 0, h.logs)
|
||||
self.assertEqual(h.removed, [])
|
||||
|
||||
def refused(self, h, rule, *argv):
|
||||
rc = run(h, *(argv or ("unit-forget", self.NAME)))
|
||||
self.assertIn(rc, (2, 3), h.logs)
|
||||
self.assertEqual(h.removed, [], h.logs)
|
||||
self.assertTrue(any(f"[{rule}]" in l for l in h.logs), h.logs)
|
||||
|
||||
def test_F5_a_present_drive_keeps_its_unit(self):
|
||||
self.refused(self.host(links=[self.UUID_LINK]), "F5")
|
||||
|
||||
def test_F4_a_mounted_drive_keeps_its_unit(self):
|
||||
self.refused(self.host(mounted=["/mnt/hdd_1"]), "F4")
|
||||
|
||||
def test_F3_an_enabled_unit_is_not_removed(self):
|
||||
self.refused(self.host(links=["/etc/systemd/system/multi-user.target.wants/" + self.NAME]), "F3")
|
||||
|
||||
def test_F2_a_network_unit_or_a_foreign_unit_is_not_removed(self):
|
||||
self.refused(self.host(text=LOCAL_UNIT.replace("Managed by felhom-agent", "Written by someone")), "F2")
|
||||
net = LOCAL_UNIT.replace("# Managed by felhom-agent", "# Managed by felhom-agent felhom network storage")
|
||||
self.refused(self.host(text=net), "F2")
|
||||
|
||||
def test_F1_a_symlinked_unit_is_not_followed(self):
|
||||
self.refused(self.host(kind="symlink"), "F1")
|
||||
|
||||
def test_A1_names(self):
|
||||
for argv in (("unit-forget", "mnt-hdd_1.automount"), ("unit-forget", "sshd.service"),
|
||||
("unit-forget", "../mnt-hdd_1.mount"), ("unit-forget",), ("unit-forget", self.NAME, "x")):
|
||||
self.refused(self.host(), "A1", *argv)
|
||||
|
||||
def test_U3_a_unit_whose_name_lies_about_where_is_not_removed(self):
|
||||
self.refused(self.host(text=LOCAL_UNIT.replace("Where=/mnt/hdd_1", "Where=/mnt/other")), "U3")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for configs/felhom-thin-check (R-1012): it lists the thin pool's device ids next to LVM's and NEVER removes
|
||||
anything. Fake `lvs`, `dmsetup` and `thin_ls` record every call; nothing real is touched. Each refusal has a positive
|
||||
control (the healthy read), so a test cannot pass because the helper refuses everything.
|
||||
Run: python3 configs/test_felhom_thin_check.py (also run by internal/thincheck's Go test)
|
||||
"""
|
||||
import os, pathlib, re, subprocess, tempfile, unittest
|
||||
|
||||
HELPER = pathlib.Path(__file__).resolve().parent / "felhom-thin-check"
|
||||
|
||||
|
||||
class Box:
|
||||
def __init__(self, pool_ids=(1, 2), lvm=(("vm-9201-disk-0", 1), ("vm-9201-disk-1", 2)), attr="twi-aotz--",
|
||||
held="-", reserve_rc=0, thin_ls_rc=0):
|
||||
self.t = tempfile.TemporaryDirectory()
|
||||
r = pathlib.Path(self.t.name)
|
||||
self.log = r / "calls"
|
||||
self.log.write_text("")
|
||||
lvm_lines = "\\n".join(f" {n},{i}" for n, i in lvm)
|
||||
pool_lines = "\\n".join(str(i) for i in pool_ids)
|
||||
self.bins = {}
|
||||
for name, body in {
|
||||
"lvs": f'''case "$*" in *lv_attr*) echo " {attr}";; *thin_id*) printf "{lvm_lines}\\n";; esac''',
|
||||
"dmsetup": f'''case "$1" in
|
||||
status) echo "0 112975872 thin-pool 585 6860/262144 517080/882624 {held} rw discard_passdown queue_if_no_space - 1024";;
|
||||
message) [ "$4" = reserve_metadata_snap ] && exit {reserve_rc}; exit 0;;
|
||||
esac''',
|
||||
"thin_ls": f'''[ {thin_ls_rc} -ne 0 ] && exit {thin_ls_rc}; printf "{pool_lines}\\n"''',
|
||||
}.items():
|
||||
p = r / name
|
||||
p.write_text(f'#!/bin/bash\necho "{name} $*" >> "{self.log}"\n{body}\n')
|
||||
p.chmod(0o755)
|
||||
self.bins[name] = p
|
||||
self.lock = r / "lock"
|
||||
|
||||
def run(self, *args):
|
||||
script = (f'source "{HELPER}"; LVS="{self.bins["lvs"]}"; DMSETUP="{self.bins["dmsetup"]}"; '
|
||||
f'THIN_LS="{self.bins["thin_ls"]}"; LOCK="{self.lock}"; main "$@"')
|
||||
r = subprocess.run(["bash", "-c", script, "x", *args], capture_output=True, text=True)
|
||||
return r.returncode, r.stdout, r.stderr
|
||||
|
||||
def calls(self):
|
||||
return self.log.read_text().splitlines()
|
||||
|
||||
|
||||
def messages(calls):
|
||||
return [c.split()[-1] for c in calls if c.startswith("dmsetup message")]
|
||||
|
||||
|
||||
class Reads(unittest.TestCase):
|
||||
def test_healthy_pool_lists_both_sides_and_releases(self):
|
||||
b = Box()
|
||||
rc, out, err = b.run("pve", "data")
|
||||
self.assertEqual(rc, 0, err)
|
||||
self.assertEqual(out.splitlines(), ["lvm-before 1 vm-9201-disk-0", "lvm-before 2 vm-9201-disk-1",
|
||||
"pool 1", "pool 2",
|
||||
"lvm-after 1 vm-9201-disk-0", "lvm-after 2 vm-9201-disk-1", "ok"])
|
||||
self.assertEqual(messages(b.calls()), ["reserve_metadata_snap", "release_metadata_snap"])
|
||||
self.assertIn("thin_ls --metadata-snap --no-headers -o DEV /dev/mapper/pve-data_tmeta", b.calls())
|
||||
|
||||
def test_the_2026_10_14_orphan_is_listed(self):
|
||||
rc, out, _ = Box(pool_ids=(1, 2, 4)).run("pve", "data")
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertIn("pool 4", out.splitlines())
|
||||
|
||||
def test_dashed_names_map_to_the_device_mapper_name(self):
|
||||
b = Box()
|
||||
rc, _, err = b.run("felhom-scratch", "thin-pool")
|
||||
self.assertEqual(rc, 0, err)
|
||||
self.assertIn("dmsetup status felhom--scratch-thin--pool-tpool", b.calls())
|
||||
|
||||
|
||||
class Refuses(unittest.TestCase):
|
||||
def refused(self, box, rc_want, *args):
|
||||
rc, out, err = box.run(*args)
|
||||
self.assertEqual(rc, rc_want, (out, err))
|
||||
self.assertNotIn("ok", out.splitlines())
|
||||
return box.calls()
|
||||
|
||||
def test_arguments(self):
|
||||
for args in (("pve",), ("pve", "data", "x"), ("pve;id", "data"), ("-pve", "data"), ("pve", "../data"),
|
||||
("pve", "da ta")):
|
||||
calls = self.refused(Box(), 1, *args)
|
||||
self.assertEqual(calls, [], args)
|
||||
|
||||
def test_not_a_thin_pool(self):
|
||||
calls = self.refused(Box(attr="Vwi-aotz--"), 1, "pve", "vm-9201-disk-1")
|
||||
self.assertEqual(messages(calls), [])
|
||||
|
||||
def test_a_snapshot_someone_else_holds_is_never_touched(self):
|
||||
b = Box(held="1234")
|
||||
rc, _, err = b.run("pve", "data")
|
||||
self.assertEqual(rc, 3)
|
||||
self.assertEqual(messages(b.calls()), []) # neither reserve nor release
|
||||
# the reason line ends with the reason, not the exit code (found live on demo-hp 2026-10-17)
|
||||
self.assertTrue(err.strip().endswith("not touched"), err)
|
||||
|
||||
def test_reserve_failure_does_not_release(self):
|
||||
calls = self.refused(Box(reserve_rc=1), 3, "pve", "data")
|
||||
self.assertEqual(messages(calls), ["reserve_metadata_snap"])
|
||||
|
||||
def test_thin_ls_failure_still_releases(self):
|
||||
calls = self.refused(Box(thin_ls_rc=1), 4, "pve", "data")
|
||||
self.assertEqual(messages(calls), ["reserve_metadata_snap", "release_metadata_snap"])
|
||||
|
||||
|
||||
class NeverDeletes(unittest.TestCase):
|
||||
def test_the_only_pool_messages_are_reserve_and_release(self):
|
||||
src = HELPER.read_text()
|
||||
sent = re.findall(r'"\$DMSETUP" message "\$TPOOL" 0 (\S+)', src)
|
||||
self.assertEqual(sorted(set(sent)), ["release_metadata_snap", "reserve_metadata_snap"])
|
||||
for word in ("delete", "lvremove", "thin_repair", "create_thin", "create_snap", " rm "):
|
||||
code = "\n".join(l for l in src.splitlines() if not l.lstrip().startswith("#"))
|
||||
self.assertNotIn(word, code, word)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for configs/felhom-vzdump-partial-clean (R-1006): it removes a DEAD half-written vzdump backup and refuses
|
||||
everything else. Each refusal has a positive control: the same setup without the one condition is removed — so a
|
||||
test cannot pass because the helper refuses everything. A fake storage.cfg and a fake /proc; nothing real is touched.
|
||||
Run: python3 configs/test_felhom_vzdump_partial_clean.py (also run by internal/backup's Go test)
|
||||
"""
|
||||
import os, pathlib, subprocess, tempfile, time, unittest
|
||||
|
||||
HELPER = pathlib.Path(__file__).resolve().parent / "felhom-vzdump-partial-clean"
|
||||
BOOT = 1_791_967_481
|
||||
NAME = "vzdump-lxc-9201-2026_10_14-10_42_25"
|
||||
|
||||
|
||||
class Box:
|
||||
"""A temp tree: a dir storage `local` (path vz), an lvmthin `local-lvm`, a dir storage `other`, and a fake /proc."""
|
||||
|
||||
def __init__(self):
|
||||
self.t = tempfile.TemporaryDirectory()
|
||||
r = pathlib.Path(self.t.name)
|
||||
self.vz, self.other, self.nas, self.proc = r / "vz", r / "other", r / "nas", r / "proc"
|
||||
for d in (self.vz / "dump", self.other / "dump", self.nas / "dump", self.proc):
|
||||
d.mkdir(parents=True)
|
||||
(self.proc / "stat").write_text(f"cpu 1 2 3\nbtime {BOOT}\n")
|
||||
self.cfg = r / "storage.cfg"
|
||||
self.cfg.write_text(f"dir: local\n\tpath {self.vz}\n\tcontent iso,backup\n\n"
|
||||
f"lvmthin: local-lvm\n\tthinpool data\n\tvgname pve\n\n"
|
||||
f"dir: other\n\tpath {self.other}\n\tcontent backup\n\n"
|
||||
f"nfs: nas\n\tpath {self.nas}\n\tserver 192.0.2.1\n\texport /x\n\tcontent backup\n")
|
||||
self.pid = 1000
|
||||
|
||||
def partial(self, name=NAME + ".tar.dat", mtime=BOOT - 100, where=None, size=4096):
|
||||
p = (where or self.vz / "dump") / name
|
||||
if name.endswith(".tmp"):
|
||||
(p / "etc" / "vzdump").mkdir(parents=True)
|
||||
(p / "etc" / "vzdump" / "pct.conf").write_text("x")
|
||||
for q in [p / "etc" / "vzdump" / "pct.conf", p / "etc" / "vzdump", p / "etc", p]:
|
||||
os.utime(q, (mtime, mtime))
|
||||
else:
|
||||
p.write_bytes(b"\0" * size)
|
||||
os.utime(p, (mtime, mtime))
|
||||
return p
|
||||
|
||||
def process(self, cmdline, fds=(), cwd=None):
|
||||
self.pid += 1
|
||||
d = self.proc / str(self.pid)
|
||||
(d / "fd").mkdir(parents=True)
|
||||
(d / "cmdline").write_bytes(b"\0".join(a.encode() for a in cmdline) + b"\0")
|
||||
for i, f in enumerate(fds):
|
||||
os.symlink(str(f), d / "fd" / str(i + 3))
|
||||
os.symlink(str(cwd or "/"), d / "cwd")
|
||||
|
||||
def run(self, storage, vmid, name, quiet=120):
|
||||
script = (f'source "{HELPER}"; STORAGE_CFG="{self.cfg}"; PROC="{self.proc}"; QUIET_S={quiet}; '
|
||||
f'main "$@"')
|
||||
r = subprocess.run(["bash", "-c", script, "x", storage, str(vmid), name], capture_output=True, text=True)
|
||||
return r.returncode, r.stdout + r.stderr
|
||||
|
||||
|
||||
class Removes(unittest.TestCase):
|
||||
def test_dead_tar_dat_before_boot(self):
|
||||
b = Box(); p = b.partial()
|
||||
rc, out = b.run("local", 9201, p.name)
|
||||
self.assertEqual(rc, 0, out); self.assertFalse(p.exists()); self.assertIn("written before this boot", out)
|
||||
|
||||
def test_dead_tmp_dir_before_boot(self):
|
||||
b = Box(); p = b.partial(NAME + ".tmp")
|
||||
rc, out = b.run("local", 9201, p.name)
|
||||
self.assertEqual(rc, 0, out); self.assertFalse(p.exists())
|
||||
|
||||
def test_quiet_partial_after_boot(self):
|
||||
b = Box(); p = b.partial(mtime=int(time.time()) - 300)
|
||||
rc, out = b.run("local", 9201, p.name)
|
||||
self.assertEqual(rc, 0, out); self.assertFalse(p.exists()); self.assertIn("untouched", out)
|
||||
|
||||
def test_qemu_vma_dat(self):
|
||||
b = Box(); p = b.partial("vzdump-qemu-341-2026_10_14-10_42_25.vma.dat")
|
||||
rc, out = b.run("local", 341, p.name)
|
||||
self.assertEqual(rc, 0, out); self.assertFalse(p.exists())
|
||||
|
||||
|
||||
class Keeps(unittest.TestCase):
|
||||
"""Each case: (setup that must be REFUSED, the same setup that must be REMOVED)."""
|
||||
|
||||
def pair(self, refuse, control, quiet=120):
|
||||
b = Box(); target, args = refuse(b)
|
||||
rc, out = b.run(*args, quiet=quiet)
|
||||
self.assertNotEqual(rc, 0, f"must refuse: {out}")
|
||||
self.assertIn("REFUSED", out)
|
||||
self.assertTrue(os.path.lexists(target), "a refused entry must still exist")
|
||||
c = Box(); ctarget, cargs = control(c)
|
||||
crc, cout = c.run(*cargs, quiet=quiet)
|
||||
self.assertEqual(crc, 0, f"positive control must be removed: {cout}")
|
||||
self.assertFalse(os.path.lexists(ctarget))
|
||||
|
||||
def test_complete_archive_of_same_vmid(self):
|
||||
self.pair(lambda b: (b.partial(NAME + ".tar.zst"), ("local", 9201, NAME + ".tar.zst")),
|
||||
lambda b: (b.partial(), ("local", 9201, NAME + ".tar.dat")))
|
||||
|
||||
def test_log_and_notes(self):
|
||||
for suffix in (".log", ".tar.zst.notes"):
|
||||
self.pair(lambda b, s=suffix: (b.partial(NAME + s), ("local", 9201, NAME + s)),
|
||||
lambda b: (b.partial(), ("local", 9201, NAME + ".tar.dat")))
|
||||
|
||||
def test_partial_of_another_vmid(self):
|
||||
other = "vzdump-lxc-9202-2026_10_14-10_42_25.tar.dat"
|
||||
self.pair(lambda b: (b.partial(other), ("local", 9201, other)),
|
||||
lambda b: (b.partial(other), ("local", 9202, other)))
|
||||
|
||||
def test_partial_on_a_non_dir_storage(self):
|
||||
self.pair(lambda b: (b.partial(), ("local-lvm", 9201, NAME + ".tar.dat")),
|
||||
lambda b: (b.partial(), ("local", 9201, NAME + ".tar.dat")))
|
||||
|
||||
def test_partial_on_a_network_storage_with_a_path(self):
|
||||
# an nfs storage HAS a path: only the „dir only" rule (both its guards) can refuse it
|
||||
self.pair(lambda b: (b.partial(where=b.nas / "dump"), ("nas", 9201, NAME + ".tar.dat")),
|
||||
lambda b: (b.partial(where=b.other / "dump"), ("other", 9201, NAME + ".tar.dat")))
|
||||
|
||||
def test_partial_on_another_storage_than_named(self):
|
||||
self.pair(lambda b: (b.partial(where=b.other / "dump"), ("local", 9201, NAME + ".tar.dat")),
|
||||
lambda b: (b.partial(where=b.other / "dump"), ("other", 9201, NAME + ".tar.dat")))
|
||||
|
||||
def test_unknown_storage(self):
|
||||
self.pair(lambda b: (b.partial(), ("nosuch", 9201, NAME + ".tar.dat")),
|
||||
lambda b: (b.partial(), ("local", 9201, NAME + ".tar.dat")))
|
||||
|
||||
def test_vzdump_task_in_flight(self):
|
||||
def refuse(b):
|
||||
p = b.partial(); b.process(["task UPID:demo-hp:001:002:6ACF4071:vzdump:9201:felhom-agent@pve!agent:"]); return p, ("local", 9201, p.name)
|
||||
def control(b):
|
||||
p = b.partial(); b.process(["task UPID:demo-hp:001:002:6ACF4071:vzrestore:9202:root@pam:"]); return p, ("local", 9201, p.name)
|
||||
self.pair(refuse, control)
|
||||
|
||||
def test_any_vzdump_job_even_without_the_vmid(self):
|
||||
# a job for several guests: `vzdump::` — and a vzdump for ANOTHER guest — both refuse
|
||||
for cmd in (["task UPID:demo-hp:001:002:6ACF4071:vzdump::root@pam:"],
|
||||
["task UPID:demo-hp:001:002:6ACF4071:vzdump:9202:felhom-agent@pve!agent:"]):
|
||||
def refuse(b, c=cmd):
|
||||
p = b.partial(); b.process(c); return p, ("local", 9201, p.name)
|
||||
def control(b):
|
||||
p = b.partial(); b.process(["task UPID:demo-hp:001:002:6ACF4071:vzstart:9202:root@pam:"]); return p, ("local", 9201, p.name)
|
||||
self.pair(refuse, control)
|
||||
|
||||
def test_vzdump_command_for_the_vmid(self):
|
||||
def refuse(b):
|
||||
p = b.partial(); b.process(["/usr/bin/perl", "/usr/bin/vzdump", "9201", "--storage", "local"]); return p, ("local", 9201, p.name)
|
||||
def control(b):
|
||||
p = b.partial(); b.process(["/usr/bin/perl", "/usr/bin/vzdumpfoo", "9201"]); return p, ("local", 9201, p.name)
|
||||
self.pair(refuse, control)
|
||||
|
||||
def test_file_held_open_by_an_orphan_writer(self):
|
||||
def refuse(b):
|
||||
p = b.partial(); b.process(["zstd", "--threads=1"], fds=[p]); return p, ("local", 9201, p.name)
|
||||
def control(b):
|
||||
p = b.partial(); b.process(["zstd", "--threads=1"], fds=[b.vz / "dump" / "something-else"]); return p, ("local", 9201, p.name)
|
||||
self.pair(refuse, control)
|
||||
|
||||
def test_tmp_dir_with_a_process_inside(self):
|
||||
def refuse(b):
|
||||
p = b.partial(NAME + ".tmp"); b.process(["tar", "cpf", "-"], cwd=p / "etc"); return p, ("local", 9201, p.name)
|
||||
def control(b):
|
||||
p = b.partial(NAME + ".tmp"); b.process(["tar", "cpf", "-"], cwd=b.vz); return p, ("local", 9201, p.name)
|
||||
self.pair(refuse, control)
|
||||
|
||||
def test_recently_written_after_boot(self):
|
||||
self.pair(lambda b: (b.partial(mtime=int(time.time()) - 10), ("local", 9201, NAME + ".tar.dat")),
|
||||
lambda b: (b.partial(mtime=int(time.time()) - 300), ("local", 9201, NAME + ".tar.dat")))
|
||||
|
||||
def test_symlink(self):
|
||||
def refuse(b):
|
||||
real = b.partial(where=b.vz)
|
||||
link = b.vz / "dump" / (NAME + ".tar.dat")
|
||||
os.symlink(real, link)
|
||||
return link, ("local", 9201, link.name)
|
||||
# quiet=0: the link was made just now, so only the symlink rule may refuse it
|
||||
self.pair(refuse, lambda b: (b.partial(), ("local", 9201, NAME + ".tar.dat")), quiet=0)
|
||||
|
||||
def test_near_miss_names(self):
|
||||
for bad in (NAME + ".tar.dat.bak", "vzdump-lxc-92011-2026_10_14-10_42_25.tar.dat", NAME + ".tar.da",
|
||||
"x" + NAME + ".tar.dat", NAME + ".TAR.DAT", "vzdump-lxc-9201-2026_10_14-10_42.tar.dat"):
|
||||
self.pair(lambda b, n=bad: (b.partial(n), ("local", 9201, n)),
|
||||
lambda b: (b.partial(), ("local", 9201, NAME + ".tar.dat")))
|
||||
|
||||
def test_path_traversal_and_bad_args(self):
|
||||
b = Box(); p = b.partial()
|
||||
for args in (("local", 9201, "../dump/" + p.name), ("local", 9201, ""), ("loc al", 9201, p.name),
|
||||
("local", "9201x", p.name), ("local", 12, p.name)):
|
||||
rc, out = b.run(*args)
|
||||
self.assertNotEqual(rc, 0, f"{args}: {out}")
|
||||
self.assertTrue(p.exists())
|
||||
|
||||
def test_wrong_kind(self):
|
||||
def refuse(b): # a FILE named .tmp
|
||||
p = b.vz / "dump" / (NAME + ".tmp"); p.write_bytes(b"x"); os.utime(p, (BOOT - 100, BOOT - 100)); return p, ("local", 9201, p.name)
|
||||
self.pair(refuse, lambda b: (b.partial(NAME + ".tmp"), ("local", 9201, NAME + ".tmp")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Binary file not shown.
@@ -0,0 +1,8 @@
|
||||
module gitea.dooplex.hu/admin/felhom-agent
|
||||
|
||||
go 1.27.3
|
||||
|
||||
require (
|
||||
golang.org/x/crypto v0.58.0
|
||||
golang.org/x/sys v0.49.0
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM=
|
||||
golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY=
|
||||
@@ -0,0 +1,81 @@
|
||||
package authz
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CanonicalBlob builds the canonical OpBlob bytes (phase4 §2 field order: keys sorted at
|
||||
// every level, no insignificant whitespace, no trailing newline, UTF-8). This is the SINGLE
|
||||
// production source of the signed bytes — the operator signing CLI (cmd/felhom-opsign) and the
|
||||
// in-Go test minting both call it, so the signer can NEVER drift from what the verifier expects
|
||||
// (the verifier authenticates over the RAW received bytes, so these bytes ARE the contract).
|
||||
//
|
||||
// params is canonicalized internally (parsed + re-marshaled → object keys sorted, whitespace
|
||||
// stripped) so the same op+params always yields identical bytes; "" → "{}". Returns an error
|
||||
// only when params is not valid JSON.
|
||||
func CanonicalBlob(op, hostID, guestID, keyID, nonce, paramsJSON string, issued, expires time.Time) ([]byte, error) {
|
||||
params := strings.TrimSpace(paramsJSON)
|
||||
if params == "" {
|
||||
params = "{}"
|
||||
}
|
||||
var pv interface{}
|
||||
if err := json.Unmarshal([]byte(params), &pv); err != nil {
|
||||
return nil, fmt.Errorf("authz: params is not valid JSON: %w", err)
|
||||
}
|
||||
pc, err := json.Marshal(pv) // Go marshals object keys sorted, compact
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("authz: canonicalizing params: %w", err)
|
||||
}
|
||||
return []byte(fmt.Sprintf(
|
||||
`{"expires_at":%q,"issued_at":%q,"key_id":%q,"nonce":%q,"op":%q,"params":%s,"target":{"guest_id":%q,"host_id":%q}}`,
|
||||
expires.UTC().Format(time.RFC3339), issued.UTC().Format(time.RFC3339),
|
||||
keyID, nonce, op, pc, guestID, hostID)), nil
|
||||
}
|
||||
|
||||
// Target binds an op to a specific box (and optionally a guest) — the anti-retarget
|
||||
// field. The §7 reference omitted the json tags; production needs them so the
|
||||
// signed canonical bytes decode correctly.
|
||||
type Target struct {
|
||||
HostID string `json:"host_id"`
|
||||
GuestID string `json:"guest_id"`
|
||||
}
|
||||
|
||||
// OpBlob is the canonical signed object (phase4 §2). The signature covers the
|
||||
// EXACT bytes of this object's canonical JSON (keys sorted at every level, no
|
||||
// insignificant whitespace, no trailing newline, UTF-8) — produced by the
|
||||
// operator CLI / hub, verified here over the raw received bytes.
|
||||
type OpBlob struct {
|
||||
Op string `json:"op"`
|
||||
Target Target `json:"target"`
|
||||
Params json.RawMessage `json:"params"`
|
||||
Nonce string `json:"nonce"`
|
||||
IssuedAt time.Time `json:"issued_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
KeyID string `json:"key_id"`
|
||||
}
|
||||
|
||||
// VerifiedOp is the authenticated, parsed op returned on success — everything the
|
||||
// reconcile layer (slice 4) needs to route and execute, not just the op string.
|
||||
type VerifiedOp struct {
|
||||
Op string // the operation, e.g. "guest_destroy"
|
||||
HostID string // target host (== this agent's host)
|
||||
// GuestID is non-empty for a guest-scoped op; the caller routes by it. "" =
|
||||
// host-scoped op. The verifier does NOT need to know all guest ids.
|
||||
GuestID string
|
||||
Params json.RawMessage
|
||||
Nonce string
|
||||
IssuedAt time.Time
|
||||
ExpiresAt time.Time
|
||||
|
||||
// KeyID is the blob's self-declared key id — ADVISORY / audit only, never an
|
||||
// authz input. Authz is the key-material allow-list match (Signer below).
|
||||
KeyID string
|
||||
// Signer is the allow-listed key whose material matched the signature.
|
||||
Signer AllowedSigner
|
||||
// KeyIDMatchesSigner is false when the blob's advisory KeyID disagrees with
|
||||
// the matched signer's id (a benign audit signal, not a rejection).
|
||||
KeyIDMatchesSigner bool
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
// Package authz is the control-plane-authorization layer: it verifies
|
||||
// operator-signed destructive ops before the agent executes them. It is what the
|
||||
// reconcile loop (slice 4) calls to gate destructive desired-state deltas and
|
||||
// signed one-shot jobs (03 §4, 04). The signing mechanism is proven (Phase 4,
|
||||
// 14/14) — this package is its production form: a key-type-agnostic SSHSIG
|
||||
// verifier, the full anti-replay/authorization pipeline, and a durable,
|
||||
// crash-safe nonce store.
|
||||
//
|
||||
// # Mechanism (LOCKED — do not redesign)
|
||||
//
|
||||
// - SSHSIG via golang.org/x/crypto/ssh; no hand-rolled crypto, no raw-Ed25519
|
||||
// fallback. pub.Verify dispatches on the key's own algorithm, so the same path
|
||||
// accepts ed25519 / sk-ssh-ed25519 (FIDO2) / rsa / ecdsa — a hardware operator
|
||||
// key later is a box no-op (Phase 4 §5/§6, doc 04 §7).
|
||||
// - Fixed namespace felhom-op-v1 (package constant, never caller-supplied).
|
||||
// - The verifier verifies over the RAW received blob bytes and never
|
||||
// canonicalizes — the canonical form (sorted-key, whitespace-free JSON) is the
|
||||
// signer's contract, shared by the hub and the felhom-sign CLI.
|
||||
//
|
||||
// # Pipeline order (load-bearing — Verify)
|
||||
//
|
||||
// parse armor → namespace → parse pubkey → allow-list (by key MATERIAL, not
|
||||
// key_id) → crypto verify → parse blob → target → time window → nonce LAST
|
||||
//
|
||||
// Each post-crypto stage rejects even with an otherwise-valid signature. The nonce
|
||||
// is recorded last, so an invalid signature can never consume a nonce. key_id is
|
||||
// advisory/audit only — authz is the key-material allow-list match.
|
||||
//
|
||||
// # Shared-contract dependency (flag for later, not built here)
|
||||
//
|
||||
// Signatures only verify if the op-generator (hub) and the felhom-sign CLI produce
|
||||
// BYTE-IDENTICAL canonical JSON (keys sorted at every level, no insignificant
|
||||
// whitespace, no trailing newline, UTF-8 — Phase 4 §2). The verifier deliberately
|
||||
// does NOT re-canonicalize, so a divergence between those two producers surfaces as
|
||||
// a crypto failure here. A shared canonicalizer that both import would be the right
|
||||
// home for that contract; it is out of scope for this slice.
|
||||
package authz
|
||||
@@ -0,0 +1,29 @@
|
||||
package authz
|
||||
|
||||
import "errors"
|
||||
|
||||
// Typed rejection sentinels — one per pipeline stage so the reconcile layer can
|
||||
// distinguish "rejected" (a real signed op that failed a check) from "malformed"
|
||||
// from a future "not yet signed". All are errors.Is-friendly: Verify wraps them
|
||||
// with %w plus context.
|
||||
var (
|
||||
// ErrMalformed: the armor/SSHSIG/blob could not be parsed (not a rejection of
|
||||
// a well-formed op — bad input).
|
||||
ErrMalformed = errors.New("authz: malformed signature or blob")
|
||||
// ErrNamespace: SSHSIG namespace != the fixed felhom-op-v1 domain separator.
|
||||
ErrNamespace = errors.New("authz: namespace mismatch")
|
||||
// ErrUnknownSigner: the signing key's material is not in the pinned allow-list.
|
||||
ErrUnknownSigner = errors.New("authz: signer not in allowed set")
|
||||
// ErrBadSignature: cryptographic verification failed (tamper / wrong key).
|
||||
ErrBadSignature = errors.New("authz: signature did not verify")
|
||||
// ErrTarget: target.host_id is not this box.
|
||||
ErrTarget = errors.New("authz: target mismatch")
|
||||
// ErrExpired: now > expires_at.
|
||||
ErrExpired = errors.New("authz: op expired")
|
||||
// ErrNotYetValid: now < issued_at (minus clock-skew tolerance).
|
||||
ErrNotYetValid = errors.New("authz: op not yet valid")
|
||||
// ErrWindowTooLong: expires_at - issued_at > MaxWindow (R-1280).
|
||||
ErrWindowTooLong = errors.New("authz: op validity window too long")
|
||||
// ErrReplay: the nonce was already recorded in the window.
|
||||
ErrReplay = errors.New("authz: replay (nonce already seen)")
|
||||
)
|
||||
@@ -0,0 +1,103 @@
|
||||
package authz
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/pem"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// Test helpers that MINT armored SSHSIGs in-Go (hermetic) — the inverse of the
|
||||
// production framing. They reuse the production signedData()/sshsigBlob so a test
|
||||
// can never drift from the verifier's notion of the signed bytes.
|
||||
|
||||
// canonicalBlob delegates to the production CanonicalBlob (so the in-Go test minting can never
|
||||
// drift from the real signed-bytes path). Panics on a params error — tests pass valid JSON.
|
||||
func canonicalBlob(op, hostID, guestID, keyID, nonce, paramsJSON string, issued, expires time.Time) []byte {
|
||||
b, err := CanonicalBlob(op, hostID, guestID, keyID, nonce, paramsJSON, issued, expires)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// mintArmor builds an armored SSHSIG over message, using sign to produce the inner
|
||||
// ssh.Signature over the recomputed SSHSIG signed-data.
|
||||
func mintArmor(t *testing.T, pubMarshaled []byte, namespace, hashName string, message []byte, sign func([]byte) ssh.Signature) []byte {
|
||||
t.Helper()
|
||||
sb := &sshsigBlob{Version: 1, PublicKey: string(pubMarshaled), Namespace: namespace, Reserved: "", HashAlgo: hashName}
|
||||
signed, err := signedData(sb, message)
|
||||
if err != nil {
|
||||
t.Fatalf("signedData: %v", err)
|
||||
}
|
||||
sig := sign(signed)
|
||||
sb.Signature = string(ssh.Marshal(&sig))
|
||||
raw := append([]byte(sshsigMagic), ssh.Marshal(sb)...)
|
||||
return pem.EncodeToMemory(&pem.Block{Type: "SSH SIGNATURE", Bytes: raw})
|
||||
}
|
||||
|
||||
// newEd25519Signer returns an ssh.PublicKey + a sign closure for a fresh ed25519 key.
|
||||
func newEd25519Signer(t *testing.T) (ssh.PublicKey, func([]byte) ssh.Signature) {
|
||||
t.Helper()
|
||||
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sshPub, err := ssh.NewPublicKey(pub)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sign := func(signed []byte) ssh.Signature {
|
||||
return ssh.Signature{Format: ssh.KeyAlgoED25519, Blob: ed25519.Sign(priv, signed)}
|
||||
}
|
||||
return sshPub, sign
|
||||
}
|
||||
|
||||
// newSyntheticSKSigner emulates a FIDO2 sk-ssh-ed25519@openssh.com key with NO
|
||||
// hardware (Phase 4 §5). It builds a spec-faithful sk public key and an sk-format
|
||||
// signature: ed25519 over sha256(application)‖flags‖counter‖sha256(signed_data),
|
||||
// sig.Blob = the raw ed25519 signature, sig.Rest = flags‖counter. It must verify
|
||||
// through the UNCHANGED Verify path.
|
||||
func newSyntheticSKSigner(t *testing.T) (ssh.PublicKey, func([]byte) ssh.Signature) {
|
||||
t.Helper()
|
||||
edPub, edPriv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const application = "ssh:"
|
||||
skBlob := ssh.Marshal(struct {
|
||||
Name string
|
||||
KeyBytes []byte
|
||||
Application string
|
||||
}{"sk-ssh-ed25519@openssh.com", []byte(edPub), application})
|
||||
skPub, err := ssh.ParsePublicKey(skBlob)
|
||||
if err != nil {
|
||||
t.Fatalf("parse synthetic sk pubkey: %v", err)
|
||||
}
|
||||
if skPub.Type() != "sk-ssh-ed25519@openssh.com" {
|
||||
t.Fatalf("sk pubkey type = %q", skPub.Type())
|
||||
}
|
||||
|
||||
sign := func(signed []byte) ssh.Signature {
|
||||
const flagUserPresence = byte(0x01) // required, else Verify rejects
|
||||
const counter = uint32(1)
|
||||
appDigest := sha256.Sum256([]byte(application))
|
||||
dataDigest := sha256.Sum256(signed)
|
||||
// original = appDigest ‖ flags ‖ counter(BE) ‖ dataDigest (x/crypto layout)
|
||||
var original []byte
|
||||
original = append(original, appDigest[:]...)
|
||||
original = append(original, flagUserPresence)
|
||||
original = binary.BigEndian.AppendUint32(original, counter)
|
||||
original = append(original, dataDigest[:]...)
|
||||
edSig := ed25519.Sign(edPriv, original)
|
||||
// sig.Rest = skFields{Flags, Counter} = flags ‖ counter(BE)
|
||||
rest := append([]byte{flagUserPresence}, binary.BigEndian.AppendUint32(nil, counter)...)
|
||||
return ssh.Signature{Format: "sk-ssh-ed25519@openssh.com", Blob: edSig, Rest: rest}
|
||||
}
|
||||
return skPub, sign
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package authz
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MemoryNonceStore is a non-durable NonceStore for tests. Replay protection does
|
||||
// NOT survive process restart — never use it on a real host.
|
||||
type MemoryNonceStore struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]time.Time
|
||||
}
|
||||
|
||||
// NewMemoryNonceStore builds an empty in-memory store.
|
||||
func NewMemoryNonceStore() *MemoryNonceStore {
|
||||
return &MemoryNonceStore{seen: make(map[string]time.Time)}
|
||||
}
|
||||
|
||||
// SeenOrRecord reports whether nonce was already recorded, recording it if not.
|
||||
func (m *MemoryNonceStore) SeenOrRecord(nonce string, exp time.Time) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if _, ok := m.seen[nonce]; ok {
|
||||
return true
|
||||
}
|
||||
m.seen[nonce] = exp
|
||||
return false
|
||||
}
|
||||
|
||||
// FileNonceStore is the durable, crash-safe NonceStore for the host. Mechanism:
|
||||
// an fsync'd append-only JSONL log with an in-memory index, periodic compaction,
|
||||
// and expiry-only pruning.
|
||||
//
|
||||
// Durability guarantee: a nonce is on disk AND fsync'd before SeenOrRecord returns
|
||||
// false, so the caller acting on a verified op always does so AFTER the durable
|
||||
// record. A crash between verify and execute therefore drops the op (fail-safe
|
||||
// direction) and never enables a replay. Replay protection survives restarts: the
|
||||
// log is replayed into the index on Open.
|
||||
//
|
||||
// Pruning: a nonce is dropped only after its exp (compaction), never before —
|
||||
// pruning before expiry would reopen the replay window. (An expired nonce can't be
|
||||
// replayed anyway: the time-window check rejects an expired op before the nonce
|
||||
// check, so pruning is housekeeping, not an authz hole.)
|
||||
//
|
||||
// Concurrency: a single mutex guards the file handle and index (single-process; the
|
||||
// agent is concurrent — 03 §10).
|
||||
type FileNonceStore struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
f *os.File
|
||||
idx map[string]time.Time
|
||||
sinceCompact int
|
||||
now func() time.Time
|
||||
|
||||
// CompactEvery is the append count that triggers a compaction (default 1000).
|
||||
CompactEvery int
|
||||
}
|
||||
|
||||
type nonceRecord struct {
|
||||
Nonce string `json:"n"`
|
||||
Exp time.Time `json:"e"`
|
||||
}
|
||||
|
||||
// OpenFileNonceStore opens (or creates) the durable store at path, replaying any
|
||||
// existing log into the index.
|
||||
func OpenFileNonceStore(path string) (*FileNonceStore, error) {
|
||||
s := &FileNonceStore{
|
||||
path: path,
|
||||
idx: make(map[string]time.Time),
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
CompactEvery: 1000,
|
||||
}
|
||||
if err := s.load(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.f = f
|
||||
syncDir(filepath.Dir(path)) // make a freshly-created file's dir entry durable
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *FileNonceStore) load() error {
|
||||
b, err := os.ReadFile(s.path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, line := range bytes.Split(b, []byte("\n")) {
|
||||
line = bytes.TrimSpace(line)
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
var r nonceRecord
|
||||
if json.Unmarshal(line, &r) != nil {
|
||||
continue // skip a torn trailing line from a crash mid-append
|
||||
}
|
||||
s.idx[r.Nonce] = r.Exp
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SeenOrRecord durably records an unseen nonce before returning false. On any I/O
|
||||
// failure it returns true (fail-safe: the op is NOT executed rather than risk an
|
||||
// unrecorded nonce enabling a later replay).
|
||||
func (s *FileNonceStore) SeenOrRecord(nonce string, exp time.Time) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if _, ok := s.idx[nonce]; ok {
|
||||
return true
|
||||
}
|
||||
rec, _ := json.Marshal(nonceRecord{Nonce: nonce, Exp: exp})
|
||||
rec = append(rec, '\n')
|
||||
if _, err := s.f.Write(rec); err != nil {
|
||||
return true
|
||||
}
|
||||
if err := s.f.Sync(); err != nil {
|
||||
return true
|
||||
}
|
||||
s.idx[nonce] = exp
|
||||
s.sinceCompact++
|
||||
s.maybeCompact()
|
||||
return false
|
||||
}
|
||||
|
||||
// Close releases the file handle.
|
||||
func (s *FileNonceStore) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.f != nil {
|
||||
return s.f.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// maybeCompact rewrites the log keeping only non-expired entries once enough
|
||||
// appends have accrued. Caller holds the mutex. Compaction is housekeeping: the
|
||||
// recorded nonce is already durable, so a compaction failure never fails the op.
|
||||
func (s *FileNonceStore) maybeCompact() {
|
||||
if s.CompactEvery <= 0 || s.sinceCompact < s.CompactEvery {
|
||||
return
|
||||
}
|
||||
s.sinceCompact = 0
|
||||
now := s.now()
|
||||
|
||||
live := make(map[string]time.Time, len(s.idx))
|
||||
var buf bytes.Buffer
|
||||
for n, e := range s.idx {
|
||||
if e.Before(now) {
|
||||
continue // prune AFTER expiry only — safe
|
||||
}
|
||||
live[n] = e
|
||||
rec, _ := json.Marshal(nonceRecord{Nonce: n, Exp: e})
|
||||
buf.Write(rec)
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
|
||||
tmp := s.path + ".tmp"
|
||||
if err := os.WriteFile(tmp, buf.Bytes(), 0o600); err != nil {
|
||||
return // keep using the existing handle; nonce already durable
|
||||
}
|
||||
if tf, err := os.OpenFile(tmp, os.O_WRONLY, 0o600); err == nil {
|
||||
_ = tf.Sync()
|
||||
_ = tf.Close()
|
||||
}
|
||||
if s.f != nil {
|
||||
_ = s.f.Close()
|
||||
}
|
||||
if err := os.Rename(tmp, s.path); err != nil {
|
||||
s.f, _ = os.OpenFile(s.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
||||
return
|
||||
}
|
||||
syncDir(filepath.Dir(s.path))
|
||||
s.f, _ = os.OpenFile(s.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
||||
s.idx = live
|
||||
}
|
||||
|
||||
// syncDir best-effort fsyncs a directory so a create/rename is durable.
|
||||
func syncDir(dir string) {
|
||||
if d, err := os.Open(dir); err == nil {
|
||||
_ = d.Sync()
|
||||
_ = d.Close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package authz
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMemoryNonceStore(t *testing.T) {
|
||||
m := NewMemoryNonceStore()
|
||||
exp := time.Now().Add(time.Hour)
|
||||
if m.SeenOrRecord("a", exp) {
|
||||
t.Fatal("first record should be unseen")
|
||||
}
|
||||
if !m.SeenOrRecord("a", exp) {
|
||||
t.Fatal("second record should be seen")
|
||||
}
|
||||
if m.SeenOrRecord("b", exp) {
|
||||
t.Fatal("distinct nonce should be unseen")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileNonceStore_RecordAndReload(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "nonces.log")
|
||||
exp := refNow.Add(time.Hour)
|
||||
|
||||
s1, err := OpenFileNonceStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s1.SeenOrRecord("dead", exp) {
|
||||
t.Fatal("first record should be unseen")
|
||||
}
|
||||
if err := s1.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Reopen: the recorded nonce must still be seen (durable across restart).
|
||||
s2, err := OpenFileNonceStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if !s2.SeenOrRecord("dead", exp) {
|
||||
t.Fatal("nonce not durable across reopen")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileNonceStore_CompactionPrunesExpiredOnly(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "nonces.log")
|
||||
s, err := OpenFileNonceStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.now = func() time.Time { return refNow }
|
||||
s.CompactEvery = 2 // force a compaction after two appends
|
||||
|
||||
s.SeenOrRecord("expired", refNow.Add(-time.Hour)) // exp in the past
|
||||
s.SeenOrRecord("live", refNow.Add(time.Hour)) // triggers compaction
|
||||
if err := s.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Reopen: the live nonce survived, the expired one was pruned (housekeeping;
|
||||
// an expired op is rejected by the time check before the nonce check anyway).
|
||||
s2, err := OpenFileNonceStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if !s2.SeenOrRecord("live", refNow.Add(time.Hour)) {
|
||||
t.Error("live nonce should have survived compaction")
|
||||
}
|
||||
if s2.SeenOrRecord("expired", refNow.Add(-time.Hour)) {
|
||||
t.Error("expired nonce should have been pruned (was still present)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileNonceStore_SkipsTornLine(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "nonces.log")
|
||||
// a valid record line + a torn/garbage trailing line from a hypothetical crash
|
||||
content := `{"n":"good","e":"` + refNow.Add(time.Hour).Format(time.RFC3339Nano) + `"}` + "\n" + `{"n":"tor`
|
||||
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := OpenFileNonceStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open with torn line should not fail: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if !s.SeenOrRecord("good", refNow.Add(time.Hour)) {
|
||||
t.Error("valid record before the torn line should have loaded")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package authz
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewAllowedSigner(t *testing.T) {
|
||||
line, err := os.ReadFile("testdata/operator.pub")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s, err := NewAllowedSigner("felhom-op-1", RoleOperational, string(line))
|
||||
if err != nil {
|
||||
t.Fatalf("NewAllowedSigner: %v", err)
|
||||
}
|
||||
if s.KeyID != "felhom-op-1" || s.Role != RoleOperational || s.PublicKey == nil {
|
||||
t.Errorf("signer = %+v", s)
|
||||
}
|
||||
if s.PublicKey.Type() != "ssh-ed25519" {
|
||||
t.Errorf("key type = %q", s.PublicKey.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAllowedSigner_BadRole(t *testing.T) {
|
||||
line, _ := os.ReadFile("testdata/operator.pub")
|
||||
if _, err := NewAllowedSigner("k", "bogus", string(line)); err == nil {
|
||||
t.Fatal("invalid role should error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAllowedSigner_BadLine(t *testing.T) {
|
||||
if _, err := NewAllowedSigner("k", RoleOperational, "not a key"); err == nil {
|
||||
t.Fatal("malformed key line should error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package authz
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"hash"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// SSHSIG framing — ported verbatim-in-spirit from phase4-signing-findings.md §7.
|
||||
// The only manual work is SSHSIG *framing*; all crypto and key-type dispatch is
|
||||
// x/crypto/ssh's (pub.Verify dispatches on the key's own algorithm, which is what
|
||||
// makes the verifier key-type-agnostic — ed25519 / sk-ssh-ed25519 / rsa / ecdsa).
|
||||
// No hand-rolled crypto.
|
||||
|
||||
const sshsigMagic = "SSHSIG"
|
||||
|
||||
// sshsigBlob is the binary SSHSIG body (after the 6-byte magic). Field order is
|
||||
// the SSH wire order — do not reorder.
|
||||
type sshsigBlob struct {
|
||||
Version uint32
|
||||
PublicKey string
|
||||
Namespace string
|
||||
Reserved string
|
||||
HashAlgo string
|
||||
Signature string
|
||||
}
|
||||
|
||||
func hashByName(n string) (hash.Hash, error) {
|
||||
switch n {
|
||||
case "sha256":
|
||||
return sha256.New(), nil
|
||||
case "sha512":
|
||||
return sha512.New(), nil
|
||||
}
|
||||
return nil, fmt.Errorf("%w: unsupported SSHSIG hash %q", ErrMalformed, n)
|
||||
}
|
||||
|
||||
// parseArmoredSSHSIG decodes the `-----BEGIN SSH SIGNATURE-----` armor into the
|
||||
// SSHSIG body: pem.Decode → strip the literal 6-byte magic (not length-prefixed)
|
||||
// → ssh.Unmarshal.
|
||||
func parseArmoredSSHSIG(armored []byte) (*sshsigBlob, error) {
|
||||
block, _ := pem.Decode(armored)
|
||||
if block == nil || block.Type != "SSH SIGNATURE" {
|
||||
return nil, fmt.Errorf("%w: not an SSH SIGNATURE armor", ErrMalformed)
|
||||
}
|
||||
if len(block.Bytes) < len(sshsigMagic) || string(block.Bytes[:len(sshsigMagic)]) != sshsigMagic {
|
||||
return nil, fmt.Errorf("%w: missing SSHSIG magic", ErrMalformed)
|
||||
}
|
||||
var sb sshsigBlob
|
||||
if err := ssh.Unmarshal(block.Bytes[len(sshsigMagic):], &sb); err != nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrMalformed, err)
|
||||
}
|
||||
if sb.Version != 1 {
|
||||
return nil, fmt.Errorf("%w: bad SSHSIG version %d", ErrMalformed, sb.Version)
|
||||
}
|
||||
return &sb, nil
|
||||
}
|
||||
|
||||
// signedData recomputes the bytes the signature actually covers, per the SSHSIG
|
||||
// spec: "SSHSIG" || ssh.Marshal(namespace, reserved, hash_algorithm, H(message)),
|
||||
// where H is the named hash. The message is the RAW received blob bytes — the
|
||||
// verifier never canonicalizes (the canonical form is the signer's contract).
|
||||
func signedData(sb *sshsigBlob, msg []byte) ([]byte, error) {
|
||||
h, err := hashByName(sb.HashAlgo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.Write(msg)
|
||||
md := h.Sum(nil)
|
||||
body := ssh.Marshal(struct {
|
||||
Namespace string
|
||||
Reserved string
|
||||
HashAlgo string
|
||||
Hash []byte
|
||||
}{sb.Namespace, sb.Reserved, sb.HashAlgo, md})
|
||||
return append([]byte(sshsigMagic), body...), nil
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
{"expires_at":"2026-06-09T00:00:00Z","issued_at":"2026-06-08T00:00:00Z","key_id":"felhom-op-1","nonce":"a1b2c3d4e5f60718293a4b5c6d7e8f90","op":"guest_destroy","params":{"purge":true},"target":{"guest_id":"9001","host_id":"demo-felhom"}}
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
-----BEGIN SSH SIGNATURE-----
|
||||
U1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgNXOOuMvD3Fh9MJYspBRWLXyQAd
|
||||
WVeBICspeB9eL1xfIAAAAMZmVsaG9tLW9wLXYxAAAAAAAAAAZzaGE1MTIAAABTAAAAC3Nz
|
||||
aC1lZDI1NTE5AAAAQG+bj+GNodNw7cfGYg3HWTDyJiu3g/5Aez1xlZQ540JGUIG9FV7vv8
|
||||
wrgN0r+rNh+ytEAM6UTOyI7g3LOjuVJgY=
|
||||
-----END SSH SIGNATURE-----
|
||||
Loaded 100 of 530 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user