From 301f4e9e92bb76a5f5d5a34f0e33ed6e33905d5e Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Tue, 20 Oct 2026 09:32:53 +0200 Subject: [PATCH] Night 2 (2026-10-20) evidence: P1 hub 0.162.0 deploy, installer 1.35.0 publish, agent 0.161.0 release+delivery+host_busy live; P2 R-893 red-proofs; P3 R-1019 classifier before/after --- .../night2-2026-10-20/P1/1a-hub-deploy.txt | 42 +++++++++ .../night2-2026-10-20/P1/1b-agent-release.txt | 65 +++++++++++++ .../P1/1b-host-busy-poll.txt | 48 ++++++++++ .../P1/1c-installer-publish.txt | 11 +++ .../night2-2026-10-20/P2/redproof-unit.txt | 10 ++ .../night2-2026-10-20/P2/redproof_r893.py | 50 ++++++++++ .../P3/classifier-before-after.txt | 94 +++++++++++++++++++ 7 files changed, 320 insertions(+) create mode 100644 documentation/audits/night2-2026-10-20/P1/1a-hub-deploy.txt create mode 100644 documentation/audits/night2-2026-10-20/P1/1b-agent-release.txt create mode 100644 documentation/audits/night2-2026-10-20/P1/1b-host-busy-poll.txt create mode 100644 documentation/audits/night2-2026-10-20/P1/1c-installer-publish.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/redproof-unit.txt create mode 100644 documentation/audits/night2-2026-10-20/P2/redproof_r893.py create mode 100644 documentation/audits/night2-2026-10-20/P3/classifier-before-after.txt diff --git a/documentation/audits/night2-2026-10-20/P1/1a-hub-deploy.txt b/documentation/audits/night2-2026-10-20/P1/1a-hub-deploy.txt new file mode 100644 index 00000000..83ee33dd --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P1/1a-hub-deploy.txt @@ -0,0 +1,42 @@ +# Part 1a — hub 0.162.0 deploy (operator yes 2026-10-20), read 2026-10-20T06:19:21Z +manifest commit e1d349f1; sync of Deployment/hub only (only it was OutOfSync) +argocd sync=Synced health=Healthy rev=b0ae2c574a8870e0e55455ceae77639b68a3d76f +pod hub-8c44cf5cb-srth9 start 2026-10-20T06:03:39Z image gitea.dooplex.hu/admin/felhom-hub:0.162.0 imageID gitea.dooplex.hu/admin/felhom-hub@sha256:69285e640a9c5e4ce59b1fbb5cd94623fa47a71798e9d86fa0767ee6c69a6ac8 + +## start-up and R-1001 log lines +2026/10/20 08:03:42 [INFO] felhom-hub 0.162.0 starting +2026/10/20 08:04:19 [INFO] Host staleness checker initialized: 3 ok, 1 already alarmed (seeded silently, R-1001), 0 stale, 0 down (left unseeded → first Check emits; host_stale after 45m0s, host_down after 1h30m0s) +2026/10/20 08:04:57 [INFO] Received report from tester-1 (5760 bytes) +2026/10/20 08:04:57 [INFO] Received report from demo-felhom (6065 bytes) +2026/10/20 08:04:58 [INFO] Received report from demo-hp (10036 bytes) +2026/10/20 08:05:24 [INFO] restore-test staleness: Tester-2 pbs tier still unproven, already reported before the restart — not re-sent (R-1001) +2026/10/20 08:05:24 [INFO] restore-test staleness: Tester-2 local tier still unproven, already reported before the restart — not re-sent (R-1001) +2026/10/20 08:07:33 [INFO] Received report from demo-felhom (6081 bytes) +2026/10/20 08:07:34 [INFO] Received report from tester-1 (5755 bytes) +2026/10/20 08:07:36 [INFO] Received report from demo-hp (10004 bytes) +2026/10/20 08:08:50 [INFO] host-report from demo-hp-bb76ea (1 guests, 4 storage targets, 2 backups, 2 restore-tests, 2 pbs-snapshots, 21418 bytes) +2026/10/20 08:09:21 [INFO] host-report from tester-1-d70be4 (1 guests, 3 storage targets, 2 backups, 2 restore-tests, 2 pbs-snapshots, 20577 bytes) +2026/10/20 08:10:36 [INFO] host-report from demo-felhom-8363b5 (1 guests, 4 storage targets, 2 backups, 2 restore-tests, 3 pbs-snapshots, 21708 bytes) + +## events with id > 6129 (the last id before the restart): +(none above = no event) + +## before: each earlier restart re-sent Tester-2's alarms +6055|host_down|2026-10-19 02:15:47 +6056|restore_test_stale|2026-10-19 02:15:51 +6057|restore_test_stale|2026-10-19 02:15:51 +6074|host_down|2026-10-19 09:30:04 +6075|restore_test_stale|2026-10-19 09:30:09 +6076|restore_test_stale|2026-10-19 09:30:09 +6087|host_down|2026-10-19 14:12:26 +6088|restore_test_stale|2026-10-19 14:12:58 +6089|restore_test_stale|2026-10-19 14:12:59 + +## floor +min_controller_version|0.335.0 +min_controller_version_declared_min_agent|0.335.0=0.158.1 + +## Day-0 agent dropdown (GET /configuration, CC token) +['', '0.160.0', '0.159.2', '0.159.1', '0.159.0', '0.158.1', '0.158.0', '0.157.0', '0.156.1', '0.156.0', '0.155.0', '0.154.0', '0.153.0', '0.152.0', '0.151.0', '0.150.0'] + +## mailbox (Gmail, second channel): last Tester-2 host_down / restore_test_stale mails = 2026-10-19T14:12:27Z / 14:12:59Z (the 0.161.0 restart); none after the 06:03 UTC restart; the daily 03:00Z expected_* mails present (mailbox live) diff --git a/documentation/audits/night2-2026-10-20/P1/1b-agent-release.txt b/documentation/audits/night2-2026-10-20/P1/1b-agent-release.txt new file mode 100644 index 00000000..f51b541d --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P1/1b-agent-release.txt @@ -0,0 +1,65 @@ +# Part 1b — agent 0.161.0 (operator yes 2026-10-20), read 2026-10-20T07:05:57Z + +## release (scripts/release-agent.sh from a clean clone at 5152123) +[release-agent] built ok: sha256 ff4e6dc52dd70234913445ecdeb88d31dea8e394d77c48704d26ff19b999d3b5 +[release-agent] config bundle built: sha256 485f36c957d83f772aa22cac24e5e73f44da5cf8b957b1fd513bec2bc1d12c47 +[release-agent] tagging v0.161.0 at 5152123 … +[publish-agent] upload OK (HTTP 201) +[publish-agent] round-trip GET verified (sha256 matches) +[release-agent] config bundle published (HTTP 201) + RELEASED — and installable, verified by download, not by this script's own say-so. + sha256 : ff4e6dc52dd70234913445ecdeb88d31dea8e394d77c48704d26ff19b999d3b5 + bundle : 485f36c957d83f772aa22cac24e5e73f44da5cf8b957b1fd513bec2bc1d12c47 (felhom-config-bundle.json — vouch it with the agent) + * [new tag] v0.161.0 -> v0.161.0 + +R-1083 NOT in this release: an agent key for 9202 would make its controller book whole-guest backups through +the agent, one tier of which writes to ep0 (demo-hp namespace) — a change on ep0 the brief forbids; 9202 is +also outside the felhom pool. Recorded in the R-1083 row. + +## delivery: one signed agent_update job per box (felhom-opsign, key felhom-op-1) +### demo-hp (signed 06:24:11Z) +Oct 20 08:38:50 demo-hp felhom-agent[2622137]: time=2026-10-20T08:38:50.375+02:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=e3ca40956e0d5fc9 op=agent_update key_id=felhom-op-1 nonce=9e6c1034fbc50692fd1cbade967b32e6 +Oct 20 08:38:51 demo-hp felhom-selfupdate-guarded[791860]: applied 0.161.0 (prev 0.160.0, sha ff4e6dc52dd70234913445ecdeb88d31dea8e394d77c48704d26ff19b999d3b5); detached restart scheduled +Oct 20 08:38:51 demo-hp felhom-agent[2622137]: time=2026-10-20T08:38:51.112+02:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.161.0 report="{\"agent_update\": {\"sha256\": \"ff4e6dc52dd70234913445ec +Oct 20 08:38:51 demo-hp felhom-agent[2622137]: time=2026-10-20T08:38:51.112+02:00 level=WARN msg="signedjobs: signed op COMPLETED" job=e3ca40956e0d5fc9 op=agent_update +Oct 20 08:38:53 demo-hp felhom-agent[791945]: time=2026-10-20T08:38:53.170+02:00 level=INFO msg="felhom-agent daemon starting" version=0.161.0 host_id=demo-hp-bb76ea hub_url=https://hub.felhom.eu interval_s=900 +Oct 20 08:38:54 demo-hp felhom-agent[791945]: time=2026-10-20T08:38:54.213+02:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0 +Oct 20 08:39:55 demo-hp felhom-agent[791945]: time=2026-10-20T08:39:55.311+02:00 level=WARN msg="selfupdate: update committed" version=0.161.0 wrapper="" +### tester-1 (signed 06:42:27Z) +Oct 20 08:54:21 felhom felhom-agent[2443463]: time=2026-10-20T08:54:21.074+02:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=9a6cc5c3d190a44c op=agent_update key_id=felhom-op-1 nonce=afe8d4d0a5389e1520b318ddf22ba9f2 +Oct 20 08:54:22 felhom felhom-agent[2443463]: time=2026-10-20T08:54:22.211+02:00 level=WARN msg="signedjobs: signed op COMPLETED" job=9a6cc5c3d190a44c op=agent_update +Oct 20 08:54:24 felhom felhom-agent[560821]: time=2026-10-20T08:54:24.350+02:00 level=INFO msg="felhom-agent daemon starting" version=0.161.0 host_id=tester-1-d70be4 hub_url=https://hub.felhom.eu interval_s=900 +Oct 20 08:54:25 felhom felhom-agent[560821]: time=2026-10-20T08:54:25.325+02:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0 +Oct 20 08:55:26 felhom felhom-agent[560821]: time=2026-10-20T08:55:26.096+02:00 level=WARN msg="selfupdate: update committed" version=0.161.0 wrapper="" +### demo-felhom (signed 06:42:27Z) +Oct 20 08:55:28 demo-felhom felhom-agent[498196]: time=2026-10-20T08:55:28.000+02:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=3c9c9c5b9ab01f25 op=agent_update key_id=felhom-op-1 nonce=84e9007d7d8fc1e4fef0d5d3a5250840 +Oct 20 08:55:28 demo-felhom felhom-agent[498196]: time=2026-10-20T08:55:28.566+02:00 level=WARN msg="signedjobs: signed op COMPLETED" job=3c9c9c5b9ab01f25 op=agent_update +Oct 20 08:55:30 demo-felhom felhom-agent[2429752]: time=2026-10-20T08:55:30.694+02:00 level=INFO msg="felhom-agent daemon starting" version=0.161.0 host_id=demo-felhom-8363b5 hub_url=https://hub.felhom.eu interval_s=900 +Oct 20 08:55:31 demo-felhom felhom-agent[2429752]: time=2026-10-20T08:55:31.379+02:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0 +Oct 20 08:56:32 demo-felhom felhom-agent[2429752]: time=2026-10-20T08:56:32.693+02:00 level=WARN msg="selfupdate: update committed" version=0.161.0 wrapper="" + +## hub view (latest host report per box) +demo-felhom-8363b5|0.161.0|2026-10-20 06:55:44 +demo-hp-bb76ea|0.161.0|2026-10-20 06:54:06 +tester-1-d70be4|0.161.0|2026-10-20 06:54:30 + +## host_busy, live on demo-hp (R-921): see 1b-host-busy-poll.txt — 3 free, 24 busy (backup:local), 21 free +"Mentés most" press 06:41:12Z through POST /api/guest-backup/trigger (the dashboard endpoint), 200 started +Oct 20 08:42:05 demo-hp felhom-agent[791945]: time=2026-10-20T08:42:05.179+02:00 level=INFO msg="local-api: backup reached snapshotted (app may resume)" vmid=9201 target=local job=backup-9201-1792478523540456451 +Oct 20 08:49:31 demo-hp felhom-agent[791945]: time=2026-10-20T08:49:31.529+02:00 level=INFO msg="backup: completed" vmid=9201 target=local archive=local:backup/vzdump-lxc-9201-2026_10_20-08_42_04.tar.zst size_bytes=7150133935 unco + +## R-1070: the tunnel probe the agent runs (pct exec -- docker inspect -f ... cloudflared) +9202 (no tunnel): error: no such object: cloudflared rc=1 +9201 (tunnel): running rc=0 +fixture row in internal/hub/cloudflared_test.go: 25: {"absent, docker 29", "", "error: no such object: cloudflared", errors.New("exit status 1"), TunnelAbsent, "no tunnel set up"}, + +## tests on the released commit (5152123) +--- PASS: TestClassifyTunnel (0.00s) +ok gitea.dooplex.hu/admin/felhom-agent/internal/hub 0.011s +--- PASS: TestHostHealthVerdict_R1070_NoTunnelSetUp (0.00s) +ok gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.013s +--- PASS: TestBackupStatus_R921_HostBusy (0.00s) +--- PASS: TestStaleLock_R1010_ForcesOnceOnlyWhenTheVolumeIsGone (0.00s) +ok gitea.dooplex.hu/admin/felhom-agent/internal/localapi 0.022s +--- PASS: TestDeleteSnapshotForce_R1010 (0.00s) +ok gitea.dooplex.hu/admin/felhom-agent/internal/proxmox 0.009s diff --git a/documentation/audits/night2-2026-10-20/P1/1b-host-busy-poll.txt b/documentation/audits/night2-2026-10-20/P1/1b-host-busy-poll.txt new file mode 100644 index 00000000..848bc19f --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P1/1b-host-busy-poll.txt @@ -0,0 +1,48 @@ +2026-10-20T06:41:12Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:41:30Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:41:48Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:42:06Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:42:24Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:42:42Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:43:02Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:43:20Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:43:38Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:44:11Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:44:28Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:44:46Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:45:04Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:45:26Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:45:44Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:46:05Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:46:23Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:46:42Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:47:00Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:47:18Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:47:36Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:47:54Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:48:12Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:48:30Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:48:48Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:49:05Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:49:23Z path= /backup/status http= 200 pin=ok keys= ['host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": true, "what": "backup:local"} tiers_in_flight= [] +2026-10-20T06:49:42Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:50:00Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:50:17Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:50:35Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:50:53Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:51:11Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:51:28Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:51:46Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:52:04Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:52:22Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:52:39Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:52:57Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:53:15Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:53:33Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:53:51Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:54:08Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:54:26Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:54:44Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:55:02Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:55:19Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] +2026-10-20T06:55:37Z path= /backup/status http= 200 pin=ok keys= ['backup', 'host_busy', 'job_id', 'phase', 'tiers', 'vmid'] host_busy= {"busy": false} tiers_in_flight= [] diff --git a/documentation/audits/night2-2026-10-20/P1/1c-installer-publish.txt b/documentation/audits/night2-2026-10-20/P1/1c-installer-publish.txt new file mode 100644 index 00000000..5c9ebcfa --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P1/1c-installer-publish.txt @@ -0,0 +1,11 @@ +# Part 1c — installer 1.35.0 publish (operator yes 2026-10-20), read 2026-10-20T06:20:23Z +tag refs: +1aa2e5585b7d13dc7b07fa9cc5e0711c34af291a refs/tags/installer-v1.35.0 +e1d349f1001b013fbcd2f26b25d71f9c649412a3 refs/tags/installer-v1.35.0^{} +pins: commit b0ae2c57 (manifests/webpage.yaml, both git-sync refs); ArgoCD sync of Deployment/felhom-webpage only +tag file sha256: f7a196fd5b3389e684aa0766968f29738e0b818d6ac8a79a7e7516fa96c99873 +served file sha256: f7a196fd5b3389e684aa0766968f29738e0b818d6ac8a79a7e7516fa96c99873 +served version: SCRIPT_VERSION="1.35.0" +die_live_box lines in the served file: 4 +before (06:16Z): served 1.34.0, sha256 df722ee4... (= the 1.34.0 tag file) +felhom-webpage-7f5c7968b5-g62t6 3/3 Running 0 2m49s diff --git a/documentation/audits/night2-2026-10-20/P2/redproof-unit.txt b/documentation/audits/night2-2026-10-20/P2/redproof-unit.txt new file mode 100644 index 00000000..2331b9dd --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/redproof-unit.txt @@ -0,0 +1,10 @@ +put-back skipped RED (good) r893_putback_test.go:135: the live volume was poured back as "" — want the exact pre-restore copy +volume-replay branch without undo RED (good) r893_putback_test.go:170: the replay removes a volume before it recreates it; the copy must be poured back even at a count of 0, got "" +fit check without the copy RED (good) r893_putback_test.go:187: the refusal must come before the stop — StopStack ran 2 time(s) +probe error ignored RED (good) r893_putback_test.go:213: StopStack ran 2 time(s) on an unmeasured copy +no drop on exit RED (good) r893_putback_test.go:232: the copy must be removed after a restore that worked, left [/tmp/TestR893A_ASuccessfulRestoreRemovesTheCopy3779926857/001/backups/restore-undo/immich/20261020T065144Z/immich_pgdata.tar] +copy not kept on a failed put-back RED (good) r893_putback_test.go:259: the copy must stay on disk for support, found [] +copy error ignored RED (good) r893_putback_test.go:276: nothing may move after a failed copy: replays 1, rollbacks 1, definition writes [] +slice-1 hold after the rollback RED (good) r893_mixed_restore_test.go:170: option A: the app must be started again after the put-back — calls [stop recreate startsvc:app-db recreate stop] +DB-start branch only restarts RED (good) r893_mixed_restore_test.go:202: the live definition must be written back BEFORE the start — calls [stop recreate startsvc:app-db start] +restored; failures-to-go-red: 0 diff --git a/documentation/audits/night2-2026-10-20/P2/redproof_r893.py b/documentation/audits/night2-2026-10-20/P2/redproof_r893.py new file mode 100644 index 00000000..db7fab1b --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P2/redproof_r893.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Red-proof the R-893 option A tests: apply one mutation, run its test, expect FAIL, restore the file.""" +import subprocess, sys + +ROOT = "/mnt/5_hdd/felhom.eu/git/felhom-controller/controller" +U = ROOT + "/internal/backup/offbox_restore_undo.go" +R = ROOT + "/internal/backup/offbox_reconstitute.go" + +CASES = [ + ("put-back skipped", U, " if volumesMoved {\n if err := m.putBack(stack, c); err != nil {", + " if false {\n if err := m.putBack(stack, c); err != nil {", "TestR893A_AFailedReplayPutsTheLiveVolumesBackExactly"), + ("volume-replay branch without undo", R, "if definitionMoved || nVols > 0 || undo != nil {", "if definitionMoved || nVols > 0 {", + "TestR893A_AFailedVolumeReplayStillPutsBackWhenItCountedNothing"), + ("fit check without the copy", R, "if free < need+volNeed {", "if free < need {", "TestR893A_TheCopyMustFitBeforeAnythingStops"), + ("probe error ignored", U, " if pErr != nil {\n return 0, nil, pErr\n }", " if pErr != nil {\n continue\n }", + "TestR893A_AnUnmeasurableVolumeRefusesBeforeAnythingStops"), + ("no drop on exit", R, " defer undo.drop(m, stack)\n", "", "TestR893A_ASuccessfulRestoreRemovesTheCopy"), + ("copy not kept on a failed put-back", U, " c.keep = true\n", "", "TestR893A_AFailedPutBackHoldsTheAppAndKeepsTheCopy"), + ("copy error ignored", R, " if cErr != nil {\n m.logger.Printf(\"[ERROR] [offbox] %s: the copy of the live volumes failed", + " if cErr != nil && false {\n m.logger.Printf(\"[ERROR] [offbox] %s: the copy of the live volumes failed", + "TestR893A_AFailedCopyRestoresNothingAndStartsTheApp"), + ("slice-1 hold after the rollback", R, + " return res, m.putBackAfterFailedRestore(stack, iErr, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, \"err.backup.db_restore_failed_held_mixed\")", + " m.holdAppAfterMixedRestore(stack, iErr, liveDef)\n return res, util.MsgError(\"err.backup.db_restore_failed_held_mixed\", stack)", + "TestR893_AVersionChangeThenAFailedReplayStartsTheAppAtItsLiveDefinition"), + ("DB-start branch only restarts", R, + " return res, m.putBackAfterFailedRestore(stack, err, liveDef, undo, res.VolumesReplayed > 0, res.FilesPlaced, restartStack, \"err.backup.restore_failed_held_mixed\")", + " _ = restartStack()\n return res, err", + "TestR893_AVersionChangeThenAFailedDBStartStartsTheAppAtItsLiveDefinition"), +] + +bad = 0 +for name, path, old, new, test in CASES: + src = open(path).read() + if src.count(old) != 1: + print("SETUP-ERROR", name, "match count", src.count(old)); bad += 1; continue + open(path, "w").write(src.replace(old, new)) + try: + p = subprocess.run(["go", "test", "./internal/backup/", "-run", "^" + test + "$", "-count=1"], cwd=ROOT, + capture_output=True, text=True) + out = p.stdout + p.stderr + line = next((l.strip() for l in out.splitlines() if "_test.go:" in l), out.strip().splitlines()[-1] if out.strip() else "") + verdict = "RED (good)" if p.returncode != 0 else "STILL GREEN (BAD)" + if p.returncode == 0: + bad += 1 + print("%-38s %-18s %s" % (name, verdict, line[:220])) + finally: + open(path, "w").write(src) +print("restored; failures-to-go-red:", bad) +sys.exit(1 if bad else 0) diff --git a/documentation/audits/night2-2026-10-20/P3/classifier-before-after.txt b/documentation/audits/night2-2026-10-20/P3/classifier-before-after.txt new file mode 100644 index 00000000..ec6fd85a --- /dev/null +++ b/documentation/audits/night2-2026-10-20/P3/classifier-before-after.txt @@ -0,0 +1,94 @@ +# R-1019 — the PRODUCTION classifier (controller main 93fb251) over the templates; hddPath=/mnt/drive +# BEFORE (catalog HEAD 5e37aa6): +=== RUN TestZZ_R1019_Probe +== qbittorrent hasClassification=true + bind root=userdata rel="media/torrents" ro=false class="optional" origin=explicit + secondary copies 1 path(s) [/mnt/drive/userdata/media/torrents] + offsite copies 0 path(s) [] +== transmission hasClassification=true + bind root=userdata rel="media/transmission" ro=false class="optional" origin=explicit + bind root=userdata rel="media/transmission/watch" ro=false class="optional" origin=explicit + secondary copies 1 path(s) [/mnt/drive/userdata/media/transmission] + offsite copies 0 path(s) [] +== nzbget hasClassification=true + bind root=userdata rel="media/nzbget" ro=false class="optional" origin=explicit + secondary copies 1 path(s) [/mnt/drive/userdata/media/nzbget] + offsite copies 0 path(s) [] +== sabnzbd hasClassification=true + bind root=userdata rel="media/usenet" ro=false class="optional" origin=explicit + secondary copies 1 path(s) [/mnt/drive/userdata/media/usenet] + offsite copies 0 path(s) [] +== metube hasClassification=true + bind root=userdata rel="media/metube" ro=false class="optional" origin=explicit + secondary copies 1 path(s) [/mnt/drive/userdata/media/metube] + offsite copies 0 path(s) [] +== sonarr hasClassification=true + bind root=userdata rel="media" ro=false class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== radarr hasClassification=true + bind root=userdata rel="media" ro=false class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== prowlarr hasClassification=true + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== jellyfin hasClassification=true + bind root=userdata rel="media" ro=true class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== plex hasClassification=true + bind root=userdata rel="media" ro=true class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== emby hasClassification=true + bind root=userdata rel="media" ro=true class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] + +# AFTER (the working tree with the five download folders excluded): +=== RUN TestZZ_R1019_Probe +== qbittorrent hasClassification=true + bind root=userdata rel="media/torrents" ro=false class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== transmission hasClassification=true + bind root=userdata rel="media/transmission" ro=false class="excluded" origin=explicit + bind root=userdata rel="media/transmission/watch" ro=false class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== nzbget hasClassification=true + bind root=userdata rel="media/nzbget" ro=false class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== sabnzbd hasClassification=true + bind root=userdata rel="media/usenet" ro=false class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== metube hasClassification=true + bind root=userdata rel="media/metube" ro=false class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== sonarr hasClassification=true + bind root=userdata rel="media" ro=false class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== radarr hasClassification=true + bind root=userdata rel="media" ro=false class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== prowlarr hasClassification=true + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== jellyfin hasClassification=true + bind root=userdata rel="media" ro=true class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== plex hasClassification=true + bind root=userdata rel="media" ro=true class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) [] +== emby hasClassification=true + bind root=userdata rel="media" ro=true class="excluded" origin=explicit + secondary copies 0 path(s) [] + offsite copies 0 path(s) []