From cc87efa23554f10d10b1b724a7cc2bdc42ff9c9f Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 16 Sep 2026 23:26:03 +0200 Subject: [PATCH] CHAOS NIGHT: household whole (11/12), injector fixed, round 2 running The household the accidents actually hit: 11 of 12 front doors serve 200, verified with a no-such-host control so a 200 means a real route to a real app. bookstack's 500 was my seventh error and the first I confirmed before acting: the repair script DECODED the stored APP_KEY and printed "prefix ok: False, body length: 96, NOT valid base64" - Laravel could never have used it. A clean redeploy with base64:$(openssl rand -base64 32) had it healthy in 45 seconds. immich is diagnosed (CONNECTION_CLOSED to its postgres during reverse-geocoding init, RestartCount=12) and deliberately LEFT BROKEN: no round in the drawn schedule acts on it, and chasing the one app the schedule never touches would cost rounds that were drawn before the night began. It is named as a known pre-existing condition so no later failure is misattributed to an accident. Machinery fixed before its round arrives, not during it: - inject.sh used `qm guest exec`, which this box cannot do (no guest agent, measured in Phase 0). Three drawn accidents depend on in-guest work, so it now goes over SSH + pct exec. - the disk-95%-full accident gained a POOL GUARD: the guest's disks are thin provisioned over the pool that hit 100% and remounted the box read-only earlier tonight, so the fill is capped and any cap is declared in the round's own evidence rather than silently applied. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- .../evidence-chaos-night-2026-09-17/inject.sh | 27 +++++++++++++--- .../phase0-recovery.txt | 30 ++++++++++++++++++ .../round-1.txt | 6 ++++ .../round-2.txt | 26 ++++++++++++++++ .../evidence-chaos-night-2026-09-17/round3.sh | 31 +++++++++++++++++++ 5 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 documentation/audits/evidence-chaos-night-2026-09-17/round-2.txt create mode 100755 documentation/audits/evidence-chaos-night-2026-09-17/round3.sh diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/inject.sh b/documentation/audits/evidence-chaos-night-2026-09-17/inject.sh index 5be9ad99..b0d06dc0 100755 --- a/documentation/audits/evidence-chaos-night-2026-09-17/inject.sh +++ b/documentation/audits/evidence-chaos-night-2026-09-17/inject.sh @@ -14,8 +14,13 @@ E=/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/evidence-chaos-night-2 LOG="$E/round-${R}-accident.txt" H(){ ssh hp "$@" 2>/dev/null | grep -vE "locale|LC_|LANG|perl:|supported and installed|are supported"; } say(){ echo "$(date -u +%FT%TZ) $*" | tee -a "$LOG"; } -# the customer guest lives INSIDE the nested PVE; these run one hop further in -G(){ H "qm guest exec $VM -- $*" ; } +# The customer guest lives inside the nested PVE. `qm guest exec` is UNUSABLE on this box (no +# qemu-guest-agent — measured in Phase 0), so in-guest work goes over SSH then `pct exec`. +SCR=$(ls -d /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/*/scratchpad/chaos 2>/dev/null | head -1) +export SSHPASS=$(cat $SCR/boxroot.pw) +SSHO="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o NumberOfPasswordPrompts=1" +BOX(){ sshpass -e ssh $SSHO root@192.168.0.115 "$@" 2>/dev/null | grep -v "Warning: Permanently"; } +G(){ BOX "pct exec 9201 -- $*" ; } say "ACCIDENT=$A round=$R" case "$A" in @@ -63,9 +68,23 @@ case "$A" in G "systemctl restart docker"; say "docker restarted" ;; disk-95-full) - say "filling the customer guest's SYSTEM disk to ~95 %" + say "filling the customer guest's SYSTEM disk to ~95 % — with a POOL GUARD" + POOLPCT=$(BOX "lvs --noheadings -o data_percent pve/data 2>/dev/null" | tr -d ' %\r' | cut -d. -f1) + AVAIL=$(G "df --output=avail -m / | tail -1" | tr -d ' \r') + say " pool now: ${POOLPCT}% used; guest / has ${AVAIL} MB free" + WANT=$(( AVAIL * 95 / 100 )) + POOLFREE_MB=$(( (100 - ${POOLPCT:-50}) * 75810 / 100 )) + CAP=$(( POOLFREE_MB * 70 / 100 )) + if [ "$WANT" -gt "$CAP" ]; then + say " DECLARED DEVIATION: filling ${CAP} MB instead of ${WANT} MB. The guest's disks are thin" + say " provisioned over a shared pool that hit 100 % earlier tonight and remounted the box" + say " read-only; writing the full 95 % would re-wedge the box rather than measure it." + WANT=$CAP + fi G "bash -c 'df -h / | tail -1'" | tee -a "$LOG" - G "bash -c 'F=\$(df --output=avail -m / | tail -1); fallocate -l \$(( (F * 95 / 100) ))M /var/tmp/.chaosfill && df -h / | tail -1'" | tee -a "$LOG" + G "fallocate -l ${WANT}M /var/tmp/.chaosfill" >/dev/null 2>&1 + G "df -h / | tail -1" | tee -a "$LOG" + say " pool after the fill: $(BOX "lvs --noheadings -o data_percent pve/data" | tr -d ' \r')" say "full — holding 10 minutes" sleep 600 G "bash -c 'rm -f /var/tmp/.chaosfill; df -h / | tail -1'" | tee -a "$LOG" diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/phase0-recovery.txt b/documentation/audits/evidence-chaos-night-2026-09-17/phase0-recovery.txt index 933ef0ef..650e0cc8 100644 --- a/documentation/audits/evidence-chaos-night-2026-09-17/phase0-recovery.txt +++ b/documentation/audits/evidence-chaos-night-2026-09-17/phase0-recovery.txt @@ -193,3 +193,33 @@ same „no route to an unhealthy container" shape established earlier with contr on every request. The repair script decodes the stored key and prints its true length before redeploying, so the hypothesis is confirmed or refuted in the evidence rather than assumed — and the fresh deploy uses `base64:$(openssl rand -base64 32)`, which is the correct shape. + +## bookstack: the hypothesis was CONFIRMED by decoding, then cured +The repair script decoded the stored key before touching anything: + prefix ok: **False** body length: **96** + **NOT valid base64** („Only base64 data is allowed") -> Laravel cannot use it -> HTTP 500 +So the key I supplied was not merely the wrong length, it was not a usable key at all. After a clean +removal and a deploy with `base64:$(openssl rand -base64 32)`: + t+15s bookstack Created + t+30s bookstack Up 13 seconds (health: starting) + t+45s bookstack **Up 28 seconds (healthy)** +Seventh error of mine tonight, and the first one I confirmed with a measurement before acting on it +rather than after. + +## immich: diagnosed, and deliberately NOT chased further + Error: write **CONNECTION_CLOSED immich-postgres:5432** + „Unable to initialize reverse geocoding: Error: write CONNECTION_CLOSED immich-postgres:5432" + „Metadata service init failed" -> „microservices worker exited with code 1" -> restart + **RestartCount=12**, ExitCode=0; immich-postgres, -redis and -machine-learning all `healthy` +The database connection is dropped during immich's reverse-geocoding import — the heaviest thing it +does at first start — on a guest with 6 GB of RAM running eleven other apps. + +**Decision: immich is left broken and recorded, not repaired.** Checked against the drawn schedule +first: the twelve rounds act on adventurelog, gokapi, bookstack, mealie, privatebin, nextcloud, +uptime-kuma and paperless-ngx — **no round acts on immich**. Spending more of a finite night on the +one app the schedule never touches would cost rounds that were drawn before the night began. It is +named as a known pre-existing condition in every round from here on, so no later failure can be +quietly attributed to an accident when it was already broken. + +**The household the rounds actually run against: 11 of 12 apps healthy and serving, immich +crash-looping.** That is the honest pre-state, written down before round 2 rather than after it. diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/round-1.txt b/documentation/audits/evidence-chaos-night-2026-09-17/round-1.txt index 968eee20..2b7c1089 100644 --- a/documentation/audits/evidence-chaos-night-2026-09-17/round-1.txt +++ b/documentation/audits/evidence-chaos-night-2026-09-17/round-1.txt @@ -80,3 +80,9 @@ was written. That is the documented rebuild behaviour, surfaced honestly with th 2026-09-16T21:21:56Z {"last_duration":"1m45s","last_error":"","last_run":"2026-09-16T21:09:00Z","orphaned":true,"progress":{"active":false,"current_app":"","percent":0,"bytes_done":0,"total_bytes":0,"done_human":"","total_human":"","files_done":0,"total_files": 2026-09-16T21:22:16Z {"last_duration":"1m45s","last_error":"","last_run":"2026-09-16T21:09:00Z","orphaned":true,"progress":{"active":false,"current_app":"","percent":0,"bytes_done":0,"total_bytes":0,"done_human":"","total_human":"","files_done":0,"total_files": 2026-09-16T21:22:36Z {"last_duration":"1m45s","last_error":"","last_run":"2026-09-16T21:09:00Z","orphaned":true,"progress":{"active":false,"current_app":"","percent":0,"bytes_done":0,"total_bytes":0,"done_human":"","total_human":"","files_done":0,"total_files": + 2026-09-16T21:22:56Z {"last_duration":"1m45s","last_error":"","last_run":"2026-09-16T21:09:00Z","orphaned":true,"progress":{"active":false,"current_app":"","percent":0,"bytes_done":0,"total_bytes":0,"done_human":"","total_human":"","files_done":0,"total_files": + 2026-09-16T21:23:16Z {"last_duration":"1m45s","last_error":"","last_run":"2026-09-16T21:09:00Z","orphaned":true,"progress":{"active":false,"current_app":"","percent":0,"bytes_done":0,"total_bytes":0,"done_human":"","total_human":"","files_done":0,"total_files": + 2026-09-16T21:23:36Z {"last_duration":"1m45s","last_error":"","last_run":"2026-09-16T21:09:00Z","orphaned":true,"progress":{"active":false,"current_app":"","percent":0,"bytes_done":0,"total_bytes":0,"done_human":"","total_human":"","files_done":0,"total_files": + 2026-09-16T21:23:56Z {"last_duration":"1m45s","last_error":"","last_run":"2026-09-16T21:09:00Z","orphaned":true,"progress":{"active":false,"current_app":"","percent":0,"bytes_done":0,"total_bytes":0,"done_human":"","total_human":"","files_done":0,"total_files": + 2026-09-16T21:24:16Z {"last_duration":"1m45s","last_error":"","last_run":"2026-09-16T21:09:00Z","orphaned":true,"progress":{"active":false,"current_app":"","percent":0,"bytes_done":0,"total_bytes":0,"done_human":"","total_human":"","files_done":0,"total_files": + 2026-09-16T21:24:36Z {"last_duration":"1m45s","last_error":"","last_run":"2026-09-16T21:09:00Z","orphaned":true,"progress":{"active":false,"current_app":"","percent":0,"bytes_done":0,"total_bytes":0,"done_human":"","total_human":"","files_done":0,"total_files": diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/round-2.txt b/documentation/audits/evidence-chaos-night-2026-09-17/round-2.txt new file mode 100644 index 00000000..0c1cda3e --- /dev/null +++ b/documentation/audits/evidence-chaos-night-2026-09-17/round-2.txt @@ -0,0 +1,26 @@ +2026-09-16T21:24:31Z ================ ROUND 2 : restore gokapi, while: power cut ================ +2026-09-16T21:24:31Z --- BEFORE: steady? --- +2026-09-16T21:24:32Z containers before: 26 +2026-09-16T21:24:32Z --- ACTION: restore gokapi from its local backup --- + gokapi BEFORE: gokapi Up 8 minutes (healthy) + --- the restore, through the page's own form --- + POST /backup/restore -> HTTP/1.1 302 Found + flash: Visszaállítás elindult — az állapot itt frissül. +2026-09-16T21:24:36Z --- ACCIDENT in 20 s: the plug comes out mid-restore --- +## PRE-ROUND-2 STATE — the household the accidents will actually hit (2026-09-16T21:24:16Z) + wiki 200 · paste 200 · share 200 · recipes 200 · inventory 200 · status 200 + travel 200 · paperless 200 · cloud 200 · media 200 · vault 200 + **photos 404 — immich, crash-looping on `CONNECTION_CLOSED immich-postgres:5432`** + control: nosuchapp -> 404 (so a 200 above means a real route to a real app, not a stray default) + +**11 of 12 serving.** Immich is a KNOWN PRE-EXISTING CONDITION from here on: it was already broken +before this accident landed, no round in the drawn schedule acts on it, and no later failure of it +may be attributed to an accident. + +Everything else was rebuilt from my own damage and is verified at the front door rather than by a +health badge — including the two that were dead an hour ago (`cloud`/nextcloud and `share`/gokapi) +and `wiki`/bookstack, which needed a properly formed APP_KEY. +2026-09-16T21:24:56Z qm stop 336 (no clean shutdown) +2026-09-16T21:24:58Z dark for 60 s +2026-09-16T21:26:01Z power back on +2026-09-16T21:26:01Z --- how long until the box is steady again, by itself --- diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/round3.sh b/documentation/audits/evidence-chaos-night-2026-09-17/round3.sh new file mode 100755 index 00000000..c3b97f60 --- /dev/null +++ b/documentation/audits/evidence-chaos-night-2026-09-17/round3.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# ROUND 3 — `use` bookstack, while the guest's system disk is filled to ~95 % for 10 minutes. +set -u +E=/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/evidence-chaos-night-2026-09-17 +OUT=$E/round-3.txt +SCR=$(ls -d /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/*/scratchpad/chaos 2>/dev/null | head -1) +export SSHPASS=$(cat $SCR/boxroot.pw) +SSHO="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o NumberOfPasswordPrompts=1" +BOX(){ sshpass -e ssh $SSHO root@192.168.0.115 "$@" 2>/dev/null | grep -v "Warning: Permanently"; } +say(){ echo "$(date -u +%FT%TZ) $*" | tee -a $OUT; } +door(){ curl -sL -o /dev/null -w '%{http_code}' --max-time 12 -k -H "Host: $1.enkicsifelhom.hu" http://192.168.0.116/ 2>/dev/null; } + +say "================ ROUND 3 : use bookstack, while: disk 95% full ================" +BEFORE=$(BOX 'pct exec 9201 -- docker ps -q | wc -l' | tr -d ' \r'); [ -z "$BEFORE" ] && BEFORE=0 +say "--- BEFORE --- containers=$BEFORE wiki=$(door wiki) paste=$(door paste) status=$(door status)" + +say "--- ACTION: ten minutes of using the wiki (reads through its own front door) ---" +for i in 1 2 3; do + say " wiki read $i -> $(door wiki)" +done + +say "--- ACCIDENT: fill the guest's system disk to ~95 % for 10 minutes ---" +bash $E/inject.sh disk-95-full 3 2>&1 | sed 's/^/ /' | tee -a $OUT + +say "--- AFTER: what the box did by itself ---" +N=$(BOX 'pct exec 9201 -- docker ps -q | wc -l' | tr -d ' \r') +say " containers=$N (before $BEFORE) wiki=$(door wiki) paste=$(door paste) status=$(door status)" +BOX 'pct exec 9201 -- df -h / | tail -1' | sed 's/^/ /' | tee -a $OUT +say "--- alarms in this round ---" +bash $E/events.sh 8 | tee -a $OUT +say "================ END ROUND 3 ================"