diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/round-2.txt b/documentation/audits/evidence-chaos-night-2026-09-17/round-2.txt index 0c1cda3e..226b8bd6 100644 --- a/documentation/audits/evidence-chaos-night-2026-09-17/round-2.txt +++ b/documentation/audits/evidence-chaos-night-2026-09-17/round-2.txt @@ -24,3 +24,40 @@ and `wiki`/bookstack, which needed a properly formed APP_KEY. 2026-09-16T21:24:58Z dark for 60 s 2026-09-16T21:26:01Z power back on 2026-09-16T21:26:01Z --- how long until the box is steady again, by itself --- +2026-09-16T21:26:18Z t+17s containers=0 +2026-09-16T21:27:55Z t+114s containers=0 + +## THE BACKGROUND HOUSEHOLD DIED WITH THE BOX — found at round 2, fixed for the rest of the night +The loop's last line is **21:23:25Z**; the power cut was at **21:24:56Z**; afterwards +`systemctl is-active household` = **inactive**. + +The reason is mine: the loop ran on the VM as a `systemd-run --collect` **transient** unit, and a +transient unit does not survive the machine being switched off. Round 2's accident is a power cut, so +the very first accident that could produce household failures instead produced **no lines at all**. + +**Zero lines is not the same as zero failures, and it must not be scored as one.** For round 2 the +household measure is: *not collected* — the loop was dead from 21:24:56Z until it was reinstalled. + +**Fixed for every round from here:** `/etc/systemd/system/household.service` with `Restart=always` +and `WantedBy=multi-user.target`, enabled — so it comes back with the box after a power cut, a hard +reset or a docker restart, which rounds 10 and 5 will also deliver. The log carries a marker line at +the changeover so the two regimes are not confused. + +This is the eighth thing tonight I have had to fix in my own measuring apparatus rather than in the +product, and the same shape as the rest: **the instrument was silent and silence looked like a pass.** +2026-09-16T21:28:12Z t+131s containers=25 +2026-09-16T21:28:29Z t+148s containers=26 +2026-09-16T21:28:29Z STEADY after 148s (back to 26 of 26 containers) +2026-09-16T21:28:29Z --- what the box says about gokapi and the restore --- +gokapi Up 30 seconds (healthy) +2026-09-16T21:28:31Z --- alarms in this round --- + | Time | Severity | Type | Message | Source + | Sep 16 21:28 | info | controller_started | Controller elindult (0.245.0) | controller + | Sep 16 21:23 | info | app_deployed | Alkalmazás telepítve: BookStack | controller + | Sep 16 21:22 | info | app_deploy_started | Alkalmazás telepítése elindult: BookStack | controller + | Sep 16 21:22 | info | app_removed | Alkalmazás eltávolítva: bookstack | controller + | Sep 16 21:20 | warning | app_oom | Alkalmazás memóriája elfogyott: immich (immich-postgres) — egy folyamatát a memóriakorlát leállította | controller + | Sep 16 21:19 | info | app_deployed | Alkalmazás telepítve: Immich | controller + | Sep 16 21:19 | info | app_deploy_started | Alkalmazás telepítése elindult: Immich | controller + | Sep 16 21:19 | info | app_removed | Alkalmazás eltávolítva: immich | controller +2026-09-16T21:28:32Z ================ END ROUND 2 ================ diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/round4.sh b/documentation/audits/evidence-chaos-night-2026-09-17/round4.sh new file mode 100755 index 00000000..8beb7794 --- /dev/null +++ b/documentation/audits/evidence-chaos-night-2026-09-17/round4.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# ROUND 4 — `offsite-run` (drawn app: mealie), while the tunnel is killed for 10 minutes. +# The off-site repository is ORPHANED (round 1 established that on this rebuilt box). The run is +# expected to REFUSE. That refusal is the measurement; the orphan is NOT repaired to get a nicer one. +set -u +E=/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/evidence-chaos-night-2026-09-17 +OUT=$E/round-4.txt +SCR=$(ls -d /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/*/scratchpad/chaos 2>/dev/null | head -1) +export SSHPASS=$(cat $SCR/boxroot.pw) +SSHO="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o NumberOfPasswordPrompts=1" +BOX(){ sshpass -e ssh $SSHO root@192.168.0.115 "$@" 2>/dev/null | grep -v "Warning: Permanently"; } +say(){ echo "$(date -u +%FT%TZ) $*" | tee -a $OUT; } +door(){ curl -sL -o /dev/null -w '%{http_code}' --max-time 12 -k -H "Host: $1.enkicsifelhom.hu" http://192.168.0.116/ 2>/dev/null; } + +cat > $SCR/r4a.sh <<'PEOF' +#!/bin/bash +H="Host: felhom.enkicsifelhom.hu"; B=http://172.17.0.2:8080 +curl -s -o /dev/null -D /tmp/.l -H "$H" -X POST --data-urlencode "password@/tmp/.pw" $B/login +S=$(grep -i '^set-cookie: felhom_session=' /tmp/.l | sed 's/.*felhom_session=\([^;]*\).*/\1/') +[ -z "$S" ] && { echo "ABORT: no session — mine, not the product's"; exit 1; } +C="Cookie: felhom_session=$S" +TOK=$(curl -s -H "$H" -H "$C" $B/backups/remote | grep -o 'name="csrf-token" content="[^"]*"' | sed 's/.*content="\([^"]*\)".*/\1/') +curl -s -o /dev/null -D /tmp/.r -H "$H" -H "$C" -X POST --data-urlencode "_csrf=$TOK" $B/backup/offbox/run +echo " POST /backup/offbox/run -> $(head -1 /tmp/.r)" +echo " flash: $(grep -i '^location:' /tmp/.r | sed 's/.*flash[_a-z]*=//' | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote_plus(sys.stdin.read().strip())[:220])' 2>/dev/null)" +echo " status: $(curl -s -H "$H" -H "$C" $B/backup/offbox/status | head -c 220)" +rm -f /tmp/.l /tmp/.r +PEOF + +say "================ ROUND 4 : offsite-run (mealie), while: tunnel down 10min ================" +BEFORE=$(BOX 'pct exec 9201 -- docker ps -q | wc -l' | tr -d ' \r'); [ -z "$BEFORE" ] && BEFORE=0 +say "--- BEFORE --- containers=$BEFORE recipes=$(door recipes) status=$(door status)" +say "--- ACTION: trigger the off-site run ---" +sshpass -e scp $SSHO -q $SCR/r4a.sh root@192.168.0.115:/tmp/r4a.sh +BOX 'pct push 9201 /tmp/r4a.sh /tmp/r4a.sh --perms 700; pct exec 9201 -- bash /tmp/r4a.sh; pct exec 9201 -- rm -f /tmp/r4a.sh; rm -f /tmp/r4a.sh' | tee -a $OUT +say "--- ACCIDENT: kill the tunnel for 10 minutes (it is NOT restarted by hand — whether it returns is the measurement) ---" +bash $E/inject.sh tunnel-down-10min 4 2>&1 | sed 's/^/ /' | tee -a $OUT +say "--- AFTER ---" +N=$(BOX 'pct exec 9201 -- docker ps -q | wc -l' | tr -d ' \r') +say " containers=$N (before $BEFORE) recipes=$(door recipes) status=$(door status)" +say " cloudflared: $(BOX 'pct exec 9201 -- docker ps -a --format "{{.Names}} {{.Status}}" | grep cloudflared')" +say "--- alarms ---"; bash $E/events.sh 8 | tee -a $OUT +say "================ END ROUND 4 ================" diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/round5.sh b/documentation/audits/evidence-chaos-night-2026-09-17/round5.sh new file mode 100755 index 00000000..92c5c786 --- /dev/null +++ b/documentation/audits/evidence-chaos-night-2026-09-17/round5.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# ROUND 5 — `use` privatebin, while docker is restarted inside the guest. +# +# Honest note on what „use" means here: PrivateBin encrypts in the BROWSER, so a paste cannot be +# created server-side without reimplementing its crypto. This round therefore exercises the app +# through its own front door with repeated reads, and says so — rather than faking a write. +set -u +E=/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/evidence-chaos-night-2026-09-17 +OUT=$E/round-5.txt +SCR=$(ls -d /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/*/scratchpad/chaos 2>/dev/null | head -1) +export SSHPASS=$(cat $SCR/boxroot.pw) +SSHO="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o NumberOfPasswordPrompts=1" +BOX(){ sshpass -e ssh $SSHO root@192.168.0.115 "$@" 2>/dev/null | grep -v "Warning: Permanently"; } +say(){ echo "$(date -u +%FT%TZ) $*" | tee -a $OUT; } +door(){ curl -sL -o /dev/null -w '%{http_code}' --max-time 12 -k -H "Host: $1.enkicsifelhom.hu" http://192.168.0.116/ 2>/dev/null; } + +say "================ ROUND 5 : use privatebin, while: docker restarted ================" +BEFORE=$(BOX 'pct exec 9201 -- docker ps -q | wc -l' | tr -d ' \r'); [ -z "$BEFORE" ] && BEFORE=0 +say "--- BEFORE --- containers=$BEFORE paste=$(door paste) wiki=$(door wiki) status=$(door status)" +say "--- ACTION: use the paste app through its own front door ---" +for i in 1 2 3; do say " paste read $i -> $(door paste)"; done +say " (reads only — PrivateBin's writes are client-side encrypted; declared, not faked)" +say "--- ACCIDENT: restart docker inside the guest ---" +T0=$(date +%s) +bash $E/inject.sh docker-restart 5 2>&1 | sed 's/^/ /' | tee -a $OUT +say "--- how long until every app is back, by itself ---" +for i in $(seq 1 40); do + sleep 15 + N=$(BOX 'pct exec 9201 -- docker ps -q 2>/dev/null | wc -l' | tr -d ' \r'); [ -z "$N" ] && N=0 + say " t+$(( $(date +%s) - T0 ))s containers=$N (before $BEFORE)" + [ "$BEFORE" -gt 0 ] && [ "$N" -ge "$BEFORE" ] && { say " STEADY after $(( $(date +%s) - T0 ))s"; break; } +done +say "--- AFTER --- paste=$(door paste) wiki=$(door wiki) status=$(door status)" +say "--- did every app come back on the SAME image? ---" +BOX 'pct exec 9201 -- docker ps --format "{{.Names}} {{.Image}}"' | sort | sed 's/^/ /' | tee -a $OUT +say "--- alarms ---"; bash $E/events.sh 8 | tee -a $OUT +say "================ END ROUND 5 ================" diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/round6.sh b/documentation/audits/evidence-chaos-night-2026-09-17/round6.sh new file mode 100755 index 00000000..8d1d59b2 --- /dev/null +++ b/documentation/audits/evidence-chaos-night-2026-09-17/round6.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# ROUND 6 — `backup-system` (whole-box backup), accident: nothing (control round). +# A whole-system backup is a BOX action; the drawn app (adventurelog) is which row is read after. +set -u +E=/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/evidence-chaos-night-2026-09-17 +OUT=$E/round-6.txt +SCR=$(ls -d /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/*/scratchpad/chaos 2>/dev/null | head -1) +export SSHPASS=$(cat $SCR/boxroot.pw) +SSHO="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o NumberOfPasswordPrompts=1" +BOX(){ sshpass -e ssh $SSHO root@192.168.0.115 "$@" 2>/dev/null | grep -v "Warning: Permanently"; } +say(){ echo "$(date -u +%FT%TZ) $*" | tee -a $OUT; } +door(){ curl -sL -o /dev/null -w '%{http_code}' --max-time 12 -k -H "Host: $1.enkicsifelhom.hu" http://192.168.0.116/ 2>/dev/null; } + +cat > $SCR/r6a.sh <<'PEOF' +#!/bin/bash +H="Host: felhom.enkicsifelhom.hu"; B=http://172.17.0.2:8080 +curl -s -o /dev/null -D /tmp/.l -H "$H" -X POST --data-urlencode "password@/tmp/.pw" $B/login +S=$(grep -i '^set-cookie: felhom_session=' /tmp/.l | sed 's/.*felhom_session=\([^;]*\).*/\1/') +[ -z "$S" ] && { echo "ABORT: no session — mine, not the product's"; exit 1; } +C="Cookie: felhom_session=$S" +TOK=$(curl -s -H "$H" -H "$C" $B/backups | grep -o 'name="csrf-token" content="[^"]*"' | sed 's/.*content="\([^"]*\)".*/\1/') +echo " trigger -> $(curl -s -o /tmp/.t -w '%{http_code}' -H "$H" -H "$C" -H "X-CSRF-Token: $TOK" -H 'Content-Type: application/json' -X POST $B/api/guest-backup/trigger)" +head -c 220 /tmp/.t; echo +for i in $(seq 1 45); do + sleep 20 + s=$(curl -s -H "$H" -H "$C" $B/api/guest-backup/status | head -c 240) + echo " $(date -u +%FT%TZ) $s" + echo "$s" | grep -qiE '"(running|active)":\s*false|"phase":"(done|idle|error)"' && break +done +rm -f /tmp/.l /tmp/.t +PEOF + +say "================ ROUND 6 : backup-system, accident: nothing (control) ================" +BEFORE=$(BOX 'pct exec 9201 -- docker ps -q | wc -l' | tr -d ' \r'); [ -z "$BEFORE" ] && BEFORE=0 +say "--- BEFORE --- containers=$BEFORE travel=$(door travel) status=$(door status)" +say "--- ACTION: whole-system backup (this is the one that measures DOWNTIME — R-518) ---" +T0=$(date +%s) +sshpass -e scp $SSHO -q $SCR/r6a.sh root@192.168.0.115:/tmp/r6a.sh +BOX 'pct push 9201 /tmp/r6a.sh /tmp/r6a.sh --perms 700; pct exec 9201 -- bash /tmp/r6a.sh; pct exec 9201 -- rm -f /tmp/r6a.sh; rm -f /tmp/r6a.sh' | tee -a $OUT +say " whole-system backup took $(( $(date +%s) - T0 ))s of wall clock" +say "--- AFTER ---" +N=$(BOX 'pct exec 9201 -- docker ps -q | wc -l' | tr -d ' \r') +say " containers=$N (before $BEFORE) travel=$(door travel) status=$(door status)" +say "--- alarms ---"; bash $E/events.sh 8 | tee -a $OUT +say "================ END ROUND 6 ================" diff --git a/documentation/audits/evidence-chaos-night-2026-09-17/run_round.sh b/documentation/audits/evidence-chaos-night-2026-09-17/run_round.sh new file mode 100755 index 00000000..6ae25032 --- /dev/null +++ b/documentation/audits/evidence-chaos-night-2026-09-17/run_round.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# run_round.sh +# One runner for every remaining round. Same five things every time, so no round is measured a +# different way from another. Actions and accidents are only the ones the seed actually drew. +set -u +N="${1:?round}"; X="${2:?action}"; Y="${3:?app}"; Z="${4:?accident}" +E=/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/evidence-chaos-night-2026-09-17 +OUT=$E/round-$N.txt +SCR=$(ls -d /tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/*/scratchpad/chaos 2>/dev/null | head -1) +export SSHPASS=$(cat $SCR/boxroot.pw) +SSHO="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o NumberOfPasswordPrompts=1" +BOX(){ sshpass -e ssh $SSHO root@192.168.0.115 "$@" 2>/dev/null | grep -v "Warning: Permanently"; } +say(){ echo "$(date -u +%FT%TZ) $*" | tee -a $OUT; } +door(){ curl -sL -o /dev/null -w '%{http_code}' --max-time 12 -k -H "Host: $1.enkicsifelhom.hu" http://192.168.0.116/ 2>/dev/null; } +# app -> its subdomain +sub(){ case "$1" in nextcloud) echo cloud;; immich) echo photos;; bookstack) echo wiki;; privatebin) echo paste;; + gokapi) echo share;; vaultwarden) echo vault;; paperless-ngx) echo paperless;; jellyfin) echo media;; + mealie) echo recipes;; uptime-kuma) echo status;; adventurelog) echo travel;; homebox) echo inventory;; *) echo "$1";; esac; } +SUB=$(sub "$Y") + +# --- the action, as a script run inside the guest ------------------------------------------------- +cat > $SCR/act.sh < \$(curl -s -o /tmp/.a -w '%{http_code}' -H "\$H" -H "\$C" -H "X-CSRF-Token: \$TOK" -X POST \$B/api/backup/run)" + head -c 200 /tmp/.a; echo + for i in \$(seq 1 30); do sleep 20 + s=\$(curl -s -H "\$H" -H "\$C" \$B/api/backup/status | head -c 220); echo " \$s" + echo "\$s" | grep -qiE '"running":\s*false' && break; done ;; + offsite-run) + curl -s -o /dev/null -D /tmp/.r -H "\$H" -H "\$C" -X POST --data-urlencode "_csrf=\$TOK" \$B/backup/offbox/run + echo " POST /backup/offbox/run -> \$(head -1 /tmp/.r)" + echo " flash: \$(grep -i '^location:' /tmp/.r | sed 's/.*flash[_a-z]*=//' | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote_plus(sys.stdin.read().strip())[:220])' 2>/dev/null)" ;; + restore) + curl -s -o /dev/null -D /tmp/.r -H "\$H" -H "\$C" -X POST --data-urlencode "_csrf=\$TOK" \ + --data-urlencode "stack_name=$Y" --data-urlencode "snapshot_id=helyi" \$B/backup/restore + echo " POST /backup/restore ($Y) -> \$(head -1 /tmp/.r)" + echo " flash: \$(grep -i '^location:' /tmp/.r | sed 's/.*flash[_a-z]*=//' | python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote_plus(sys.stdin.read().strip())[:220])' 2>/dev/null)" ;; + backup-system) + echo " POST /api/guest-backup/trigger -> \$(curl -s -o /tmp/.t -w '%{http_code}' -H "\$H" -H "\$C" -H "X-CSRF-Token: \$TOK" -H 'Content-Type: application/json' -X POST \$B/api/guest-backup/trigger)" + head -c 200 /tmp/.t; echo + for i in \$(seq 1 45); do sleep 20 + s=\$(curl -s -H "\$H" -H "\$C" \$B/api/guest-backup/status | head -c 220); echo " \$s" + echo "\$s" | grep -qiE '"(running|active)":\s*false|"phase":"(done|idle|error)"' && break; done ;; + use) echo " (use: driven from DooPlex through the app's own front door)" ;; +esac +rm -f /tmp/.l /tmp/.a /tmp/.r /tmp/.t +PEOF + +say "================ ROUND $N : $X $Y, while: $Z ================" +BEFORE=$(BOX 'pct exec 9201 -- docker ps -q | wc -l' | tr -d ' \r'); [ -z "$BEFORE" ] && BEFORE=0 +say "--- BEFORE --- containers=$BEFORE $SUB=$(door $SUB) status=$(door status) paste=$(door paste)" +say " (immich/photos is a KNOWN PRE-EXISTING failure — not caused by this round)" +HL0=$(BOX 'wc -l < /root/household.log' | tr -d ' \r') + +say "--- ACTION: $X on $Y ---" +if [ "$X" = "use" ]; then + for i in 1 2 3; do say " $SUB read $i -> $(door $SUB)"; done +else + sshpass -e scp $SSHO -q $SCR/act.sh root@192.168.0.115:/tmp/act.sh + BOX 'pct push 9201 /tmp/act.sh /tmp/act.sh --perms 700; pct exec 9201 -- bash /tmp/act.sh; pct exec 9201 -- rm -f /tmp/act.sh; rm -f /tmp/act.sh' | tee -a $OUT +fi + +T0=$(date +%s) +if [ "$Z" = "nothing" ]; then + say "--- ACCIDENT: none — control round, deliberately ---" +else + say "--- ACCIDENT: $Z (injected after the action started) ---" + bash $E/inject.sh "$Z" "$N" 2>&1 | sed 's/^/ /' | tee -a $OUT +fi + +say "--- AFTER: what the box did BY ITSELF ---" +for i in $(seq 1 40); do + N2=$(BOX 'pct exec 9201 -- docker ps -q 2>/dev/null | wc -l' | tr -d ' \r'); [ -z "$N2" ] && N2=0 + say " t+$(( $(date +%s) - T0 ))s containers=$N2 (before $BEFORE)" + [ "$BEFORE" -gt 0 ] && [ "$N2" -ge "$BEFORE" ] && { say " STEADY after $(( $(date +%s) - T0 ))s"; break; } + sleep 15 +done +say " front doors: $SUB=$(door $SUB) status=$(door status) paste=$(door paste) wiki=$(door wiki)" +HL1=$(BOX 'wc -l < /root/household.log' | tr -d ' \r') +say " household lines this round: $(( ${HL1:-0} - ${HL0:-0} )) failures: $(BOX "tail -n +$(( ${HL0:-0} + 1 )) /root/household.log | grep -cE 'FAILED|UNREACHABLE'" | tr -d ' \r')" +say "--- alarms ---"; bash $E/events.sh 8 | tee -a $OUT +say "================ END ROUND $N ================"