diff --git a/documentation/audits/night-2026-09-24/00-drill-reset.txt b/documentation/audits/night-2026-09-24/00-drill-reset.txt new file mode 100644 index 00000000..849541b3 --- /dev/null +++ b/documentation/audits/night-2026-09-24/00-drill-reset.txt @@ -0,0 +1,2 @@ +drill=635c7e68f716 live=635c7e68f716 +has_actions: False private: True diff --git a/documentation/audits/night-2026-09-24/01-9202-storage-capacity.txt b/documentation/audits/night-2026-09-24/01-9202-storage-capacity.txt new file mode 100644 index 00000000..9dc7be3b --- /dev/null +++ b/documentation/audits/night-2026-09-24/01-9202-storage-capacity.txt @@ -0,0 +1,20 @@ +paths 404 {"_raw": "404 page not found\n"} +target 200 {"data": {"known": false}, "ok": true} +Filesystem Size Used Avail Use% Mounted on +/dev/nvme0n1 938G 72G 819G 8% /mnt/felhom-drives/scratch_hdd +/dev/loop1 69G 12G 54G 18% /mnt/sys_drive +scratch_hdd + total used free shared buff/cache available +Mem: 25898 799 23998 34 1134 25098 +Swap: 512 1 510 +7 +/dev/loop1 69G 12G 54G 18% /var/lib/docker +felhom-controller Up 2 hours (healthy) +privatebin Up 9 hours (healthy) +paperless-webserver Up 9 hours (healthy) +paperless-postgres Up 9 hours (healthy) +paperless-redis Up 9 hours (healthy) +gokapi Restarting (1) 1 second ago +filebrowser Up 12 hours (healthy) +traefik Up 12 hours + diff --git a/documentation/audits/night-2026-09-24/02-9202-repoint-drill.txt b/documentation/audits/night-2026-09-24/02-9202-repoint-drill.txt new file mode 100644 index 00000000..2c6fccaf --- /dev/null +++ b/documentation/audits/night-2026-09-24/02-9202-repoint-drill.txt @@ -0,0 +1,8 @@ + repo_url: https://gitea.dooplex.hu/admin/app-catalog-drill.git + sync_interval: 15m + token: + username: "admin" +hub: +update: + health_timeout: 90s + diff --git a/documentation/audits/night-2026-09-24/A1/00-spike.json b/documentation/audits/night-2026-09-24/A1/00-spike.json new file mode 100644 index 00000000..7680e518 --- /dev/null +++ b/documentation/audits/night-2026-09-24/A1/00-spike.json @@ -0,0 +1,49 @@ +{ + "deployed": true, + "files_seeded": 3, + "press": { + "accepted": true, + "http": "202", + "phases": [ + { + "t": 0.0, + "phase": "backing-up", + "label": "Biztonsági mentés készül a frissítés előtt…", + "updating": true, + "error": null, + "hold": null + }, + { + "t": 17.5, + "phase": "safety-dump", + "label": "Adatbázis pillanatkép…", + "updating": true, + "error": null, + "hold": null + }, + { + "t": 19.5, + "phase": "pulling", + "label": "Új verzió letöltése…", + "updating": true, + "error": null, + "hold": null + }, + { + "t": 21.6, + "phase": "failed", + "label": "A frissítés nem sikerült", + "updating": false, + "error": "Az új verzió letöltése nem sikerült, ezért a frissítés elmaradt. Az alkalmazás a korábbi verzióval fut tovább.", + "hold": null + } + ], + "duration_s": 21.6, + "final_phase": "failed", + "update_error": "Az új verzió letöltése nem sikerült, ezért a frissítés elmaradt. Az alkalmazás a korábbi verzióval fut tovább.", + "hold_reason": null, + "state": "running" + }, + "tier2_record": "null\n", + "layout": "== live userdata\ntotal 16\ndrwxr-sr-x 4 root 1000 4096 Sep 24 09:11 .\ndrwxrwsr-x 17 root 1000 4096 Sep 24 09:12 ..\ndrwxr-sr-x 3 root root 4096 Sep 24 09:10 appdata\ndrwxr-sr-x 3 root 1000 4096 Sep 24 09:11 backups\n/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-1-7cb158.txt 37 2026-09-24 09:11 www-data:www-data 644\n/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-0-e19728.txt 37 2026-09-24 09:11 www-data:www-data 644\n/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-2-8620d3.txt 37 2026-09-24 09:11 www-data:www-data 644\n== mirror copies of the seeds\n" +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-24/A1/00-spike.log b/documentation/audits/night-2026-09-24/A1/00-spike.log new file mode 100644 index 00000000..cc8319ff --- /dev/null +++ b/documentation/audits/night-2026-09-24/A1/00-spike.log @@ -0,0 +1,31 @@ +11:09:53 [1] made the drive paths this app requires: ['/mnt/felhom-drives/scratch_hdd/userdata/nextcloud'] +11:09:53 [1] required fields filled beyond DOMAIN/SUBDOMAIN: ['NEXTCLOUD_ADMIN_PASSWORD', 'HDD_PATH'] +11:09:54 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'} +11:11:24 [1] deployed, controller state=running, pinned={'nextcloud': 'nextcloud:34.0.4-apache', 'nextcloud-db': 'mariadb:12.3', 'nextcloud-redis': 'redis:7-alpine'} +11:11:33 nextcloud: occ user:add :: The account "drill76bf48" was created successfully Display name set to "drill76bf48" +11:11:37 nextcloud: seeded user drill76bf48 +11:11:44 nextcloud file PUT felhom-seed-0-e19728.txt ok=True +11:11:44 nextcloud file PUT felhom-seed-1-7cb158.txt ok=True +11:11:45 nextcloud file PUT felhom-seed-2-8620d3.txt ok=True +11:11:46 drill: nextcloud 34.0.99-notag (backing-up runs, pull fails) +11:11:51 [6] Update -> 202 {'ok': True, 'data': {'accepted': True, 'completed': False}, 'message': 'Frissítés elindult – az állapot a kártyán követhető'} +11:11:51 + 0.0s phase=backing-up label=Biztonsági mentés készül a frissítés előtt… err=None hold=None +11:12:08 + 17.5s phase=safety-dump label=Adatbázis pillanatkép… err=None hold=None +11:12:10 + 19.5s phase=pulling label=Új verzió letöltése… err=None hold=None +11:12:12 + 21.6s phase=failed label=A frissítés nem sikerült err=Az új verzió letöltése nem sikerült, ezért a frissítés elmaradt. Az alkalmazás a korábbi verzióval fut tovább. hold=None +11:12:13 drill: nextcloud back to 34.0.4 +11:12:21 Tier-2 record: +null + +11:12:24 layout: +== live userdata +total 16 +drwxr-sr-x 4 root 1000 4096 Sep 24 09:11 . +drwxrwsr-x 17 root 1000 4096 Sep 24 09:12 .. +drwxr-sr-x 3 root root 4096 Sep 24 09:10 appdata +drwxr-sr-x 3 root 1000 4096 Sep 24 09:11 backups +/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-1-7cb158.txt 37 2026-09-24 09:11 www-data:www-data 644 +/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-0-e19728.txt 37 2026-09-24 09:11 www-data:www-data 644 +/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-2-8620d3.txt 37 2026-09-24 09:11 www-data:www-data 644 +== mirror copies of the seeds + diff --git a/documentation/audits/night-2026-09-24/A1/00.stdout b/documentation/audits/night-2026-09-24/A1/00.stdout new file mode 100644 index 00000000..cc8319ff --- /dev/null +++ b/documentation/audits/night-2026-09-24/A1/00.stdout @@ -0,0 +1,31 @@ +11:09:53 [1] made the drive paths this app requires: ['/mnt/felhom-drives/scratch_hdd/userdata/nextcloud'] +11:09:53 [1] required fields filled beyond DOMAIN/SUBDOMAIN: ['NEXTCLOUD_ADMIN_PASSWORD', 'HDD_PATH'] +11:09:54 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'} +11:11:24 [1] deployed, controller state=running, pinned={'nextcloud': 'nextcloud:34.0.4-apache', 'nextcloud-db': 'mariadb:12.3', 'nextcloud-redis': 'redis:7-alpine'} +11:11:33 nextcloud: occ user:add :: The account "drill76bf48" was created successfully Display name set to "drill76bf48" +11:11:37 nextcloud: seeded user drill76bf48 +11:11:44 nextcloud file PUT felhom-seed-0-e19728.txt ok=True +11:11:44 nextcloud file PUT felhom-seed-1-7cb158.txt ok=True +11:11:45 nextcloud file PUT felhom-seed-2-8620d3.txt ok=True +11:11:46 drill: nextcloud 34.0.99-notag (backing-up runs, pull fails) +11:11:51 [6] Update -> 202 {'ok': True, 'data': {'accepted': True, 'completed': False}, 'message': 'Frissítés elindult – az állapot a kártyán követhető'} +11:11:51 + 0.0s phase=backing-up label=Biztonsági mentés készül a frissítés előtt… err=None hold=None +11:12:08 + 17.5s phase=safety-dump label=Adatbázis pillanatkép… err=None hold=None +11:12:10 + 19.5s phase=pulling label=Új verzió letöltése… err=None hold=None +11:12:12 + 21.6s phase=failed label=A frissítés nem sikerült err=Az új verzió letöltése nem sikerült, ezért a frissítés elmaradt. Az alkalmazás a korábbi verzióval fut tovább. hold=None +11:12:13 drill: nextcloud back to 34.0.4 +11:12:21 Tier-2 record: +null + +11:12:24 layout: +== live userdata +total 16 +drwxr-sr-x 4 root 1000 4096 Sep 24 09:11 . +drwxrwsr-x 17 root 1000 4096 Sep 24 09:12 .. +drwxr-sr-x 3 root root 4096 Sep 24 09:10 appdata +drwxr-sr-x 3 root 1000 4096 Sep 24 09:11 backups +/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-1-7cb158.txt 37 2026-09-24 09:11 www-data:www-data 644 +/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-0-e19728.txt 37 2026-09-24 09:11 www-data:www-data 644 +/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-2-8620d3.txt 37 2026-09-24 09:11 www-data:www-data 644 +== mirror copies of the seeds + diff --git a/documentation/audits/night-2026-09-24/A1/01-find-mirror.txt b/documentation/audits/night-2026-09-24/A1/01-find-mirror.txt new file mode 100644 index 00000000..f870755e --- /dev/null +++ b/documentation/audits/night-2026-09-24/A1/01-find-mirror.txt @@ -0,0 +1,7 @@ +/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/secondary/nextcloud +['password_hash', 'language', 'filebrowser_admin_password_enc', 'filebrowser_admin_state', 'filebrowser_admin_decided_at', 'claimed', 'claim_consumed_generation', 'offbox_enlarge_notice_seeded', 'app_update_events_seeded', 'db_validations', 'app_backup', 'storage_paths'] +app_backup {"enabled": false, "cross_drive": {"enabled": true, "method": "rsync", "destination_path": "/mnt/felhom-drives/scratch_hdd/userdata/romm", "schedule": "daily", "last_run": "2026-09-24T09:12:08Z", "last_status": "ok", "last_success": "2026-09-24T09:12:08Z", "success_tracked": true, "unit_package_date": "2026-09-24T09:12:06Z", "last_duration": "1s", "last_size_human": "1.0 GB"}} +2026/09/24 09:11:51 update_guard.go:360: [INFO] [backup] update pre-backup for nextcloud: starting (DB dump → volume dump → unit capture → Tier 2) +2026/09/24 09:12:08 tier2.go:425: [INFO] [backup] Tier 2 copied nextcloud → /mnt/felhom-drives/scratch_hdd/userdata/romm/backups/secondary/nextcloud (1.0 GB, 1 leg(s), 1s) +2026/09/24 09:12:08 update.go:748: [INFO] [stacks] update nextcloud: precondition met after the backup — Tier 2 (second drive) copy from 2026-09-24T09:12:08Z + diff --git a/documentation/audits/night-2026-09-24/A1/02-storage-paths.txt b/documentation/audits/night-2026-09-24/A1/02-storage-paths.txt new file mode 100644 index 00000000..01720575 --- /dev/null +++ b/documentation/audits/night-2026-09-24/A1/02-storage-paths.txt @@ -0,0 +1,38 @@ +[ + { + "path": "/mnt/felhom-drives/scratch_hdd", + "label": "SCRATCH HDD", + "is_default": true, + "schedulable": true, + "added_at": "" + }, + { + "path": "/mnt/felhom-drives/scratch_hdd/userdata/nextcloud", + "label": "T\u00e1rhely (nextcloud)", + "schedulable": true, + "added_at": "2026-09-21T20:07:01Z" + }, + { + "path": "/mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx", + "label": "T\u00e1rhely (paperless-ngx)", + "schedulable": true, + "added_at": "2026-09-22T19:49:16Z" + }, + { + "path": "/mnt/felhom-drives/scratch_hdd/userdata/romm", + "label": "T\u00e1rhely (romm)", + "schedulable": true, + "added_at": "2026-09-23T06:17:25Z" + }, + { + "path": "/mnt/felhom-drives/scratch_hdd/userdata/navidrome", + "label": "T\u00e1rhely (navidrome)", + "schedulable": true, + "added_at": "2026-09-23T20:39:41Z" + } +] +/mnt/felhom-drives/scratch_hdd dev=66304 +/mnt/felhom-drives/scratch_hdd/userdata/romm dev=66304 +/mnt/felhom-drives/scratch_hdd/userdata/nextcloud dev=66304 +/mnt/sys_drive dev=1793 + diff --git a/documentation/audits/night-2026-09-24/A1/spike-state.json b/documentation/audits/night-2026-09-24/A1/spike-state.json new file mode 100644 index 00000000..fab292a6 --- /dev/null +++ b/documentation/audits/night-2026-09-24/A1/spike-state.json @@ -0,0 +1,10 @@ +{ + "A": { + "uid": "drill76bf48" + }, + "files": [ + "felhom-seed-0-e19728.txt", + "felhom-seed-1-7cb158.txt", + "felhom-seed-2-8620d3.txt" + ] +} \ No newline at end of file diff --git a/documentation/audits/night-2026-09-24/A3/00-restartcounts-9201.txt b/documentation/audits/night-2026-09-24/A3/00-restartcounts-9201.txt new file mode 100644 index 00000000..987e652e --- /dev/null +++ b/documentation/audits/night-2026-09-24/A3/00-restartcounts-9201.txt @@ -0,0 +1,43 @@ +== felhom-pve 9201 +/felhom-controller restarts=0 started=2026-09-24T06:47:21.728320053Z created=2026-09-24T06:47:21.665152934Z status=running +/opengist restarts=0 started=2026-09-24T05:27:38.105496867Z created=2026-09-24T05:27:37.999067339Z status=running +/filebrowser restarts=0 started=2026-08-10T07:25:54.314481574Z created=2026-08-03T07:17:49.207519124Z status=running +/cloudflared restarts=0 started=2026-08-10T07:25:54.305365864Z created=2026-08-03T07:17:27.877738975Z status=running +/traefik restarts=0 started=2026-08-10T07:25:54.28252317Z created=2026-08-03T07:17:27.307858169Z status=running +== demo-hp 9201 +/felhom-controller restarts=0 started=2026-09-24T06:47:22.910952917Z created=2026-09-24T06:47:22.851214011Z status=running +/romm restarts=1 started=2026-09-24T06:24:01.331965421Z created=2026-09-24T06:23:50.510337353Z status=exited +/romm-db restarts=0 started=2026-09-24T06:23:50.582438225Z created=2026-09-24T06:23:50.442471253Z status=running +/romm-redis restarts=0 started=2026-09-24T06:23:50.58081926Z created=2026-09-24T06:23:50.44233201Z status=running +/privatebin restarts=0 started=2026-09-24T06:23:49.846285628Z created=2026-09-24T06:23:49.719138742Z status=running +/paperless-webserver restarts=0 started=2026-09-24T06:23:49.163739687Z created=2026-09-24T06:23:38.372940745Z status=running +/paperless-postgres restarts=1 started=2026-09-24T06:23:38.429088097Z created=2026-09-24T06:23:38.320522241Z status=exited +/paperless-redis restarts=0 started=2026-09-24T06:23:38.430701893Z created=2026-09-24T06:23:38.319755874Z status=running +/opengist restarts=0 started=2026-09-24T06:23:37.721694028Z created=2026-09-24T06:23:37.642376996Z status=running +/kimai restarts=0 started=2026-09-24T06:23:37.128258433Z created=2026-09-24T06:23:31.129377733Z status=running +/kimai-db restarts=0 started=2026-09-24T06:23:31.359145063Z created=2026-09-24T06:23:30.952715292Z status=running +/docmost restarts=0 started=2026-09-24T06:23:30.235452286Z created=2026-09-24T06:23:19.437626933Z status=running +/docmost-postgres restarts=0 started=2026-09-24T06:23:19.496388399Z created=2026-09-24T06:23:19.37876085Z status=running +/docmost-redis restarts=0 started=2026-09-24T06:23:19.494877226Z created=2026-09-24T06:23:19.37867101Z status=running +/calibre-web restarts=0 started=2026-09-24T06:23:18.854429812Z created=2026-09-24T06:23:18.787605278Z status=running +/bookstack restarts=0 started=2026-09-24T06:23:18.310109974Z created=2026-09-24T06:23:12.589022372Z status=running +/bookstack-db restarts=0 started=2026-09-24T06:23:12.632654803Z created=2026-09-24T06:23:12.549527569Z status=running +/bentopdf restarts=0 started=2026-09-24T06:23:12.059267557Z created=2026-09-24T06:23:11.99527037Z status=running +/adventurelog restarts=0 started=2026-09-24T06:23:11.508570026Z created=2026-09-24T06:23:05.776216336Z status=running +/adventurelog-postgres restarts=0 started=2026-09-24T06:23:05.825336746Z created=2026-09-24T06:23:05.734611021Z status=running +/adventurelog-frontend restarts=0 started=2026-09-24T06:23:05.823825062Z created=2026-09-24T06:23:05.734380877Z status=running +/filebrowser restarts=0 started=2026-09-01T17:48:22.221360326Z created=2026-09-01T17:48:21.186435353Z status=running +/cloudflared restarts=0 started=2026-09-01T17:47:45.039632728Z created=2026-08-21T16:00:39.902342861Z status=running +/traefik restarts=0 started=2026-09-01T17:47:45.010434902Z created=2026-08-21T16:00:38.84850712Z status=running +== demo-hp 9202 +/nextcloud restarts=0 started=2026-09-24T09:12:06.474520938Z status=running +/nextcloud-db restarts=0 started=2026-09-24T09:12:00.742780654Z status=running +/nextcloud-redis restarts=0 started=2026-09-24T09:12:00.739989825Z status=running +/felhom-controller restarts=0 started=2026-09-24T09:09:12.245253133Z status=running +/privatebin restarts=0 started=2026-09-24T00:30:23.085214239Z status=running +/paperless-webserver restarts=0 started=2026-09-24T00:30:21.642402588Z status=running +/paperless-postgres restarts=0 started=2026-09-24T00:30:10.920422124Z status=running +/paperless-redis restarts=0 started=2026-09-24T00:30:10.917647798Z status=running +/gokapi restarts=538 started=2026-09-24T09:20:53.73851955Z status=restarting +/filebrowser restarts=0 started=2026-09-23T21:06:20.703331594Z status=running +/traefik restarts=0 started=2026-09-23T21:06:20.702072245Z status=running diff --git a/documentation/audits/night-2026-09-24/A3/01-demo-hp-9201-exited.txt b/documentation/audits/night-2026-09-24/A3/01-demo-hp-9201-exited.txt new file mode 100644 index 00000000..96c07151 --- /dev/null +++ b/documentation/audits/night-2026-09-24/A3/01-demo-hp-9201-exited.txt @@ -0,0 +1,43 @@ +perl: warning: Setting locale failed. +perl: warning: Please check that your locale settings: + LANGUAGE = (unset), + LC_ALL = (unset), + LC_CTYPE = "UTF-8", + LC_NUMERIC = (unset), + LC_COLLATE = (unset), + LC_TIME = (unset), + LC_MESSAGES = (unset), + LC_MONETARY = (unset), + LC_ADDRESS = (unset), + LC_IDENTIFICATION = (unset), + LC_MEASUREMENT = (unset), + LC_PAPER = (unset), + LC_TELEPHONE = (unset), + LC_NAME = (unset), + LANG = "en_US.UTF-8" + are supported and installed on your system. +perl: warning: Falling back to a fallback locale ("en_US.UTF-8"). +adventurelog Up 3 hours (unhealthy) +adventurelog-frontend Up 3 hours (healthy) +adventurelog-postgres Up 3 hours (healthy) +bentopdf Up 3 hours (healthy) +bookstack Up 3 hours (unhealthy) +bookstack-db Up 3 hours (healthy) +calibre-web Up 3 hours (healthy) +cloudflared Up 3 weeks +docmost Up 3 hours (healthy) +docmost-postgres Up 3 hours (healthy) +docmost-redis Up 3 hours (healthy) +felhom-controller Up 3 hours (healthy) +filebrowser Up 3 weeks (healthy) +kimai Up 3 hours (healthy) +kimai-db Up 3 hours (healthy) +opengist Up 3 hours (healthy) +paperless-postgres Up 3 hours (healthy) +paperless-redis Up 3 hours (healthy) +paperless-webserver Up 3 hours (healthy) +privatebin Up 3 hours (healthy) +romm Up 3 hours (healthy) +romm-db Up 3 hours (healthy) +romm-redis Up 3 hours (healthy) +traefik Up 3 weeks diff --git a/documentation/audits/night-2026-09-24/A3/02-restartcounts-clean.txt b/documentation/audits/night-2026-09-24/A3/02-restartcounts-clean.txt new file mode 100644 index 00000000..6687b56b --- /dev/null +++ b/documentation/audits/night-2026-09-24/A3/02-restartcounts-clean.txt @@ -0,0 +1,28 @@ +== felhom-pve 9201 +/felhom-controller restarts=0 status=running +/opengist restarts=0 status=running +/filebrowser restarts=0 status=running +/cloudflared restarts=0 status=running +/traefik restarts=0 status=running +== demo-hp 9201 +/felhom-controller restarts=0 status=running +/romm restarts=1 status=exited +/romm-db restarts=0 status=running +/romm-redis restarts=0 status=running +/privatebin restarts=0 status=running +/opengist restarts=0 status=running +/kimai restarts=0 status=running +/kimai-db restarts=0 status=running +/docmost restarts=0 status=running +/docmost-postgres restarts=0 status=running +/docmost-redis restarts=0 status=running +/calibre-web restarts=0 status=running +/bookstack restarts=0 status=running +/bookstack-db restarts=0 status=running +/bentopdf restarts=0 status=running +/adventurelog restarts=0 status=running +/adventurelog-postgres restarts=0 status=running +/adventurelog-frontend restarts=0 status=running +/filebrowser restarts=0 status=running +/cloudflared restarts=0 status=running +/traefik restarts=0 status=running diff --git a/documentation/audits/night-2026-09-24/A3/03-n100-9201-ps.txt b/documentation/audits/night-2026-09-24/A3/03-n100-9201-ps.txt new file mode 100644 index 00000000..66c2b569 --- /dev/null +++ b/documentation/audits/night-2026-09-24/A3/03-n100-9201-ps.txt @@ -0,0 +1,5 @@ +cloudflared|Up 6 weeks +felhom-controller|Up 3 hours (healthy) +filebrowser|Up 6 weeks (healthy) +opengist|Up 4 hours (healthy) +traefik|Up 6 weeks diff --git a/documentation/audits/night-2026-09-24/A3/04-gokapi-rate.txt b/documentation/audits/night-2026-09-24/A3/04-gokapi-rate.txt new file mode 100644 index 00000000..0ce376a8 --- /dev/null +++ b/documentation/audits/night-2026-09-24/A3/04-gokapi-rate.txt @@ -0,0 +1,4 @@ +2026-09-24T09:22:00Z +539 restarting 2026-09-24T09:21:53.990300603Z 2026-09-24T09:21:54.185036712Z +2026-09-24T09:29:03Z +546 restarting 2026-09-24T09:28:55.748746549Z 2026-09-24T09:28:55.945764829Z diff --git a/documentation/audits/night-2026-09-24/PROGRESS.md b/documentation/audits/night-2026-09-24/PROGRESS.md new file mode 100644 index 00000000..2f072e91 --- /dev/null +++ b/documentation/audits/night-2026-09-24/PROGRESS.md @@ -0,0 +1,9 @@ +# PROGRESS — night 2026-09-24 (run 2026-09-24 from 11:07 CEST) + +Step log. Newest at the bottom. Times CEST unless marked Z. + +- 11:07 baselines verified: controller 7c3b3a969475 (v0.268.0), agent d9864a94bf62, felhom.eu 86c4b9a0d8bf (hub 0.122.0), catalog 635c7e68f716. Register 335 rows / 679,393 B. +- 11:08 `09` §3 decisions 26–28 recorded. +- 11:09 9202 repointed to the drill catalog (controller.yaml saved as .pre-night0924) +- 11:20 A1 (whole restore + R-668) and A2 (decision 27) built, tested, red-proofed +- 11:34 A3 built: detector (6 restarts/10 min — measured gokapi 1/min), stop+hold+event, Start lifts; red-proofs detector + skips diff --git a/documentation/audits/night-2026-09-24/redproofs/A1-file-rules.txt b/documentation/audits/night-2026-09-24/redproofs/A1-file-rules.txt new file mode 100644 index 00000000..5385d259 --- /dev/null +++ b/documentation/audits/night-2026-09-24/redproofs/A1-file-rules.txt @@ -0,0 +1,12 @@ +== rule2 +--- FAIL: TestWhole_TheFourFileRules (0.00s) + tier2_whole_test.go:86: live file photos/b.txt ("B edited by the household") is gone or changed — rule 1 +FAIL +== rule4 +--- FAIL: TestWhole_TheFourFileRules (0.00s) + tier2_whole_test.go:89: the older live copy of c.txt is gone — rule 4 must keep it beside as photos/c.txt.felhom-20260924T210000Z; files: [docs/deep/e.txt photos/a.txt photos/b.txt photos/c.txt photos/d.txt photos/only-live.txt] +FAIL +== rule3 +--- FAIL: TestWhole_TheFourFileRules (0.00s) + tier2_whole_test.go:97: a.txt / e.txt, missing live, were not brought back: "" "" +FAIL diff --git a/documentation/audits/night-2026-09-24/redproofs/A1-r668.txt b/documentation/audits/night-2026-09-24/redproofs/A1-r668.txt new file mode 100644 index 00000000..0f26abc3 --- /dev/null +++ b/documentation/audits/night-2026-09-24/redproofs/A1-r668.txt @@ -0,0 +1,5 @@ +--- FAIL: TestR668_MissingStoragePathIsNotASecondDrive (0.01s) + r668_missing_path_test.go:45: chose the missing path /tmp/TestR668_MissingStoragePathIsNotASecondDrive1761835767/001/drive-gone as the second drive +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/backup 0.014s +FAIL diff --git a/documentation/audits/night-2026-09-24/redproofs/A2-r666.txt b/documentation/audits/night-2026-09-24/redproofs/A2-r666.txt new file mode 100644 index 00000000..10e1014f --- /dev/null +++ b/documentation/audits/night-2026-09-24/redproofs/A2-r666.txt @@ -0,0 +1,3 @@ +--- FAIL: TestR666_HeldWithNoWholeCopyRemovesOnlyKeepingData (0.00s) + r666_keep_data_test.go:45: error = "A(z) /mnt/felhom-drives/x tárhely jelenleg nem elérhető — az alkalmazás nem távolítható el, amíg a meghajtó vissza nem csatlakozik.", want the support sentence +FAIL diff --git a/documentation/audits/night-2026-09-24/redproofs/A3-d28-skips.txt b/documentation/audits/night-2026-09-24/redproofs/A3-d28-skips.txt new file mode 100644 index 00000000..ddc82707 --- /dev/null +++ b/documentation/audits/night-2026-09-24/redproofs/A3-d28-skips.txt @@ -0,0 +1,5 @@ +--- FAIL: TestD28_StopHoldTellInOrder_AndTheSkips (0.00s) + d28_unhealthy_stop_test.go:41: calls = [stop:gokapi hold:gokapi:crash_loop notify:gokapi stop:quiesced hold:quiesced:crash_loop notify:quiesced] — want gokapi stopped, held, told; the quiesced, busy and held apps untouched +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/cmd/controller 0.008s +FAIL diff --git a/documentation/audits/night-2026-09-24/redproofs/A3-hub.txt b/documentation/audits/night-2026-09-24/redproofs/A3-hub.txt new file mode 100644 index 00000000..fadf4ed6 --- /dev/null +++ b/documentation/audits/night-2026-09-24/redproofs/A3-hub.txt @@ -0,0 +1,5 @@ +--- FAIL: TestAppStoppedUnhealthyIsAHouseholdEvent (0.00s) + chaosnight_events_test.go:102: app_stopped_unhealthy must be in allowedEventTypes — the controller's push would 400 +FAIL +FAIL gitea.dooplex.hu/admin/felhom-hub/internal/api 0.019s +FAIL diff --git a/documentation/audits/night-2026-09-24/redproofs/A3-r667-detector.txt b/documentation/audits/night-2026-09-24/redproofs/A3-r667-detector.txt new file mode 100644 index 00000000..b5b92797 --- /dev/null +++ b/documentation/audits/night-2026-09-24/redproofs/A3-r667-detector.txt @@ -0,0 +1,8 @@ +--- FAIL: TestR667_TheMeasuredCrashLoopTrips (0.00s) + unhealthy_test.go:34: gokapi's loop did not trip: kind="" n=0 +--- FAIL: TestR667_ObserveSumsPerAppAndSkipsUpdating (0.00s) + unhealthy_test.go:97: verdicts = [], want exactly gokapi's crash loop +FAIL +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.011s +FAIL +ok gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.008s diff --git a/documentation/audits/night-2026-09-24/tools/fixtures.py b/documentation/audits/night-2026-09-24/tools/fixtures.py new file mode 100644 index 00000000..28cd756c --- /dev/null +++ b/documentation/audits/night-2026-09-24/tools/fixtures.py @@ -0,0 +1,1085 @@ +#!/usr/bin/env python3 +"""Box-side seed/verify fixtures for walk.py, guest 9202. + +THE ONE RULE (R-156), carried verbatim from `app-catalog-felhom.eu/scripts/upgrade_fixtures.py`: +*nothing is ever seeded into a volume by hand.* Every seed here goes in through the app's OWN +interface — its HTTP API through the household's real front door (traefik, `Host: .`), +or its own CLI running inside its own container. A raw SQL INSERT or a planted file is never used. + +If an app has no non-browser route, its fixture returns None and the edge is recorded +`inconclusive — no non-browser seed route`, WITH WHAT WAS TRIED. That is a result, not a gap. + +Each fixture: + seed(w, sub, say) -> an opaque token, or None + verify(w, sub, tok, say) -> True / False +verify() must ask the APP, never the filesystem: a migration is supposed to rewrite files. +Where a fixture can prove itself (a negative control that must read as absent) it does so on EVERY +call, so a readback that has broken into always saying "found" fails instead of passing everything. +""" +import base64, json, re, secrets, time + + +def _gx(w, container, *cmd, timeout=240): + """Run a command inside the app's OWN container on 9202 (its own CLI, not our SQL).""" + import shlex + line = " ".join(shlex.quote(c) for c in cmd) + return w.guest(f"docker exec {container} {line} 2>&1", timeout=timeout) + + +# ============================================================================================= +class PrivateBin: + """PrivateBin's own JSON API. A paste is a POST and reading it back is a GET — an + application-level round trip. File-backed, no database: this single seed IS the file half.""" + sub = "paste" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/", want=("200",)): + return None + marker = "upg-" + secrets.token_hex(8) + ct = base64.b64encode(marker.encode()).decode() + body = json.dumps({ + "v": 2, + "adata": [[base64.b64encode(secrets.token_bytes(16)).decode(), + base64.b64encode(secrets.token_bytes(8)).decode(), + 100000, 256, 128, "aes", "gcm", "none"], "plaintext", 0, 0], + "ct": ct, "meta": {"expire": "never"}}) + rc, code, out = w.app_curl(sub, "/", "-H", "X-Requested-With: JSONHttpRequest", + "-H", "Content-Type: application/json", + data=body, method="POST") + try: + j = json.loads(out) + except Exception: + say(f" privatebin: POST returned non-JSON (http {code}): {out[:200]}") + return None + if j.get("status") != 0 or not j.get("id"): + say(f" privatebin: POST refused: {out[:250]}") + return None + say(f" privatebin: seeded paste id={j['id']}") + return {"id": j["id"], "marker": ct} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200",), tries=36): + return False + # negative control, every call: a paste id that cannot exist must NOT read back + rc, code, out = w.app_curl(sub, "/?pasteid=" + secrets.token_hex(8), + "-H", "X-Requested-With: JSONHttpRequest") + if t["marker"] in out: + say(" privatebin: READBACK UNUSABLE — a paste id that cannot exist returned the marker") + return False + rc, code, out = w.app_curl(sub, "/?pasteid=" + t["id"], + "-H", "X-Requested-With: JSONHttpRequest") + got = code == "200" and t["marker"] in out + say(f" privatebin: readback http={code} marker_present={got}") + return got + + +# ============================================================================================= +class Docmost: + """Docmost's own REST API: create the first workspace+user, then prove the account survives by + asking the app to AUTHENTICATE it. Login is version-stable across the API churn.""" + sub = "docs" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/", want=("200", "302", "404")): + return None + email = f"drill-{secrets.token_hex(4)}@gate.invalid" + pw = "Drill-" + secrets.token_hex(10) + body = json.dumps({"workspaceName": "drill", "name": "drill", "email": email, "password": pw}) + rc, code, out = w.app_curl(sub, "/api/auth/setup", "-H", "Content-Type: application/json", + data=body, method="POST") + say(f" docmost: /api/auth/setup http={code} rc={rc}") + if code not in ("200", "201"): + say(f" docmost: setup refused: {out[:250]}") + return None + return {"email": email, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200", "302", "404"), tries=36): + return False + # negative control: a password that was never set must NOT authenticate + bad = json.dumps({"email": t["email"], "password": "definitely-" + secrets.token_hex(8)}) + rc, code, _ = w.app_curl(sub, "/api/auth/login", "-H", "Content-Type: application/json", + data=bad, method="POST") + if code in ("200", "201"): + say(" docmost: READBACK UNUSABLE — a wrong password authenticated") + return False + body = json.dumps({"email": t["email"], "password": t["pw"]}) + rc, code, out = w.app_curl(sub, "/api/auth/login", "-H", "Content-Type: application/json", + data=body, method="POST") + ok = code in ("200", "201") + say(f" docmost: login as the seeded user http={code} ok={ok}") + if not ok: + say(f" docmost: login body {out[:200]}") + return ok + + +# ============================================================================================= +class BookStack: + """BookStack mints no API token without a browser, so BOTH halves go through `php artisan` — + BookStack's OWN CLI, inside its own container, against its own User model. + + The exit code carries no information here (`bookstack:reset-mfa` exits 1 for a user it FOUND + and for one it did not), so the discriminator is the OUTPUT: the positive sentence required and + the not-found sentence required absent. The negative control runs on every verify. + + LIMITATION (R-460): this seeds the DATABASE half only. The FILE half needs the API token the + app cannot mint headlessly — so a bookstack edge is at best HALF-proven here. + """ + sub = "wiki" + + def _artisan(self, w, *args): + for path in ("/app/www/artisan", "/var/www/html/artisan"): + out = _gx(w, "bookstack", "php", path, *args) + if "Could not open input file" not in out: + return " ".join(out.split()) + return " ".join(out.split()) + + def _lookup(self, w, email): + out = self._artisan(w, "bookstack:reset-mfa", f"--email={email}") + found = f"Email: {email}" in out + missing = "could not be found" in out + if found == missing: + return None, out + return found, out + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/login", want=("200",), tries=72): + return None + email = f"drill-{secrets.token_hex(4)}@gate.invalid" + pw = "Drill-" + secrets.token_hex(10) + out = self._artisan(w, "bookstack:create-admin", f"--email={email}", + f"--name=drill-{secrets.token_hex(3)}", f"--password={pw}") + say(f" bookstack: artisan create-admin :: {out[:140]}") + if "successfully created" not in out: + return None + return {"email": email, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/login", want=("200",), tries=72): + say(" bookstack: the app never served /login") + return False + absent, _ = self._lookup(w, f"nobody-{secrets.token_hex(6)}@gate.invalid") + if absent is not False: + say(f" bookstack: READBACK UNUSABLE — an email that cannot exist did not read absent ({absent})") + return False + found, out = self._lookup(w, t["email"]) + say(f" bookstack: readback of the seeded account found={found} :: {out[:140]}") + return found is True + + +# ============================================================================================= +class Gitea: + """Gitea's own admin CLI creates the first user; its own REST API (basic auth) then creates a + repository and reads it back. Both are the app's own interfaces.""" + sub = "git" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/", want=("200", "302")): + return None + user = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + out = _gx(w, "gitea", "su", "git", "-c", + f"gitea admin user create --username {user} --password {pw} " + f"--email {user}@gate.invalid --admin --must-change-password=false") + say(f" gitea: admin user create :: {' '.join(out.split())[:140]}") + if "has been successfully created" not in out and "successfully created" not in out: + return None + repo = "drillrepo" + secrets.token_hex(3) + rc, code, body = w.app_curl(sub, "/api/v1/user/repos", "-u", f"{user}:{pw}", + "-H", "Content-Type: application/json", + data=json.dumps({"name": repo, "private": True}), method="POST") + say(f" gitea: create repo http={code}") + if code not in ("201", "200"): + say(f" gitea: repo refused {body[:200]}") + return None + return {"user": user, "pw": pw, "repo": repo} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=36): + return False + rc, code, _ = w.app_curl(sub, f"/api/v1/repos/{t['user']}/nope{secrets.token_hex(4)}", + "-u", f"{t['user']}:{t['pw']}") + if code == "200": + say(" gitea: READBACK UNUSABLE — a repo that cannot exist returned 200") + return False + rc, code, body = w.app_curl(sub, f"/api/v1/repos/{t['user']}/{t['repo']}", + "-u", f"{t['user']}:{t['pw']}") + ok = code == "200" and t["repo"] in body + say(f" gitea: readback of the seeded repo http={code} ok={ok}") + return ok + + +# ============================================================================================= +class Navidrome: + """Navidrome's own REST API: create the first admin through /auth/createAdmin, then prove the + account survives by logging in through the same door.""" + sub = "music" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/", want=("200", "302")): + return None + user = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + rc, code, out = w.app_curl(sub, "/auth/createAdmin", "-H", "Content-Type: application/json", + data=json.dumps({"username": user, "password": pw}), method="POST") + say(f" navidrome: createAdmin http={code}") + if code not in ("200", "201"): + say(f" navidrome: refused {out[:200]}") + return None + return {"user": user, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=36): + return False + bad = json.dumps({"username": t["user"], "password": "wrong-" + secrets.token_hex(6)}) + rc, code, _ = w.app_curl(sub, "/auth/login", "-H", "Content-Type: application/json", + data=bad, method="POST") + if code in ("200", "201"): + say(" navidrome: READBACK UNUSABLE — a wrong password authenticated") + return False + body = json.dumps({"username": t["user"], "password": t["pw"]}) + rc, code, out = w.app_curl(sub, "/auth/login", "-H", "Content-Type: application/json", + data=body, method="POST") + ok = code in ("200", "201") + say(f" navidrome: login as the seeded user http={code} ok={ok}") + return ok + + +# ============================================================================================= +class Vaultwarden: + """Vaultwarden's own account API: register an account, then prove it survives by asking the app + to issue a token for it (its own login endpoint, the household's own route).""" + sub = "vault" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/alive", want=("200",)): + return None + email = f"drill-{secrets.token_hex(4)}@gate.invalid" + # Vaultwarden stores an already-hashed master key; the value is opaque to the server. + key = base64.b64encode(secrets.token_bytes(32)).decode() + body = json.dumps({"email": email, "name": "drill", "masterPasswordHash": key, + "key": "0." + base64.b64encode(secrets.token_bytes(48)).decode(), + "kdf": 0, "kdfIterations": 600000}) + rc, code, out = w.app_curl(sub, "/api/accounts/register", + "-H", "Content-Type: application/json", + data=body, method="POST") + say(f" vaultwarden: register http={code}") + if code not in ("200", "204"): + say(f" vaultwarden: refused {out[:250]}") + return None + return {"email": email, "key": key} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/alive", want=("200",), tries=36): + return False + def login(pwhash): + return w.app_curl(sub, "/identity/connect/token", + "-H", "Content-Type: application/x-www-form-urlencoded", + data=("grant_type=password&scope=api%20offline_access" + f"&client_id=web&deviceType=9&deviceIdentifier=drill" + f"&deviceName=drill&username={t['email']}&password={pwhash}"), + method="POST") + rc, code, _ = login(base64.b64encode(secrets.token_bytes(32)).decode()) + if code == "200": + say(" vaultwarden: READBACK UNUSABLE — a wrong master key authenticated") + return False + rc, code, out = login(t["key"].replace("+", "%2B").replace("=", "%3D").replace("/", "%2F")) + ok = code == "200" and "access_token" in out + say(f" vaultwarden: token for the seeded account http={code} ok={ok}") + if not ok: + say(f" vaultwarden: body {out[:200]}") + return ok + + +# ============================================================================================= +class Django: + """A Django app's OWN management CLI, inside its own container, against its own User model. + + Same category as BookStack's `php artisan`: the app's own code and its own ORM, never a raw SQL + INSERT and never a planted file (R-156). `createsuperuser --noinput` is Django's own documented + non-interactive route, and the readback asks the SAME ORM whether the account exists. + + THE FIXTURE PROVES ITSELF ON EVERY CALL: each verify() also asks for a username that cannot + exist and requires the answer False. A readback that has broken into always saying True + therefore fails instead of passing everything. + + LIMITATION, recorded rather than papered over: this seeds the DATABASE half only. An app whose + data is also FILES (adventurelog's images) has a file half this fixture does not touch. + """ + + def __init__(self, container, sub, ready_path="/", ready=("200", "302", "301", "404"), + python="python", workdir=None): + # `python` and `workdir` are per-app because the image decides them: adventurelog's + # interpreter is on PATH, tandoor ships a VENV and the bare `python` cannot import Django + # at all ("Couldn't import Django. Are you sure it's installed…"). Measured, not guessed. + self.container = container + self.sub = sub + self.ready_path = ready_path + self.ready = ready + self.python = python + self.workdir = workdir + + def _wd(self): + return f"-w {self.workdir} " if self.workdir else "" + + def _manage(self, w, code): + # -c is passed to `manage.py shell`; the app's own shell, its own ORM. + return w.guest( + f"docker exec {self._wd()}{self.container} {self.python} manage.py shell " + f"-c {json.dumps(code)} 2>&1", timeout=300) + + def _exists(self, w, username): + # ONE LINE, semicolon-separated. A `\n` inside a double-quoted shell argument reaches + # python as a literal backslash-n and is a SyntaxError — which is exactly how the first + # adventurelog run read as `inconclusive`. The fixture refused to guess, which is right, + # but the instrument was the thing that was broken. + out = self._manage(w, ( + "from django.contrib.auth import get_user_model; " + f"print('DRILL_ANSWER=' + str(get_user_model().objects.filter(username={username!r}).exists()))" + )) + m = re.search(r"DRILL_ANSWER=(True|False)", out) + return (m.group(1) == "True") if m else None, " ".join(out.split())[-300:] + + def seed(self, w, sub, say): + if not w.wait_app(sub, self.ready_path, want=self.ready, tries=90): + return None + user = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + out = w.guest( + f"docker exec -e DJANGO_SUPERUSER_PASSWORD={pw} {self._wd()}{self.container} " + f"{self.python} manage.py createsuperuser --noinput " + f"--username {user} --email {user}@gate.invalid 2>&1", timeout=300) + say(f" {self.container}: createsuperuser :: {' '.join(out.split())[:160]}") + got, detail = self._exists(w, user) + if got is not True: + say(f" {self.container}: the account did not appear in the app's own ORM :: {detail[:200]}") + return None + say(f" {self.container}: seeded superuser {user}") + return {"user": user, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, self.ready_path, want=self.ready, tries=90): + say(f" {self.container}: the app never served {self.ready_path}") + return False + absent, detail = self._exists(w, "nobody" + secrets.token_hex(6)) + if absent is not False: + say(f" {self.container}: READBACK UNUSABLE — a username that cannot exist did not " + f"read as absent ({absent}) :: {detail[:200]}") + return False + found, detail = self._exists(w, t["user"]) + say(f" {self.container}: readback of the seeded account found={found}") + if found is not True: + say(f" {self.container}: :: {detail[:250]}") + return found is True + + +# ============================================================================================= +class Nextcloud: + """Nextcloud's OWN admin CLI, `occ`, inside its own container: its own code, its own user + backend. Not a SQL INSERT and not a planted file (R-156). + + `occ user:info` is the readback, and it PROVES ITSELF on every call: a uid that cannot exist + must answer "user not found". A readback that has broken into always succeeding therefore + fails instead of passing everything. + + This is the app chosen for the MariaDB engine-major edge (`09` §3 decision 5, R-469 lifted): + the app image does NOT move, only the `mariadb:` sidecar, so the edge carries exactly one + migration and a failure is readable. + """ + sub = "cloud" + + def _occ(self, w, *args, timeout=420): + import shlex + line = " ".join(shlex.quote(a) for a in args) + return w.guest(f"docker exec -u www-data nextcloud php occ {line} 2>&1", timeout=timeout) + + def _info(self, w, uid): + out = self._occ(w, "user:info", uid) + flat = " ".join(out.split()) + if "user not found" in flat.lower() or "could not be found" in flat.lower(): + return False, flat + if f"user_id: {uid}" in flat or f"- user_id: {uid}" in flat or f"user_id: {uid}" in out: + return True, flat + return None, flat + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/status.php", want=("200",), tries=120): + return None + # /status.php answers 200 while the image's own first-run install is still going — measured + # 2026-09-23 night on a loaded bench: `occ` then says "Nextcloud is not installed". Ask occ. + for i in range(60): + st = self._occ(w, "status", timeout=120) + if "installed: true" in st: + break + time.sleep(5) + else: + say(f" nextcloud: occ status never said installed: {' '.join(st.split())[:160]}") + return None + uid = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + out = w.guest( + f"docker exec -u www-data -e OC_PASS={pw} nextcloud php occ user:add " + f"--password-from-env --display-name={uid} {uid} 2>&1", timeout=420) + say(f" nextcloud: occ user:add :: {' '.join(out.split())[:160]}") + got, flat = self._info(w, uid) + if got is not True: + say(f" nextcloud: the account did not appear via occ user:info :: {flat[:220]}") + return None + say(f" nextcloud: seeded user {uid}") + return {"uid": uid, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/status.php", want=("200",), tries=120): + say(" nextcloud: the app never served /status.php") + return False + absent, flat = self._info(w, "nobody" + secrets.token_hex(6)) + if absent is not False: + say(f" nextcloud: READBACK UNUSABLE — a uid that cannot exist did not read absent " + f"({absent}) :: {flat[:200]}") + return False + found, flat = self._info(w, t["uid"]) + say(f" nextcloud: readback of the seeded user found={found}") + if found is not True: + say(f" nextcloud: :: {flat[:250]}") + return found is True + + +# ============================================================================================= +class Grafana: + """Grafana's own HTTP API as the admin the DEPLOY created. The password is the one the + controller showed the household — read from the app's own `app.yaml`, not invented — and the + data (a folder) goes in and comes back through the app's own REST API.""" + sub = "grafana" + + def _auth(self, w, name="grafana"): + # app.yaml stores this ENCRYPTED (`ENC:…`), so it cannot be read back off the box — which + # is correct, and is why the harness uses the value IT generated for the deploy. + pw = (w.GENERATED.get(name) or {}).get("GF_SECURITY_ADMIN_PASSWORD") or "admin" + return f"admin:{pw}" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/api/health", want=("200",), tries=72): + return None + au = self._auth(w) + title = "drill-" + secrets.token_hex(5) + rc, code, body = w.app_curl(sub, "/api/folders", "-u", au, + "-H", "Content-Type: application/json", + data=json.dumps({"title": title}), method="POST") + say(f" grafana: create folder http={code}") + if code not in ("200", "201"): + say(f" grafana: refused {body[:220]}") + return None + try: + uid = json.loads(body)["uid"] + except Exception: + say(f" grafana: no uid in {body[:200]}") + return None + return {"uid": uid, "title": title} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/api/health", want=("200",), tries=72): + return False + au = self._auth(w) + rc, code, _ = w.app_curl(sub, "/api/folders/nope" + secrets.token_hex(5), "-u", au) + if code == "200": + say(" grafana: READBACK UNUSABLE — a folder uid that cannot exist returned 200") + return False + rc, code, body = w.app_curl(sub, f"/api/folders/{t['uid']}", "-u", au) + ok = code == "200" and t["title"] in body + say(f" grafana: readback of the seeded folder http={code} ok={ok}") + return ok + + +# ============================================================================================= +class AudiobookShelf: + """audiobookshelf's own /init endpoint creates the first root account; its own /login proves + the account survived. Both are the app's own API.""" + sub = "audiobooks" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/status", want=("200",), tries=72): + return None + user = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + rc, code, body = w.app_curl(sub, "/init", "-H", "Content-Type: application/json", + data=json.dumps({"newRoot": {"username": user, "password": pw}}), + method="POST") + say(f" audiobookshelf: /init http={code}") + if code not in ("200", "204"): + say(f" audiobookshelf: refused {body[:220]}") + return None + return {"user": user, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/status", want=("200",), tries=72): + return False + bad = json.dumps({"username": t["user"], "password": "wrong-" + secrets.token_hex(6)}) + rc, code, _ = w.app_curl(sub, "/login", "-H", "Content-Type: application/json", + data=bad, method="POST") + if code == "200": + say(" audiobookshelf: READBACK UNUSABLE — a wrong password authenticated") + return False + rc, code, body = w.app_curl(sub, "/login", "-H", "Content-Type: application/json", + data=json.dumps({"username": t["user"], "password": t["pw"]}), + method="POST") + ok = code == "200" and t["user"] in body + say(f" audiobookshelf: login as the seeded root http={code} ok={ok}") + return ok + + +# ============================================================================================= +class ActualBudget: + """Actual's own bootstrap API sets the server password; its own login proves it survived.""" + sub = "budget" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=72): + return None + pw = "Drill-" + secrets.token_hex(10) + rc, code, body = w.app_curl(sub, "/account/bootstrap", + "-H", "Content-Type: application/json", + data=json.dumps({"password": pw}), method="POST") + say(f" actualbudget: /account/bootstrap http={code} :: {body[:140]}") + if code not in ("200", "201") or '"status":"ok"' not in body: + return None + return {"pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=72): + return False + def login(p): + return w.app_curl(sub, "/account/login", "-H", "Content-Type: application/json", + data=json.dumps({"loginMethod": "password", "password": p}), + method="POST") + rc, code, body = login("wrong-" + secrets.token_hex(6)) + if '"status":"ok"' in body: + say(" actualbudget: READBACK UNUSABLE — a wrong password authenticated") + return False + rc, code, body = login(t["pw"]) + ok = '"status":"ok"' in body + say(f" actualbudget: login with the seeded password http={code} ok={ok}") + if not ok: + say(f" actualbudget: body {body[:200]}") + return ok + + +# ============================================================================================= +class Mealie: + """Mealie ships a documented first-run admin. We log in as it through the app's own OAuth-style + token endpoint, create a recipe through the app's own API, and read the recipe back.""" + sub = "recipes" + + def _token(self, w, sub, pw="MyPassword"): + rc, code, body = w.app_curl( + sub, "/api/auth/token", "-H", "Content-Type: application/x-www-form-urlencoded", + data=f"username=changeme%40example.com&password={pw}", method="POST") + if code != "200": + return None, f"http={code} {body[:200]}" + try: + return json.loads(body)["access_token"], "" + except Exception: + return None, body[:200] + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/api/app/about", want=("200",), tries=90): + return None + tok, why = self._token(w, sub) + if not tok: + say(f" mealie: could not authenticate as the first-run admin :: {why}") + return None + name = "drill-" + secrets.token_hex(5) + rc, code, body = w.app_curl(sub, "/api/recipes", "-H", f"Authorization: Bearer {tok}", + "-H", "Content-Type: application/json", + data=json.dumps({"name": name}), method="POST") + say(f" mealie: create recipe http={code}") + if code not in ("200", "201"): + say(f" mealie: refused {body[:220]}") + return None + slug = body.strip().strip('"') + return {"slug": slug, "name": name} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/api/app/about", want=("200",), tries=90): + return False + tok, why = self._token(w, sub) + if not tok: + say(f" mealie: could not authenticate after the update :: {why}") + return False + rc, code, _ = w.app_curl(sub, "/api/recipes/nope" + secrets.token_hex(5), + "-H", f"Authorization: Bearer {tok}") + if code == "200": + say(" mealie: READBACK UNUSABLE — a slug that cannot exist returned 200") + return False + rc, code, body = w.app_curl(sub, f"/api/recipes/{t['slug']}", + "-H", f"Authorization: Bearer {tok}") + ok = code == "200" and t["name"] in body + say(f" mealie: readback of the seeded recipe http={code} ok={ok}") + return ok + + +# ============================================================================================= +class N8n: + """n8n's own owner-setup API creates the first account; its own login proves it survived.""" + sub = "auto" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/healthz", want=("200",), tries=90): + return None + email = f"drill-{secrets.token_hex(4)}@gate.invalid" + pw = "Drill" + secrets.token_hex(8) + "1" + rc, code, body = w.app_curl(sub, "/rest/owner/setup", "-H", "Content-Type: application/json", + data=json.dumps({"email": email, "firstName": "drill", + "lastName": "drill", "password": pw}), + method="POST") + say(f" n8n: /rest/owner/setup http={code}") + if code not in ("200", "201"): + say(f" n8n: refused {body[:220]}") + return None + return {"email": email, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/healthz", want=("200",), tries=90): + return False + def login(p): + return w.app_curl(sub, "/rest/login", "-H", "Content-Type: application/json", + data=json.dumps({"emailOrLdapLoginId": t["email"], "password": p}), + method="POST") + rc, code, _ = login("wrong-" + secrets.token_hex(6)) + if code == "200": + say(" n8n: READBACK UNUSABLE — a wrong password authenticated") + return False + rc, code, body = login(t["pw"]) + ok = code == "200" and t["email"] in body + say(f" n8n: login as the seeded owner http={code} ok={ok}") + return ok + + +# ============================================================================================= +class Zipline: + """Zipline's own setup/login API. Zipline 4 creates the first user through its own endpoint.""" + sub = "img" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/api/healthcheck", want=("200",), tries=90): + if not w.wait_app(sub, "/", want=("200", "302", "307"), tries=30): + return None + user = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + for path in ("/api/auth/register", "/api/auth/setup"): + rc, code, body = w.app_curl(sub, path, "-H", "Content-Type: application/json", + data=json.dumps({"username": user, "password": pw}), + method="POST") + say(f" zipline: {path} http={code} :: {body[:160]}") + if code in ("200", "201"): + return {"user": user, "pw": pw} + say(" zipline: neither register nor setup accepted a first user") + return None + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200", "302", "307"), tries=60): + return False + def login(p): + return w.app_curl(sub, "/api/auth/login", "-H", "Content-Type: application/json", + data=json.dumps({"username": t["user"], "password": p}), + method="POST") + rc, code, _ = login("wrong-" + secrets.token_hex(6)) + if code == "200": + say(" zipline: READBACK UNUSABLE — a wrong password authenticated") + return False + rc, code, body = login(t["pw"]) + ok = code == "200" + say(f" zipline: login as the seeded user http={code} ok={ok}") + return ok + + +# ============================================================================================= +class Vikunja: + """Vikunja's own REST API: register a user, log in, create a project, read the project back. + Four calls, all the app's own front door.""" + sub = "tasks" + + def _token(self, w, sub, t, pw=None): + rc, code, body = w.app_curl(sub, "/api/v1/login", "-H", "Content-Type: application/json", + data=json.dumps({"username": t["user"], + "password": pw or t["pw"]}), method="POST") + if code != "200": + return None, f"http={code} {body[:160]}" + try: + return json.loads(body)["token"], "" + except Exception: + return None, body[:160] + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/api/v1/info", want=("200",), tries=72): + return None + user = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + rc, code, body = w.app_curl(sub, "/api/v1/register", "-H", "Content-Type: application/json", + data=json.dumps({"username": user, "password": pw, + "email": f"{user}@gate.invalid"}), + method="POST") + say(f" vikunja: register http={code}") + if code not in ("200", "201"): + say(f" vikunja: refused {body[:220]}") + return None + t = {"user": user, "pw": pw} + tok, why = self._token(w, sub, t) + if not tok: + say(f" vikunja: could not log in after registering :: {why}") + return None + title = "drill-" + secrets.token_hex(5) + # Vikunja CREATES with PUT, not POST — a POST answers `405 Method Not Allowed`, which + # reads like a broken fixture and is really the wrong verb. Measured 2026-09-21. + rc, code, body = w.app_curl(sub, "/api/v1/projects", "-H", f"Authorization: Bearer {tok}", + "-H", "Content-Type: application/json", + data=json.dumps({"title": title}), method="PUT") + say(f" vikunja: create project http={code}") + if code not in ("200", "201"): + say(f" vikunja: project refused {body[:220]}") + return None + t["title"] = title + t["pid"] = json.loads(body).get("id") + return t + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/api/v1/info", want=("200",), tries=72): + return False + bad, why = self._token(w, sub, t, pw="wrong-" + secrets.token_hex(6)) + if bad: + say(" vikunja: READBACK UNUSABLE — a wrong password authenticated") + return False + tok, why = self._token(w, sub, t) + if not tok: + say(f" vikunja: the seeded account no longer authenticates :: {why}") + return False + rc, code, body = w.app_curl(sub, f"/api/v1/projects/{t['pid']}", + "-H", f"Authorization: Bearer {tok}") + ok = code == "200" and t["title"] in body + say(f" vikunja: readback of the seeded project http={code} ok={ok}") + return ok + + +# ============================================================================================= +class OpenGist: + """Opengist's own sign-up and sign-in FORMS. + + Two things had to be measured. Its sign-up is CSRF-protected: a bare POST answers 500 with an + HTML page, which reads like a broken app and is really a missing token — fetch the form, keep + its cookie, send its `_csrf` back. And its REST API refuses the account's own password + (`401 {"message":"Bad crendentials"}`) because it wants a token the app will not mint without a + browser. So the SEEDED DATA is the account itself and the READBACK is a real sign-in, which is + the same shape the docmost and navidrome fixtures use. + + LIMITATION, recorded rather than papered over: this is the DATABASE half. A gist's CONTENT is + not seeded, because that needs the API token above. + """ + sub = "gist" + + def _form(self, w, sub, path, jar, fields): + rc, code, html = w.app_curl(sub, path, "-b", jar, "-c", jar) + m = re.search(r'name="_csrf"[^>]*value="([^"]+)"', html or "") + if not m: + return None, f"no _csrf on {path} (http={code})" + body = "&".join([f"_csrf={m.group(1)}"] + [f"{k}={v}" for k, v in fields.items()]) + rc, code, out = w.app_curl(sub, path, "-b", jar, "-c", jar, + "-H", "Content-Type: application/x-www-form-urlencoded", + data=body, method="POST") + return code, out + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=72): + return None + user = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + jar = f"/tmp/og-{secrets.token_hex(4)}.jar" + code, out = self._form(w, sub, "/register", jar, {"username": user, "password": pw}) + say(f" opengist: /register (with its own _csrf) http={code}") + if code not in ("200", "302", "303"): + say(f" opengist: refused {str(out)[:200]}") + return None + return {"user": user, "pw": pw} + + def verify(self, w, sub, t, say): + # Wait for the LOGIN FORM, not for the root page. Measured 2026-09-21: immediately after a + # successful update the root answers while /login does not yet carry its `_csrf`, so the + # sign-in silently fails and the app looks like it lost the account. It had not. + # 1.15 moved every page under `/-/` (`/-/login`, `/-/all`; `/login` answers 404) — measured + # 2026-09-23 night. Ask the app which shape it serves instead of assuming one. + pre, home_path = "", "/" + for _ in range(72): + if w.app_curl(sub, "/-/login")[1] == "200": + pre, home_path = "/-", "/-/all" + break + if w.app_curl(sub, "/login")[1] == "200": + break + time.sleep(5) + else: + say(" opengist: neither /login nor /-/login came back after the update") + return False + say(f" opengist: sign-in form at {pre}/login") + for _ in range(24): + rc, code, html = w.app_curl(sub, pre + "/login") + if code == "200" and '_csrf' in (html or ""): + break + time.sleep(5) + jar = f"/tmp/og-{secrets.token_hex(4)}.jar" + code, _ = self._form(w, sub, pre + "/login", jar, + {"username": t["user"], "password": "wrong-" + secrets.token_hex(5)}) + rc, c2, home = w.app_curl(sub, home_path, "-b", jar) + if t["user"] in (home or ""): + say(" opengist: READBACK UNUSABLE — a wrong password signed in") + return False + jar2 = f"/tmp/og-{secrets.token_hex(4)}.jar" + code, _ = self._form(w, sub, pre + "/login", jar2, {"username": t["user"], "password": t["pw"]}) + rc, c2, home = w.app_curl(sub, home_path, "-b", jar2) + signed_in = t["user"] in (home or "") + # The ACCOUNT's own public page is the readback that does not depend on a cookie: 1.15 marks its + # session cookie Secure, so a plain-HTTP bench cannot send it back (measured 2026-09-23 night). + # A user that was never created must 404 on the same call, or the readback proves nothing. + rc, pc, prof = w.app_curl(sub, "/" + t["user"]) + rc, nc, _ = w.app_curl(sub, "/nobody" + secrets.token_hex(4)) + profile = pc == "200" and t["user"] in (prof or "") + if nc == "200": + say(" opengist: READBACK UNUSABLE — a never-created user's page answered 200") + return None + say(f" opengist: account page /{t['user']} http={pc} found={profile} (never-created user {nc}); " + f"sign-in http={code} name_on_page={signed_in}") + return profile + + +# ============================================================================================= +class Papra: + """Papra's own e-mail sign-up and sign-in endpoints.""" + sub = "papra" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/api/health", want=("200",), tries=72): + if not w.wait_app(sub, "/", want=("200", "302"), tries=30): + return None + email = f"drill-{secrets.token_hex(4)}@gate.invalid" + pw = "Drill-" + secrets.token_hex(10) + rc, code, body = w.app_curl(sub, "/api/auth/sign-up/email", + "-H", "Content-Type: application/json", + data=json.dumps({"email": email, "password": pw, + "name": "drill"}), method="POST") + say(f" papra: sign-up http={code}") + if code not in ("200", "201"): + say(f" papra: refused {body[:220]}") + return None + return {"email": email, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=72): + return False + def signin(p): + return w.app_curl(sub, "/api/auth/sign-in/email", + "-H", "Content-Type: application/json", + data=json.dumps({"email": t["email"], "password": p}), method="POST") + rc, code, _ = signin("wrong-" + secrets.token_hex(6)) + if code == "200": + say(" papra: READBACK UNUSABLE — a wrong password authenticated") + return False + rc, code, body = signin(t["pw"]) + ok = code == "200" + say(f" papra: sign-in as the seeded account http={code} ok={ok}") + return ok + + +# ============================================================================================= +class HomeAssistant: + """Home Assistant's own onboarding API creates the owner account and hands back a code the + same API exchanges for a token. Both are the app's own documented non-browser route.""" + sub = "ha" + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=120): + return None + user = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + rc, code, body = w.app_curl(sub, "/api/onboarding/users", + "-H", "Content-Type: application/json", + data=json.dumps({"client_id": f"https://{sub}.felhom.invalid/", + "name": "drill", "username": user, + "password": pw, "language": "en"}), + method="POST") + say(f" home-assistant: /api/onboarding/users http={code}") + if code not in ("200", "201"): + say(f" home-assistant: refused {body[:220]}") + return None + return {"user": user, "pw": pw} + + def _login(self, w, sub, user, pw): + """The app's own login flow: start it, then answer it. A 200 with a step_id of + `mfa`/`init` means the credentials were REFUSED; only `create_entry` is a pass.""" + rc, code, body = w.app_curl(sub, "/auth/login_flow", + "-H", "Content-Type: application/json", + data=json.dumps({"client_id": f"https://{sub}.felhom.invalid/", + "handler": ["homeassistant", None], + "redirect_uri": f"https://{sub}.felhom.invalid/", + "type": "authorize"}), method="POST") + if code not in ("200", "201"): + return None, f"flow start http={code} {body[:160]}" + try: + fid = json.loads(body)["flow_id"] + except Exception: + return None, body[:160] + rc, code, body = w.app_curl(sub, f"/auth/login_flow/{fid}", + "-H", "Content-Type: application/json", + data=json.dumps({"client_id": f"https://{sub}.felhom.invalid/", + "username": user, "password": pw}), + method="POST") + try: + j = json.loads(body) + except Exception: + return None, body[:160] + return (j.get("result") if j.get("type") == "create_entry" else None), body[:200] + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/", want=("200", "302"), tries=120): + return False + bad, why = self._login(w, sub, t["user"], "wrong-" + secrets.token_hex(6)) + if bad: + say(" home-assistant: READBACK UNUSABLE — a wrong password authenticated") + return False + good, why = self._login(w, sub, t["user"], t["pw"]) + ok = bool(good) + say(f" home-assistant: login as the seeded owner ok={ok}") + if not ok: + say(f" home-assistant: {why}") + return ok + + +# ============================================================================================= +class Romm: + """RomM's own user API, driven the way RomM's own front end drives it. + + Three things had to be measured rather than guessed, and each one answered a 403 or a 422 that + looked like a different fault: RomM sets a **`romm_csrftoken` cookie** on any GET and requires + it back in an **`x-csrftoken` header** (a bare POST is `403 CSRF token verification failed`, + which reads like an auth problem); the fields go in the **JSON body**, not the query string (a + query-string POST is `422 Field required` for every field it was just given); and `email` is + required alongside username, password and role. + + On a fresh install with no admin the first `POST /api/users` is accepted unauthenticated; + afterwards it is not — which is what makes the readback (`POST /api/login` as that user) a real + authentication rather than a repeat of the seed. + + LIMITATION: this is the DATABASE half. RomM's other half is the ROM library on the drive, which + this does not populate. + """ + sub = "arcade" + + def _csrf(self, w, sub): + jar = f"/tmp/romm-{secrets.token_hex(4)}.jar" + w.app_curl(sub, "/api/heartbeat", "-c", jar) + out = w.sh(["bash", "-lc", f"grep -i csrf {jar} | awk '{{print $7}}'"]).stdout or "" + return jar, out.strip() + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/api/heartbeat", want=("200",), tries=120): + if not w.wait_app(sub, "/", want=("200", "302"), tries=30): + return None + jar, tok = self._csrf(w, sub) + if not tok: + say(" romm: no romm_csrftoken cookie was set on /api/heartbeat") + return None + user = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(10) + rc, code, body = w.app_curl( + sub, "/api/users", "-b", jar, "-H", f"x-csrftoken: {tok}", + "-H", "Content-Type: application/json", + data=json.dumps({"username": user, "email": f"{user}@gate.invalid", + "password": pw, "role": "admin"}), method="POST") + say(f" romm: POST /api/users http={code}") + if code not in ("200", "201"): + say(f" romm: refused {body[:220]}") + return None + return {"user": user, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/api/heartbeat", want=("200",), tries=120): + return False + jar, tok = self._csrf(w, sub) + rc, code, _ = w.app_curl(sub, "/api/login", "-b", jar, "-H", f"x-csrftoken: {tok}", + "-u", f"{t['user']}:wrong-{secrets.token_hex(5)}", method="POST") + if code == "200": + say(" romm: READBACK UNUSABLE — a wrong password authenticated") + return False + rc, code, body = w.app_curl(sub, "/api/login", "-b", jar, "-H", f"x-csrftoken: {tok}", + "-u", f"{t['user']}:{t['pw']}", method="POST") + ok = code == "200" + say(f" romm: login as the seeded user http={code} ok={ok}") + if not ok: + say(f" romm: body {body[:200]}") + return ok + + + +# ============================================================================================= +class Wishlist: + """Wishlist's own SvelteKit FORM actions (added night 2026-09-23, R-612's app). Sign-up at + /signup, then prove the account survived by signing in at /login — and by a wrong password being + REFUSED on the same call, so a readback that always says "ok" fails instead of passing. + SvelteKit refuses a cross-site form post: the Origin must be the app's own https origin.""" + sub = "wishlist" + + def _post(self, w, sub, path, body): + origin = "https://" + getattr(w, "host", lambda s: f"{s}.{w.DOMAIN}")(sub) # the Host the request carries + rc, code, out = w.app_curl(sub, path, "-H", f"Origin: {origin}", "-H", "x-sveltekit-action: true", + "-H", "Content-Type: application/x-www-form-urlencoded", + data=body, method="POST") + try: + return code, json.loads(out) + except Exception: + return code, {"type": "unparsed", "raw": (out or "")[:200]} + + def seed(self, w, sub, say): + if not w.wait_app(sub, "/signup", want=("200",), tries=72): + return None + u = "drill" + secrets.token_hex(3) + pw = "Drill-" + secrets.token_hex(8) + code, j = self._post(w, sub, "/signup", + f"name=Drill&username={u}&email={u}%40example.invalid&password={pw}&tokenId=") + say(f" wishlist: /signup http={code} type={j.get('type')}") + if j.get("type") not in ("success", "redirect"): + self.tried = f"POST /signup -> {code} {str(j)[:150]}" + return None + return {"u": u, "pw": pw} + + def verify(self, w, sub, t, say): + if not w.wait_app(sub, "/login", want=("200",), tries=72): + say(" wishlist: /login never came back") + return False + c1, bad = self._post(w, sub, "/login", f"username={t['u']}&password=wrong-{secrets.token_hex(5)}") + if bad.get("type") != "failure": + say(f" wishlist: READBACK UNUSABLE — a wrong password was not refused ({bad.get('type')})") + return None + c2, good = self._post(w, sub, "/login", f"username={t['u']}&password={t['pw']}") + ok = good.get("type") in ("success", "redirect") + say(f" wishlist: sign-in as the seeded user type={good.get('type')} ok={ok} (wrong password refused)") + return ok + +FIXTURES = { + "home-assistant": HomeAssistant(), + "romm": Romm(), + "vikunja": Vikunja(), + "opengist": OpenGist(), + "papra": Papra(), + "mealie": Mealie(), + "n8n": N8n(), + "zipline": Zipline(), + "grafana": Grafana(), + "audiobookshelf": AudiobookShelf(), + "actualbudget": ActualBudget(), + "nextcloud": Nextcloud(), + "adventurelog": Django("adventurelog", "travel", "/admin/login/"), + "tandoor": Django("tandoor", "recipes", "/accounts/login/", + python="/opt/recipes/venv/bin/python", workdir="/opt/recipes"), + "privatebin": PrivateBin(), + "docmost": Docmost(), + "bookstack": BookStack(), + "gitea": Gitea(), + "navidrome": Navidrome(), + "vaultwarden": Vaultwarden(), + "wishlist": Wishlist(), +} diff --git a/documentation/audits/night-2026-09-24/tools/ncfiles.py b/documentation/audits/night-2026-09-24/tools/ncfiles.py new file mode 100644 index 00000000..75728430 --- /dev/null +++ b/documentation/audits/night-2026-09-24/tools/ncfiles.py @@ -0,0 +1,34 @@ +"""Nextcloud FILES through the front door (WebDAV as the seeded user) — the household's own route.""" +import secrets + + +def put(w, sub, t, name, body): + rc, code, out = w.app_curl(sub, f"/remote.php/dav/files/{t['uid']}/{name}", "-u", f"{t['uid']}:{t['pw']}", + "-H", "Content-Type: text/plain", method="PUT", data=body) + return code in ("201", "204") + + +def get(w, sub, t, name): + rc, code, out = w.app_curl(sub, f"/remote.php/dav/files/{t['uid']}/{name}", "-u", f"{t['uid']}:{t['pw']}") + return code, out + + +def seed_files(w, sub, t, n=3, say=print): + files = {} + for i in range(n): + name, body = f"felhom-seed-{i}-{secrets.token_hex(3)}.txt", "seed " + secrets.token_hex(16) + ok = put(w, sub, t, name, body) + say(f" nextcloud file PUT {name} ok={ok}") + if ok: + files[name] = body + return files + + +def verify_files(w, sub, t, files, say=print): + res = {} + for name, body in files.items(): + code, out = get(w, sub, t, name) + res[name] = (code == "200" and out == body) + neg_code, _ = get(w, sub, t, "felhom-never-" + secrets.token_hex(4) + ".txt") + say(f" nextcloud files read back: {sum(res.values())}/{len(res)} (negative control http={neg_code})") + return res, neg_code diff --git a/documentation/audits/night-2026-09-24/tools/repoint.py b/documentation/audits/night-2026-09-24/tools/repoint.py new file mode 100644 index 00000000..c59956fe --- /dev/null +++ b/documentation/audits/night-2026-09-24/tools/repoint.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Point guest 9202 at the drill catalog (and a 90 s health timeout), or restore the saved config. + +`09` §6.5: `git.repo_url` alone is INERT (R-615) — the cache dir must go too. The saved copy is +`controller.yaml.pre-night0924` (NOT the older `.pre-28`, which a restore must never pick up). +""" +import re, sys, io +sys.path.insert(0, '.') +import walk as w + +VOL = "/var/lib/docker/volumes/felhom-controller-data/_data" +DRILL_REPO = "https://gitea.dooplex.hu/admin/app-catalog-drill.git" + + +def creds(): + for l in io.open("/home/kisfenyo/.git-credentials").read().strip().split("\n"): + m = re.match(r'https://(admin):([^@]+)@gitea\.dooplex\.hu', l) + if m: + return m.group(1), m.group(2) + raise SystemExit("no admin credential") + + +def to_drill(): + u, t = creds() + print(w.guest(f""" +set -e +test -f {VOL}/controller.yaml.pre-night0924 || cp -p {VOL}/controller.yaml {VOL}/controller.yaml.pre-night0924 +python3 - <<'PY' +import re +p = "{VOL}/controller.yaml" +s = open(p).read() +s = re.sub(r'(^\\s+repo_url: ).*$', r'\\g<1>{DRILL_REPO}', s, count=1, flags=re.M) +s = re.sub(r'(^git:(?:\\n\\s+.*)*?\\n\\s+token: ).*$', r'\\g<1>"{t}"', s, count=1, flags=re.M) +s = re.sub(r'(^git:(?:\\n\\s+.*)*?\\n\\s+username: ).*$', r'\\g<1>"{u}"', s, count=1, flags=re.M) +if not re.search(r'^update:', s, re.M): + s += "update:\\n health_timeout: 90s\\n" +open(p, "w").write(s) +PY +rm -rf {VOL}/catalog-cache {VOL}/data/catalog-cache +docker restart felhom-controller >/dev/null +sleep 15 +grep -A6 '^git:' {VOL}/controller.yaml | sed 's/token:.*/token: /' +grep -A2 '^update:' {VOL}/controller.yaml +""")) + + +def restore(): + print(w.guest(f""" +set -e +cp -p {VOL}/controller.yaml.pre-night0924 {VOL}/controller.yaml +rm -rf {VOL}/catalog-cache {VOL}/data/catalog-cache +docker restart felhom-controller >/dev/null +sleep 15 +grep -A6 '^git:' {VOL}/controller.yaml | sed 's/token:.*/token: /' +grep -c '^update:' {VOL}/controller.yaml || true +""")) + + +if __name__ == "__main__": + to_drill() if sys.argv[1] == "drill" else restore() diff --git a/documentation/audits/night-2026-09-24/tools/spikeA1.py b/documentation/audits/night-2026-09-24/tools/spikeA1.py new file mode 100644 index 00000000..140581ca --- /dev/null +++ b/documentation/audits/night-2026-09-24/tools/spikeA1.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""A1 spike: what does nextcloud's Tier-2 mirror hold, file by file, and what would a whole restore need? +nextcloud installed from the drill (= live head), a user + 3 files through WebDAV, then an update whose tag +does not exist: its backing-up leg runs (unit + Tier 2), the pull fails, nothing moves.""" +import json, re, sys, time +sys.path.insert(0, ".") +import walk as w, fixtures as fx, ncfiles as nf +F, SUB, APP = fx.Nextcloud(), "cloud-a1", "nextcloud" +CAT = f"{w.DRILL}/templates/{APP}" +out = {} +w.login() +w.sync_rescan() +out["deployed"] = w.deploy(APP, SUB) +A = F.seed(w, SUB, w.say) +files = nf.seed_files(w, SUB, A, 3, w.say) +out["files_seeded"] = len(files) + + +def drill(edit, msg): + import fcntl + with open(f"{w.SC}/drill.lock", "w") as lk: + fcntl.flock(lk, fcntl.LOCK_EX) + w.sh(["git", "-C", w.DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120) + edit() + w.sh(["git", "-C", w.DRILL, "commit", "-qam", "NIGHT-A1 " + msg]) + w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120) + w.say("drill: " + msg) + + +orig = open(f"{CAT}/docker-compose.yml").read() +drill(lambda: open(f"{CAT}/docker-compose.yml", "w").write(orig.replace("nextcloud:34.0.4-apache", "nextcloud:34.0.99-notag")), "nextcloud 34.0.99-notag (backing-up runs, pull fails)") +w.sync_rescan(APP, "nextcloud:34.0.99-notag", tries=60) +out["press"] = w.press_update(APP) +drill(lambda: open(f"{CAT}/docker-compose.yml", "w").write(orig), "nextcloud back to 34.0.4") +w.sync_rescan() +cfg = w.guest("python3 - <<'PY'\nimport json\nd=json.load(open('/var/lib/docker/volumes/felhom-controller-data/_data/data/settings.json'))\nprint(json.dumps((d.get('cross_drive') or d.get('crossdrive') or {}).get('nextcloud'),indent=1))\nPY") +out["tier2_record"] = cfg +w.say("Tier-2 record:\n" + cfg) +lay = w.guest("""for b in /mnt/sys_drive/felhom-data/backups/secondary/nextcloud /mnt/felhom-drives/scratch_hdd/backups/secondary/nextcloud; do [ -d $b ] || continue; echo "== $b"; ls -la $b; du -sh $b/* 2>/dev/null; find $b -maxdepth 4 -type d | head -30; done +echo "== live userdata"; ls -la /mnt/felhom-drives/scratch_hdd/userdata/nextcloud 2>&1 | head; find /mnt/felhom-drives/scratch_hdd/userdata/nextcloud -name 'felhom-seed-*' -printf '%p %s %TY-%Tm-%Td %TH:%TM %u:%g %m\\n' 2>/dev/null +echo "== mirror copies of the seeds"; find /mnt/sys_drive/felhom-data/backups/secondary/nextcloud /mnt/felhom-drives/scratch_hdd/backups/secondary/nextcloud -name 'felhom-seed-*' -printf '%p %s %TY-%Tm-%Td %TH:%TM %u:%g %m\\n' 2>/dev/null""") +out["layout"] = lay +w.say("layout:\n" + lay) +json.dump({"A": {"uid": A["uid"]}, "files": list(files)}, open("../A1/spike-state.json", "w"), indent=2) +json.dump(out, open("../A1/00-spike.json", "w"), indent=2, ensure_ascii=False, default=str) +open("../A1/00-spike.log", "w").write("\n".join(w.LOG) + "\n") +json.dump({"A": A, "files": files}, open(f"{w.SC}/a1-secret-state.json", "w")) diff --git a/documentation/audits/night-2026-09-24/tools/walk.py b/documentation/audits/night-2026-09-24/tools/walk.py new file mode 100644 index 00000000..fa85d055 --- /dev/null +++ b/documentation/audits/night-2026-09-24/tools/walk.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""walk.py — ONE app's full update walk on guest 9202, through the product's own endpoints. + +EVIDENCE, NOT PRODUCT. It presses exactly the buttons a person presses: + POST /api/stacks//deploy · POST /api/sync · POST /api/stacks/rescan + POST /api/stacks//update · POST /api/stacks//remove +and reads GET /api/stacks/. No controller code exists for it. + +The walk, per `09` §6.4 and the update-night brief §4: + 1 deploy from the DRILL catalog at the LIVE pin + 2 seed through the app's OWN front door (R-156: never a volume, never SQL) + 3 read the seed back <- control C1; a fixture that cannot prove itself proves nothing + 4 „Mentés most" + 5 commit the real one-step bump to the DRILL repo, sync, rescan, read the badge in BOTH languages + 6 press the guarded Update, record every phase with timestamps + 7 read the seed back through the front door + 8 the four version observables side by side + 9 write the verdict record in `09`'s JSON shape + +`inconclusive` is a first-class verdict and is NEVER collapsed into `failed`. +""" +import argparse, json, os, re, subprocess, sys, time +from datetime import datetime, timezone + +SC = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/25947a06-40b7-43dd-8334-2519e02142da/scratchpad" +EV = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/night-2026-09-24" +DRILL = "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill" +# GUEST=9201 selects demo-hp's hub-enabled guest (the mail proof); default 9202, the scratch guest. +GUEST = os.environ.get("GUEST", "9202") +BASE = {"9202": "https://192.168.0.114", "9201": "https://192.168.0.138"}[GUEST] +DOMAIN = os.environ.get("DOMAIN", "enkisfelhom.hu") +HOSTHDR = f"Host: felhom.{DOMAIN}" +HP = "demo-hp" + +LOG = [] + + +def say(*a): + line = " ".join(str(x) for x in a) + ts = datetime.now().strftime("%H:%M:%S") + print(f"{ts} {line}", flush=True) + LOG.append(f"{ts} {line}") + + +def sh(args, timeout=300, inp=None): + try: + return subprocess.run(args, capture_output=True, text=True, timeout=timeout, input=inp) + except (subprocess.TimeoutExpired, OSError) as e: + return subprocess.CompletedProcess(args, 124, "", f"{e}") + + +def guest(script, timeout=600): + """Run a bash script inside guest 9202. Piped as a file — never as an argument (quoting).""" + # ONE TEMP FILE PER CALL (night 2026-09-23): the shared /tmp/w.sh swapped scripts under + # two concurrent walks (memory: guest-helper-shares-one-tmp-file). + import secrets as _s + t = f"/tmp/w{GUEST}-{os.getpid()}-{_s.token_hex(4)}.sh" + r = sh(["ssh", "-o", "ConnectTimeout=20", "-o", "StrictHostKeyChecking=accept-new", HP, + f"export LC_ALL=C; cat > {t}; pct push {GUEST} {t} {t} >/dev/null 2>&1; " + f"pct exec {GUEST} -- bash {t}; pct exec {GUEST} -- rm -f {t}; rm -f {t}"], + timeout=timeout, inp=script) + return r.stdout or "" + + +def login(): + pw = open(f"{SC}/.ctlpw").read().strip() + sh(["curl", "-sk", "-D", f"{SC}/hdr{os.getpid()}.txt", "-o", "/dev/null", "-H", HOSTHDR, + "-X", "POST", "--data-urlencode", f"password={pw}", f"{BASE}/login"]) + h = open(f"{SC}/hdr{os.getpid()}.txt").read() + m = re.search(r"felhom_session=[A-Za-z0-9._-]+", h, re.I) + if not m: + sys.exit("login failed: no session cookie") + open(f"{SC}/sess{os.getpid()}.txt", "w").write(m.group(0)) + r = sh(["curl", "-sk", "-L", "-H", HOSTHDR, "-H", f"Cookie: {m.group(0)}", f"{BASE}/"]) + c = re.search(r'/deploy-fields` — instead of assuming DOMAIN+SUBDOMAIN. + + Measured 2026-09-21: three apps in one batch refused at the deploy with a correct 400 because + a required field was absent — `HDD_PATH` (navidrome, audiobookshelf) and an admin password + (grafana). The refusals happen BEFORE anything is created (`deploy.go:324`), which is the only + reason this was safe to discover by running it (live-probes rule). + + A `path` field must name a directory that ALREADY EXISTS (`deploy.go:330`), so one is made on + the scratch drive first — the same act the drive browser performs for a household. + """ + code, d = ctl("GET", f"/api/stacks/{name}/deploy-fields") + fields = (((d.get("data") or {}).get("metadata") or {}).get("deploy_fields")) or [] + values = {"DOMAIN": DOMAIN, "SUBDOMAIN": sub} + made = [] + for f in fields: + ev, ty = f.get("env_var"), f.get("type") + if ev in values: + continue + # `type: password` is MANDATORY whatever `required` says — `deploy.go:305-312` refuses + # when the caller sends none, deliberately ("the user needs to know their password"), + # while `.felhom.yml` declares `required: false` and the API serves that verbatim. A + # caller that trusts the contract gets a 400. Measured tonight on grafana; filed. + if not f.get("required") and ty != "password": + continue # the controller generates the optional secrets itself + if ty == "path": + p = f"{DRIVE}/{name}" + values[ev] = p + made.append(p) + elif ty in ("secret", "password"): + import secrets as _s + values[ev] = "Drill-" + _s.token_hex(12) + GENERATED.setdefault(name, {})[ev] = values[ev] + elif f.get("default"): + values[ev] = f["default"] + else: + values[ev] = f"drill-{name}" + if made: + guest("mkdir -p " + " ".join(made) + "; ls -ld " + " ".join(made)) + say(f" [1] made the drive paths this app requires: {made}") + extra = [k for k in values if k not in ("DOMAIN", "SUBDOMAIN")] + if extra: + say(f" [1] required fields filled beyond DOMAIN/SUBDOMAIN: {extra}") + return values + + +def deploy(name, sub, extra_values=None): + st = stack(name) + if st.get("deployed"): + say(f" [1] {name} already deployed — reusing") + return True + values = deploy_values(name, sub) + if extra_values: + values.update(extra_values) + code, d = ctl("POST", f"/api/stacks/{name}/deploy", {"values": values}) + say(f" [1] deploy -> {code} {str(d)[:120]}") + if code != "202": + return False + # WAIT FOR `deployed`, NOT FOR `running`. Measured 2026-09-21 on tandoor: docker reported the + # container `healthy` while the controller's own state read `unhealthy` — a gate on `running` + # alone therefore times out on an app that is up. The state is RECORDED rather than required; + # the real gate is the fixture's own `wait_app`, which asks whether the APP answers. + seen = None + for _ in range(90): + time.sleep(5) + st = stack(name) + seen = st.get("state") + # `deployed` alone is NOT enough and `state` alone is NOT right. Measured 2026-09-21: + # tandoor reads `unhealthy` while serving (R-618), so gating on "running" hangs; and romm + # read `deployed=True, state=degraded, pinned_images=None` twenty seconds in, i.e. the + # deploy had not finished writing app.yaml. The PIN is the deploy's own completion mark + # (`runComposeDeploy` writes it), so that is what to wait for. + pins = (st.get("app_config") or {}).get("pinned_images") + if st.get("deployed") and pins and seen in ("running", "unhealthy", "degraded"): + say(f" [1] deployed, controller state={seen}, " + f"pinned={(st.get('app_config') or {}).get('pinned_images')}") + if seen != "running": + say(f" [1] NOTE: the controller's own state is {seen!r}, not 'running' — recorded, " + f"not treated as a failure; the fixture's front-door wait is the real gate") + return True + say(f" [1] never became deployed (last controller state={seen!r})") + return False + + +def backup_now(name): + """R-648 (2026-09-23): NO whole-box „Mentés most" from a drill, ever. + + `POST /api/backup/run` is the only backup endpoint and it is WHOLE-BOX: on 9201 it stopped and + restarted 9 of 10 standing apps twice, and on 9202 it broke a deploy in flight (R-634). The product + has NO per-app backup endpoint (router.go: /backup/run, /backup/tier2 only); the per-app backup + exists only inside the guarded update, whose `backing-up` phase calls RunAppBackupNow for the one + app. So this presses nothing: the update takes the throwaway app's own backup, and says so in its + phase list. A seed written "after the backup" is therefore written before the update's own backup + — the undo's last-second copy is still the one that must bring it back.""" + say(f" [4] backup press SKIPPED for {name} (R-648: whole-box only; the update's backing-up phase backs up {name} alone)") + return None + +def drill_bump(app, frm, to, service_hint=None): + """Serialised across concurrent walks: one git working tree, one lock.""" + import fcntl + with open(f"{SC}/drill.lock", "w") as lk: + fcntl.flock(lk, fcntl.LOCK_EX) + sh(["git", "-C", DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120) + return _drill_bump(app, frm, to, service_hint) + + +def _drill_bump(app, frm, to, service_hint=None): + """Commit the edge to the DRILL repo. catalog_since set by hand (the drill repo has no gates). + + `frm`/`to` may be comma-separated lists of the SAME length: an app whose own version lives in + two images (adventurelog's backend and frontend) moves both in one edge, while its engine + sidecar stays where it is — `09` §3b Q3's rule is per SERVICE, and an app-half edge must move + every service that carries the app's own version and no others. + """ + comp = f"{DRILL}/templates/{app}/docker-compose.yml" + fy = f"{DRILL}/templates/{app}/.felhom.yml" + s = open(comp).read() + froms = [x.strip() for x in frm.split(",") if x.strip()] + tos = [x.strip() for x in to.split(",") if x.strip()] + if len(froms) != len(tos): + say(f" [5] from/to lists differ in length: {froms} vs {tos}") + return None + for f1, t1 in zip(froms, tos): + if f"image: {f1}" not in s: + say(f" [5] FROM ref not found in compose: {f1}") + return None + s = s.replace(f"image: {f1}", f"image: {t1}") + open(comp, "w").write(s) + f = open(fy).read() + today = datetime.now().strftime("%Y-%m-%d") + f = re.sub(r'^catalog_since:.*$', f'catalog_since: "{today}"', f, count=1, flags=re.M) + open(fy, "w").write(f) + sh(["git", "-C", DRILL, "add", "-A"]) + sh(["git", "-C", DRILL, "commit", "-q", "-m", f"DRILL {app}: {frm} -> {to}"]) + r = sh(["git", "-C", DRILL, "push", "-q", "origin", "main"], timeout=120) + h = sh(["git", "-C", DRILL, "rev-parse", "--short=12", "HEAD"]).stdout.strip() + say(f" [5] drill commit {h}: {app} {frm} -> {to} (push rc={r.returncode})") + return h + + +def sync_rescan(expect_app=None, expect_ref=None, tries=12, delay=5): + """Sync, rescan, and — when told what to expect — WAIT FOR THE BADGE TO CATCH UP. + + R-607: `POST /api/sync` answers "nincs valtozas" while the catalog HAS moved, and + `catalog_images` stays stale until a separate rescan. Tonight showed the rescan alone is not + enough either: mealie's badge read "Naprakesz" seconds after its bump was pushed, and the + Update that followed moved nothing and still reported "Frissitve". So when the caller knows + which reference should appear, this polls for it and SAYS HOW LONG IT TOOK — which is the + NUMBER R-607 asks for and has never had. + """ + t0 = time.time() + ctl("POST", "/api/sync") + time.sleep(2) + ctl("POST", "/api/stacks/rescan") + time.sleep(2) + if not expect_app or not expect_ref: + return None + for i in range(tries): + cat = stack(expect_app).get("catalog_images") or {} + if expect_ref in cat.values(): + waited = round(time.time() - t0, 1) + if i: + say(f" [sync] the badge needed {waited}s and {i+1} sync+rescan rounds to catch up " + f"to {expect_ref} — R-607's window, measured") + return waited + time.sleep(delay) + ctl("POST", "/api/sync") + time.sleep(1) + ctl("POST", "/api/stacks/rescan") + say(f" [sync] the badge NEVER caught up to {expect_ref} in {round(time.time()-t0,1)}s — " + f"catalog_images = {stack(expect_app).get('catalog_images')}") + return None + + +def badges(name): + out = {} + for lang, suffix in (("hu", ""), ("en", "?lang=en")): + h = page(f"/apps/{name}{suffix}") + m = re.findall(r']*title="([^"]*)"[^>]*>([^<]*)<', h) + out[lang] = [{"title": a.strip(), "text": b.strip()} for a, b in m][:3] + return out + + +def press_update(name, poll=1.0, cap_s=1800): + code, d = ctl("POST", f"/api/stacks/{name}/update") + say(f" [6] Update -> {code} {str(d)[:220]}") + if code not in ("202", "200"): + return {"accepted": False, "http": code, "refusal": d, "phases": [], "duration_s": 0} + phases, seen, t0 = [], None, time.time() + while time.time() - t0 < cap_s: + st = stack(name) + ph = st.get("update_phase") + if ph != seen: + seen = ph + rec = {"t": round(time.time() - t0, 1), "phase": ph, + "label": st.get("update_phase_label"), "updating": st.get("updating"), + "error": st.get("update_error"), "hold": st.get("hold_reason")} + phases.append(rec) + say(f" +{rec['t']:>6.1f}s phase={ph} label={rec['label']} " + f"err={rec['error']} hold={rec['hold']}") + if not st.get("updating") and ph in ("done", "failed", "undone", None) and time.time() - t0 > 3: + break + time.sleep(poll) + st = stack(name) + return {"accepted": True, "http": code, "phases": phases, + "duration_s": round(time.time() - t0, 1), + "final_phase": st.get("update_phase"), "update_error": st.get("update_error"), + "hold_reason": st.get("hold_reason"), "state": st.get("state")} + + +def observables(name): + st = stack(name) + ac = st.get("app_config") or {} + live = guest(f""" +grep -E '^\\s+image:' /opt/docker/stacks/{name}/docker-compose.yml 2>/dev/null | sed 's/^ *//' +echo '---inspect---' +for c in $(docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'); do + echo -n "$c "; docker inspect "$c" --format '{{{{.Config.Image}}}} running={{{{.State.Running}}}} restarts={{{{.RestartCount}}}}' +done +""") + a, _, b = live.partition("---inspect---") + return { + "pinned_images": ac.get("pinned_images"), + "installed_images": {k: (v.get("ref") if isinstance(v, dict) else v) + for k, v in (ac.get("installed_images") or {}).items()}, + "catalog_images": st.get("catalog_images"), + "live_compose_image_lines": [x for x in a.strip().splitlines() if x.strip()], + "docker_inspect": [x for x in b.strip().splitlines() if x.strip()], + } + + +def app_logs(name, lines=400): + """The app's own container log, DECODED. The endpoint answers a JSON envelope whose `logs` is + one string with escaped newlines — a scan over the envelope sees a single enormous line and + finds nothing, which reads exactly like "the app printed no migration line" and is not. R-96 + rule 3 in a new place: an absent line is not evidence when the instrument cannot see lines.""" + code, d = ctl("GET", f"/api/stacks/{name}/logs?lines={lines}") + if isinstance(d, dict): + data = d.get("data") + if isinstance(data, dict) and isinstance(data.get("logs"), str): + return data["logs"] + if isinstance(d.get("_raw"), str): + return d["_raw"] + return str(d) + + +def write_verdict(rec, appdir): + os.makedirs(appdir, exist_ok=True) + p = os.path.join(appdir, "verdict.json") + json.dump(rec, open(p, "w"), indent=2, ensure_ascii=False) + say(f" [9] verdict {rec['verdict']} -> {p}") + + +def remove(name): + """Remove through the PRODUCT, never `docker rm` (live-probes rule). The remove endpoint + refuses a running stack — `409 still running` — so the stop is part of the act, not a tidy-up.""" + c1, d1 = ctl("POST", f"/api/stacks/{name}/stop") + say(f" [X] stop -> {c1} {str(d1)[:100]}") + for _ in range(24): + time.sleep(5) + if stack(name).get("state") != "running": + break + code, d = ctl("POST", f"/api/stacks/{name}/remove", + {"remove_hdd_data": True, "remove_backups": True}) + say(f" [X] remove (with drive data) -> {code} {str(d)[:160]}") + if code == "409": + # R-442's fail-closed guard: when the storage subsystem cannot RESOLVE the app's drive + # path, the removal is REFUSED and the app is kept rather than half-deleted. On guest 9202 + # `/api/disks` answers `agent not configured`, so every app deployed with an HDD_PATH hits + # this. The household's other choice — remove the app, KEEP the data — is accepted, and the + # harness takes it, then tidies its own directory by name at teardown. + say(" [X] refused because the drive path cannot be resolved (R-442, fail-closed and right)" + " — removing the app and KEEPING the drive data instead") + code, d = ctl("POST", f"/api/stacks/{name}/remove", + {"remove_hdd_data": False, "remove_backups": True}) + say(f" [X] remove (keeping drive data) -> {code} {str(d)[:160]}") + time.sleep(5) + st = stack(name) + left = guest(f"ls -d /opt/docker/stacks/{name} 2>/dev/null; " + f"docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'") + say(f" [X] after remove: deployed={st.get('deployed')} leftovers={left.strip()!r}") + return code + + +def app_env(name, key): + """Read one deploy value the CUSTOMER was given (e.g. the generated admin password) from the + app's own `app.yaml`. This is not seeding — it is how the household logs in; the controller + shows them the same value. Data still goes in through the app's own front door.""" + out = guest(f"grep -E '^\\s*{key}:' /opt/docker/stacks/{name}/app.yaml 2>/dev/null | head -1") + if ":" in out: + return out.split(":", 1)[1].strip().strip('"').strip("'") + return "" + + +def snapshots(name): + """The restorable copies the backups page offers for this app.""" + code, d = ctl("GET", f"/api/backup/snapshots?stack={name}") + data = d.get("data") if isinstance(d, dict) else None + if isinstance(data, dict): + for k in ("snapshots", "items", "restore_points"): + if isinstance(data.get(k), list): + return data[k] + return data if isinstance(data, list) else [] + + +def restore(name, snapshot_id=None, wait_s=1200): + """The household's own way out: the „Visszaállítás a mentésből" button on the backups page. + + A FORM post, not an API call — `POST /backup/restore` with `_csrf`, `stack_name`, + `snapshot_id` — because that is the button the sentence tells them to press. + """ + snaps = snapshots(name) + if snapshot_id is None: + if not snaps: + say(f" [R] no restorable copy offered for {name}") + return {"ok": False, "why": "no snapshot offered", "snapshots": snaps} + first = snaps[0] + snapshot_id = first.get("id") or first.get("snapshot_id") or first.get("short_id") + say(f" [R] restoring {name} from snapshot {snapshot_id!r} (of {len(snaps)} offered)") + sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip() + csrf = open(f"{SC}/csrf{os.getpid()}.txt").read().strip() + r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", HOSTHDR, "-H", f"Cookie: {sess}", + "-X", "POST", + "--data-urlencode", f"_csrf={csrf}", + "--data-urlencode", f"stack_name={name}", + "--data-urlencode", f"snapshot_id={snapshot_id}", + f"{BASE}/backup/restore"], timeout=180) + head = (r.stdout or "").split("\n")[0].strip() + loc = [l for l in (r.stdout or "").split("\n") if l.lower().startswith("location:")] + say(f" [R] POST /backup/restore -> {head} {loc[:1]}") + t0 = time.time() + last = None + while time.time() - t0 < wait_s: + code, d = ctl("GET", "/api/backup/restore-status") + dd = d.get("data") or {} + cur = (dd.get("running"), dd.get("phase") or dd.get("state"), dd.get("message")) + if cur != last: + say(f" +{round(time.time()-t0,1):>6.1f}s restore {cur}") + last = cur + if not dd.get("running", False) and time.time() - t0 > 5: + break + time.sleep(2) + st = stack(name) + say(f" [R] after restore: state={st.get('state')} hold={st.get('hold_reason')!r} " + f"phase={st.get('update_phase')}") + return {"ok": True, "snapshot_id": snapshot_id, "snapshots": snaps, + "http": head, "location": loc[:1], "seconds": round(time.time() - t0, 1), + "state_after": st.get("state"), "hold_after": st.get("hold_reason"), + "observables_after": observables(name)}