night 2026-09-24: evidence so far (phase 0, A1 spike, A3 measurements, red-proofs)
gates / gates (push) Successful in 24s
gates / gates (push) Successful in 24s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
drill=635c7e68f716 live=635c7e68f716
|
||||
has_actions: False private: True
|
||||
@@ -0,0 +1,20 @@
|
||||
paths 404 {"_raw": "404 page not found\n"}
|
||||
target 200 {"data": {"known": false}, "ok": true}
|
||||
Filesystem Size Used Avail Use% Mounted on
|
||||
/dev/nvme0n1 938G 72G 819G 8% /mnt/felhom-drives/scratch_hdd
|
||||
/dev/loop1 69G 12G 54G 18% /mnt/sys_drive
|
||||
scratch_hdd
|
||||
total used free shared buff/cache available
|
||||
Mem: 25898 799 23998 34 1134 25098
|
||||
Swap: 512 1 510
|
||||
7
|
||||
/dev/loop1 69G 12G 54G 18% /var/lib/docker
|
||||
felhom-controller Up 2 hours (healthy)
|
||||
privatebin Up 9 hours (healthy)
|
||||
paperless-webserver Up 9 hours (healthy)
|
||||
paperless-postgres Up 9 hours (healthy)
|
||||
paperless-redis Up 9 hours (healthy)
|
||||
gokapi Restarting (1) 1 second ago
|
||||
filebrowser Up 12 hours (healthy)
|
||||
traefik Up 12 hours
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
repo_url: https://gitea.dooplex.hu/admin/app-catalog-drill.git
|
||||
sync_interval: 15m
|
||||
token: <redacted>
|
||||
username: "admin"
|
||||
hub:
|
||||
update:
|
||||
health_timeout: 90s
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"deployed": true,
|
||||
"files_seeded": 3,
|
||||
"press": {
|
||||
"accepted": true,
|
||||
"http": "202",
|
||||
"phases": [
|
||||
{
|
||||
"t": 0.0,
|
||||
"phase": "backing-up",
|
||||
"label": "Biztonsági mentés készül a frissítés előtt…",
|
||||
"updating": true,
|
||||
"error": null,
|
||||
"hold": null
|
||||
},
|
||||
{
|
||||
"t": 17.5,
|
||||
"phase": "safety-dump",
|
||||
"label": "Adatbázis pillanatkép…",
|
||||
"updating": true,
|
||||
"error": null,
|
||||
"hold": null
|
||||
},
|
||||
{
|
||||
"t": 19.5,
|
||||
"phase": "pulling",
|
||||
"label": "Új verzió letöltése…",
|
||||
"updating": true,
|
||||
"error": null,
|
||||
"hold": null
|
||||
},
|
||||
{
|
||||
"t": 21.6,
|
||||
"phase": "failed",
|
||||
"label": "A frissítés nem sikerült",
|
||||
"updating": false,
|
||||
"error": "Az új verzió letöltése nem sikerült, ezért a frissítés elmaradt. Az alkalmazás a korábbi verzióval fut tovább.",
|
||||
"hold": null
|
||||
}
|
||||
],
|
||||
"duration_s": 21.6,
|
||||
"final_phase": "failed",
|
||||
"update_error": "Az új verzió letöltése nem sikerült, ezért a frissítés elmaradt. Az alkalmazás a korábbi verzióval fut tovább.",
|
||||
"hold_reason": null,
|
||||
"state": "running"
|
||||
},
|
||||
"tier2_record": "null\n",
|
||||
"layout": "== live userdata\ntotal 16\ndrwxr-sr-x 4 root 1000 4096 Sep 24 09:11 .\ndrwxrwsr-x 17 root 1000 4096 Sep 24 09:12 ..\ndrwxr-sr-x 3 root root 4096 Sep 24 09:10 appdata\ndrwxr-sr-x 3 root 1000 4096 Sep 24 09:11 backups\n/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-1-7cb158.txt 37 2026-09-24 09:11 www-data:www-data 644\n/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-0-e19728.txt 37 2026-09-24 09:11 www-data:www-data 644\n/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-2-8620d3.txt 37 2026-09-24 09:11 www-data:www-data 644\n== mirror copies of the seeds\n"
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
11:09:53 [1] made the drive paths this app requires: ['/mnt/felhom-drives/scratch_hdd/userdata/nextcloud']
|
||||
11:09:53 [1] required fields filled beyond DOMAIN/SUBDOMAIN: ['NEXTCLOUD_ADMIN_PASSWORD', 'HDD_PATH']
|
||||
11:09:54 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'}
|
||||
11:11:24 [1] deployed, controller state=running, pinned={'nextcloud': 'nextcloud:34.0.4-apache', 'nextcloud-db': 'mariadb:12.3', 'nextcloud-redis': 'redis:7-alpine'}
|
||||
11:11:33 nextcloud: occ user:add :: The account "drill76bf48" was created successfully Display name set to "drill76bf48"
|
||||
11:11:37 nextcloud: seeded user drill76bf48
|
||||
11:11:44 nextcloud file PUT felhom-seed-0-e19728.txt ok=True
|
||||
11:11:44 nextcloud file PUT felhom-seed-1-7cb158.txt ok=True
|
||||
11:11:45 nextcloud file PUT felhom-seed-2-8620d3.txt ok=True
|
||||
11:11:46 drill: nextcloud 34.0.99-notag (backing-up runs, pull fails)
|
||||
11:11:51 [6] Update -> 202 {'ok': True, 'data': {'accepted': True, 'completed': False}, 'message': 'Frissítés elindult – az állapot a kártyán követhető'}
|
||||
11:11:51 + 0.0s phase=backing-up label=Biztonsági mentés készül a frissítés előtt… err=None hold=None
|
||||
11:12:08 + 17.5s phase=safety-dump label=Adatbázis pillanatkép… err=None hold=None
|
||||
11:12:10 + 19.5s phase=pulling label=Új verzió letöltése… err=None hold=None
|
||||
11:12:12 + 21.6s phase=failed label=A frissítés nem sikerült err=Az új verzió letöltése nem sikerült, ezért a frissítés elmaradt. Az alkalmazás a korábbi verzióval fut tovább. hold=None
|
||||
11:12:13 drill: nextcloud back to 34.0.4
|
||||
11:12:21 Tier-2 record:
|
||||
null
|
||||
|
||||
11:12:24 layout:
|
||||
== live userdata
|
||||
total 16
|
||||
drwxr-sr-x 4 root 1000 4096 Sep 24 09:11 .
|
||||
drwxrwsr-x 17 root 1000 4096 Sep 24 09:12 ..
|
||||
drwxr-sr-x 3 root root 4096 Sep 24 09:10 appdata
|
||||
drwxr-sr-x 3 root 1000 4096 Sep 24 09:11 backups
|
||||
/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-1-7cb158.txt 37 2026-09-24 09:11 www-data:www-data 644
|
||||
/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-0-e19728.txt 37 2026-09-24 09:11 www-data:www-data 644
|
||||
/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-2-8620d3.txt 37 2026-09-24 09:11 www-data:www-data 644
|
||||
== mirror copies of the seeds
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
11:09:53 [1] made the drive paths this app requires: ['/mnt/felhom-drives/scratch_hdd/userdata/nextcloud']
|
||||
11:09:53 [1] required fields filled beyond DOMAIN/SUBDOMAIN: ['NEXTCLOUD_ADMIN_PASSWORD', 'HDD_PATH']
|
||||
11:09:54 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'}
|
||||
11:11:24 [1] deployed, controller state=running, pinned={'nextcloud': 'nextcloud:34.0.4-apache', 'nextcloud-db': 'mariadb:12.3', 'nextcloud-redis': 'redis:7-alpine'}
|
||||
11:11:33 nextcloud: occ user:add :: The account "drill76bf48" was created successfully Display name set to "drill76bf48"
|
||||
11:11:37 nextcloud: seeded user drill76bf48
|
||||
11:11:44 nextcloud file PUT felhom-seed-0-e19728.txt ok=True
|
||||
11:11:44 nextcloud file PUT felhom-seed-1-7cb158.txt ok=True
|
||||
11:11:45 nextcloud file PUT felhom-seed-2-8620d3.txt ok=True
|
||||
11:11:46 drill: nextcloud 34.0.99-notag (backing-up runs, pull fails)
|
||||
11:11:51 [6] Update -> 202 {'ok': True, 'data': {'accepted': True, 'completed': False}, 'message': 'Frissítés elindult – az állapot a kártyán követhető'}
|
||||
11:11:51 + 0.0s phase=backing-up label=Biztonsági mentés készül a frissítés előtt… err=None hold=None
|
||||
11:12:08 + 17.5s phase=safety-dump label=Adatbázis pillanatkép… err=None hold=None
|
||||
11:12:10 + 19.5s phase=pulling label=Új verzió letöltése… err=None hold=None
|
||||
11:12:12 + 21.6s phase=failed label=A frissítés nem sikerült err=Az új verzió letöltése nem sikerült, ezért a frissítés elmaradt. Az alkalmazás a korábbi verzióval fut tovább. hold=None
|
||||
11:12:13 drill: nextcloud back to 34.0.4
|
||||
11:12:21 Tier-2 record:
|
||||
null
|
||||
|
||||
11:12:24 layout:
|
||||
== live userdata
|
||||
total 16
|
||||
drwxr-sr-x 4 root 1000 4096 Sep 24 09:11 .
|
||||
drwxrwsr-x 17 root 1000 4096 Sep 24 09:12 ..
|
||||
drwxr-sr-x 3 root root 4096 Sep 24 09:10 appdata
|
||||
drwxr-sr-x 3 root 1000 4096 Sep 24 09:11 backups
|
||||
/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-1-7cb158.txt 37 2026-09-24 09:11 www-data:www-data 644
|
||||
/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-0-e19728.txt 37 2026-09-24 09:11 www-data:www-data 644
|
||||
/mnt/felhom-drives/scratch_hdd/userdata/nextcloud/appdata/nextcloud/drill76bf48/files/felhom-seed-2-8620d3.txt 37 2026-09-24 09:11 www-data:www-data 644
|
||||
== mirror copies of the seeds
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
/mnt/felhom-drives/scratch_hdd/userdata/romm/backups/secondary/nextcloud
|
||||
['password_hash', 'language', 'filebrowser_admin_password_enc', 'filebrowser_admin_state', 'filebrowser_admin_decided_at', 'claimed', 'claim_consumed_generation', 'offbox_enlarge_notice_seeded', 'app_update_events_seeded', 'db_validations', 'app_backup', 'storage_paths']
|
||||
app_backup {"enabled": false, "cross_drive": {"enabled": true, "method": "rsync", "destination_path": "/mnt/felhom-drives/scratch_hdd/userdata/romm", "schedule": "daily", "last_run": "2026-09-24T09:12:08Z", "last_status": "ok", "last_success": "2026-09-24T09:12:08Z", "success_tracked": true, "unit_package_date": "2026-09-24T09:12:06Z", "last_duration": "1s", "last_size_human": "1.0 GB"}}
|
||||
2026/09/24 09:11:51 update_guard.go:360: [INFO] [backup] update pre-backup for nextcloud: starting (DB dump → volume dump → unit capture → Tier 2)
|
||||
2026/09/24 09:12:08 tier2.go:425: [INFO] [backup] Tier 2 copied nextcloud → /mnt/felhom-drives/scratch_hdd/userdata/romm/backups/secondary/nextcloud (1.0 GB, 1 leg(s), 1s)
|
||||
2026/09/24 09:12:08 update.go:748: [INFO] [stacks] update nextcloud: precondition met after the backup — Tier 2 (second drive) copy from 2026-09-24T09:12:08Z
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
[
|
||||
{
|
||||
"path": "/mnt/felhom-drives/scratch_hdd",
|
||||
"label": "SCRATCH HDD",
|
||||
"is_default": true,
|
||||
"schedulable": true,
|
||||
"added_at": ""
|
||||
},
|
||||
{
|
||||
"path": "/mnt/felhom-drives/scratch_hdd/userdata/nextcloud",
|
||||
"label": "T\u00e1rhely (nextcloud)",
|
||||
"schedulable": true,
|
||||
"added_at": "2026-09-21T20:07:01Z"
|
||||
},
|
||||
{
|
||||
"path": "/mnt/felhom-drives/scratch_hdd/userdata/paperless-ngx",
|
||||
"label": "T\u00e1rhely (paperless-ngx)",
|
||||
"schedulable": true,
|
||||
"added_at": "2026-09-22T19:49:16Z"
|
||||
},
|
||||
{
|
||||
"path": "/mnt/felhom-drives/scratch_hdd/userdata/romm",
|
||||
"label": "T\u00e1rhely (romm)",
|
||||
"schedulable": true,
|
||||
"added_at": "2026-09-23T06:17:25Z"
|
||||
},
|
||||
{
|
||||
"path": "/mnt/felhom-drives/scratch_hdd/userdata/navidrome",
|
||||
"label": "T\u00e1rhely (navidrome)",
|
||||
"schedulable": true,
|
||||
"added_at": "2026-09-23T20:39:41Z"
|
||||
}
|
||||
]
|
||||
/mnt/felhom-drives/scratch_hdd dev=66304
|
||||
/mnt/felhom-drives/scratch_hdd/userdata/romm dev=66304
|
||||
/mnt/felhom-drives/scratch_hdd/userdata/nextcloud dev=66304
|
||||
/mnt/sys_drive dev=1793
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"A": {
|
||||
"uid": "drill76bf48"
|
||||
},
|
||||
"files": [
|
||||
"felhom-seed-0-e19728.txt",
|
||||
"felhom-seed-1-7cb158.txt",
|
||||
"felhom-seed-2-8620d3.txt"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
== felhom-pve 9201
|
||||
/felhom-controller restarts=0 started=2026-09-24T06:47:21.728320053Z created=2026-09-24T06:47:21.665152934Z status=running
|
||||
/opengist restarts=0 started=2026-09-24T05:27:38.105496867Z created=2026-09-24T05:27:37.999067339Z status=running
|
||||
/filebrowser restarts=0 started=2026-08-10T07:25:54.314481574Z created=2026-08-03T07:17:49.207519124Z status=running
|
||||
/cloudflared restarts=0 started=2026-08-10T07:25:54.305365864Z created=2026-08-03T07:17:27.877738975Z status=running
|
||||
/traefik restarts=0 started=2026-08-10T07:25:54.28252317Z created=2026-08-03T07:17:27.307858169Z status=running
|
||||
== demo-hp 9201
|
||||
/felhom-controller restarts=0 started=2026-09-24T06:47:22.910952917Z created=2026-09-24T06:47:22.851214011Z status=running
|
||||
/romm restarts=1 started=2026-09-24T06:24:01.331965421Z created=2026-09-24T06:23:50.510337353Z status=exited
|
||||
/romm-db restarts=0 started=2026-09-24T06:23:50.582438225Z created=2026-09-24T06:23:50.442471253Z status=running
|
||||
/romm-redis restarts=0 started=2026-09-24T06:23:50.58081926Z created=2026-09-24T06:23:50.44233201Z status=running
|
||||
/privatebin restarts=0 started=2026-09-24T06:23:49.846285628Z created=2026-09-24T06:23:49.719138742Z status=running
|
||||
/paperless-webserver restarts=0 started=2026-09-24T06:23:49.163739687Z created=2026-09-24T06:23:38.372940745Z status=running
|
||||
/paperless-postgres restarts=1 started=2026-09-24T06:23:38.429088097Z created=2026-09-24T06:23:38.320522241Z status=exited
|
||||
/paperless-redis restarts=0 started=2026-09-24T06:23:38.430701893Z created=2026-09-24T06:23:38.319755874Z status=running
|
||||
/opengist restarts=0 started=2026-09-24T06:23:37.721694028Z created=2026-09-24T06:23:37.642376996Z status=running
|
||||
/kimai restarts=0 started=2026-09-24T06:23:37.128258433Z created=2026-09-24T06:23:31.129377733Z status=running
|
||||
/kimai-db restarts=0 started=2026-09-24T06:23:31.359145063Z created=2026-09-24T06:23:30.952715292Z status=running
|
||||
/docmost restarts=0 started=2026-09-24T06:23:30.235452286Z created=2026-09-24T06:23:19.437626933Z status=running
|
||||
/docmost-postgres restarts=0 started=2026-09-24T06:23:19.496388399Z created=2026-09-24T06:23:19.37876085Z status=running
|
||||
/docmost-redis restarts=0 started=2026-09-24T06:23:19.494877226Z created=2026-09-24T06:23:19.37867101Z status=running
|
||||
/calibre-web restarts=0 started=2026-09-24T06:23:18.854429812Z created=2026-09-24T06:23:18.787605278Z status=running
|
||||
/bookstack restarts=0 started=2026-09-24T06:23:18.310109974Z created=2026-09-24T06:23:12.589022372Z status=running
|
||||
/bookstack-db restarts=0 started=2026-09-24T06:23:12.632654803Z created=2026-09-24T06:23:12.549527569Z status=running
|
||||
/bentopdf restarts=0 started=2026-09-24T06:23:12.059267557Z created=2026-09-24T06:23:11.99527037Z status=running
|
||||
/adventurelog restarts=0 started=2026-09-24T06:23:11.508570026Z created=2026-09-24T06:23:05.776216336Z status=running
|
||||
/adventurelog-postgres restarts=0 started=2026-09-24T06:23:05.825336746Z created=2026-09-24T06:23:05.734611021Z status=running
|
||||
/adventurelog-frontend restarts=0 started=2026-09-24T06:23:05.823825062Z created=2026-09-24T06:23:05.734380877Z status=running
|
||||
/filebrowser restarts=0 started=2026-09-01T17:48:22.221360326Z created=2026-09-01T17:48:21.186435353Z status=running
|
||||
/cloudflared restarts=0 started=2026-09-01T17:47:45.039632728Z created=2026-08-21T16:00:39.902342861Z status=running
|
||||
/traefik restarts=0 started=2026-09-01T17:47:45.010434902Z created=2026-08-21T16:00:38.84850712Z status=running
|
||||
== demo-hp 9202
|
||||
/nextcloud restarts=0 started=2026-09-24T09:12:06.474520938Z status=running
|
||||
/nextcloud-db restarts=0 started=2026-09-24T09:12:00.742780654Z status=running
|
||||
/nextcloud-redis restarts=0 started=2026-09-24T09:12:00.739989825Z status=running
|
||||
/felhom-controller restarts=0 started=2026-09-24T09:09:12.245253133Z status=running
|
||||
/privatebin restarts=0 started=2026-09-24T00:30:23.085214239Z status=running
|
||||
/paperless-webserver restarts=0 started=2026-09-24T00:30:21.642402588Z status=running
|
||||
/paperless-postgres restarts=0 started=2026-09-24T00:30:10.920422124Z status=running
|
||||
/paperless-redis restarts=0 started=2026-09-24T00:30:10.917647798Z status=running
|
||||
/gokapi restarts=538 started=2026-09-24T09:20:53.73851955Z status=restarting
|
||||
/filebrowser restarts=0 started=2026-09-23T21:06:20.703331594Z status=running
|
||||
/traefik restarts=0 started=2026-09-23T21:06:20.702072245Z status=running
|
||||
@@ -0,0 +1,43 @@
|
||||
perl: warning: Setting locale failed.
|
||||
perl: warning: Please check that your locale settings:
|
||||
LANGUAGE = (unset),
|
||||
LC_ALL = (unset),
|
||||
LC_CTYPE = "UTF-8",
|
||||
LC_NUMERIC = (unset),
|
||||
LC_COLLATE = (unset),
|
||||
LC_TIME = (unset),
|
||||
LC_MESSAGES = (unset),
|
||||
LC_MONETARY = (unset),
|
||||
LC_ADDRESS = (unset),
|
||||
LC_IDENTIFICATION = (unset),
|
||||
LC_MEASUREMENT = (unset),
|
||||
LC_PAPER = (unset),
|
||||
LC_TELEPHONE = (unset),
|
||||
LC_NAME = (unset),
|
||||
LANG = "en_US.UTF-8"
|
||||
are supported and installed on your system.
|
||||
perl: warning: Falling back to a fallback locale ("en_US.UTF-8").
|
||||
adventurelog Up 3 hours (unhealthy)
|
||||
adventurelog-frontend Up 3 hours (healthy)
|
||||
adventurelog-postgres Up 3 hours (healthy)
|
||||
bentopdf Up 3 hours (healthy)
|
||||
bookstack Up 3 hours (unhealthy)
|
||||
bookstack-db Up 3 hours (healthy)
|
||||
calibre-web Up 3 hours (healthy)
|
||||
cloudflared Up 3 weeks
|
||||
docmost Up 3 hours (healthy)
|
||||
docmost-postgres Up 3 hours (healthy)
|
||||
docmost-redis Up 3 hours (healthy)
|
||||
felhom-controller Up 3 hours (healthy)
|
||||
filebrowser Up 3 weeks (healthy)
|
||||
kimai Up 3 hours (healthy)
|
||||
kimai-db Up 3 hours (healthy)
|
||||
opengist Up 3 hours (healthy)
|
||||
paperless-postgres Up 3 hours (healthy)
|
||||
paperless-redis Up 3 hours (healthy)
|
||||
paperless-webserver Up 3 hours (healthy)
|
||||
privatebin Up 3 hours (healthy)
|
||||
romm Up 3 hours (healthy)
|
||||
romm-db Up 3 hours (healthy)
|
||||
romm-redis Up 3 hours (healthy)
|
||||
traefik Up 3 weeks
|
||||
@@ -0,0 +1,28 @@
|
||||
== felhom-pve 9201
|
||||
/felhom-controller restarts=0 status=running
|
||||
/opengist restarts=0 status=running
|
||||
/filebrowser restarts=0 status=running
|
||||
/cloudflared restarts=0 status=running
|
||||
/traefik restarts=0 status=running
|
||||
== demo-hp 9201
|
||||
/felhom-controller restarts=0 status=running
|
||||
/romm restarts=1 status=exited
|
||||
/romm-db restarts=0 status=running
|
||||
/romm-redis restarts=0 status=running
|
||||
/privatebin restarts=0 status=running
|
||||
/opengist restarts=0 status=running
|
||||
/kimai restarts=0 status=running
|
||||
/kimai-db restarts=0 status=running
|
||||
/docmost restarts=0 status=running
|
||||
/docmost-postgres restarts=0 status=running
|
||||
/docmost-redis restarts=0 status=running
|
||||
/calibre-web restarts=0 status=running
|
||||
/bookstack restarts=0 status=running
|
||||
/bookstack-db restarts=0 status=running
|
||||
/bentopdf restarts=0 status=running
|
||||
/adventurelog restarts=0 status=running
|
||||
/adventurelog-postgres restarts=0 status=running
|
||||
/adventurelog-frontend restarts=0 status=running
|
||||
/filebrowser restarts=0 status=running
|
||||
/cloudflared restarts=0 status=running
|
||||
/traefik restarts=0 status=running
|
||||
@@ -0,0 +1,5 @@
|
||||
cloudflared|Up 6 weeks
|
||||
felhom-controller|Up 3 hours (healthy)
|
||||
filebrowser|Up 6 weeks (healthy)
|
||||
opengist|Up 4 hours (healthy)
|
||||
traefik|Up 6 weeks
|
||||
@@ -0,0 +1,4 @@
|
||||
2026-09-24T09:22:00Z
|
||||
539 restarting 2026-09-24T09:21:53.990300603Z 2026-09-24T09:21:54.185036712Z
|
||||
2026-09-24T09:29:03Z
|
||||
546 restarting 2026-09-24T09:28:55.748746549Z 2026-09-24T09:28:55.945764829Z
|
||||
@@ -0,0 +1,9 @@
|
||||
# PROGRESS — night 2026-09-24 (run 2026-09-24 from 11:07 CEST)
|
||||
|
||||
Step log. Newest at the bottom. Times CEST unless marked Z.
|
||||
|
||||
- 11:07 baselines verified: controller 7c3b3a969475 (v0.268.0), agent d9864a94bf62, felhom.eu 86c4b9a0d8bf (hub 0.122.0), catalog 635c7e68f716. Register 335 rows / 679,393 B.
|
||||
- 11:08 `09` §3 decisions 26–28 recorded.
|
||||
- 11:09 9202 repointed to the drill catalog (controller.yaml saved as .pre-night0924)
|
||||
- 11:20 A1 (whole restore + R-668) and A2 (decision 27) built, tested, red-proofed
|
||||
- 11:34 A3 built: detector (6 restarts/10 min — measured gokapi 1/min), stop+hold+event, Start lifts; red-proofs detector + skips
|
||||
@@ -0,0 +1,12 @@
|
||||
== rule2
|
||||
--- FAIL: TestWhole_TheFourFileRules (0.00s)
|
||||
tier2_whole_test.go:86: live file photos/b.txt ("B edited by the household") is gone or changed — rule 1
|
||||
FAIL
|
||||
== rule4
|
||||
--- FAIL: TestWhole_TheFourFileRules (0.00s)
|
||||
tier2_whole_test.go:89: the older live copy of c.txt is gone — rule 4 must keep it beside as photos/c.txt.felhom-20260924T210000Z; files: [docs/deep/e.txt photos/a.txt photos/b.txt photos/c.txt photos/d.txt photos/only-live.txt]
|
||||
FAIL
|
||||
== rule3
|
||||
--- FAIL: TestWhole_TheFourFileRules (0.00s)
|
||||
tier2_whole_test.go:97: a.txt / e.txt, missing live, were not brought back: "" ""
|
||||
FAIL
|
||||
@@ -0,0 +1,5 @@
|
||||
--- FAIL: TestR668_MissingStoragePathIsNotASecondDrive (0.01s)
|
||||
r668_missing_path_test.go:45: chose the missing path /tmp/TestR668_MissingStoragePathIsNotASecondDrive1761835767/001/drive-gone as the second drive
|
||||
FAIL
|
||||
FAIL gitea.dooplex.hu/admin/felhom-controller/internal/backup 0.014s
|
||||
FAIL
|
||||
@@ -0,0 +1,3 @@
|
||||
--- FAIL: TestR666_HeldWithNoWholeCopyRemovesOnlyKeepingData (0.00s)
|
||||
r666_keep_data_test.go:45: error = "A(z) /mnt/felhom-drives/x tárhely jelenleg nem elérhető — az alkalmazás nem távolítható el, amíg a meghajtó vissza nem csatlakozik.", want the support sentence
|
||||
FAIL
|
||||
@@ -0,0 +1,5 @@
|
||||
--- FAIL: TestD28_StopHoldTellInOrder_AndTheSkips (0.00s)
|
||||
d28_unhealthy_stop_test.go:41: calls = [stop:gokapi hold:gokapi:crash_loop notify:gokapi stop:quiesced hold:quiesced:crash_loop notify:quiesced] — want gokapi stopped, held, told; the quiesced, busy and held apps untouched
|
||||
FAIL
|
||||
FAIL gitea.dooplex.hu/admin/felhom-controller/cmd/controller 0.008s
|
||||
FAIL
|
||||
@@ -0,0 +1,5 @@
|
||||
--- FAIL: TestAppStoppedUnhealthyIsAHouseholdEvent (0.00s)
|
||||
chaosnight_events_test.go:102: app_stopped_unhealthy must be in allowedEventTypes — the controller's push would 400
|
||||
FAIL
|
||||
FAIL gitea.dooplex.hu/admin/felhom-hub/internal/api 0.019s
|
||||
FAIL
|
||||
@@ -0,0 +1,8 @@
|
||||
--- FAIL: TestR667_TheMeasuredCrashLoopTrips (0.00s)
|
||||
unhealthy_test.go:34: gokapi's loop did not trip: kind="" n=0
|
||||
--- FAIL: TestR667_ObserveSumsPerAppAndSkipsUpdating (0.00s)
|
||||
unhealthy_test.go:97: verdicts = [], want exactly gokapi's crash loop
|
||||
FAIL
|
||||
FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.011s
|
||||
FAIL
|
||||
ok gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.008s
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,34 @@
|
||||
"""Nextcloud FILES through the front door (WebDAV as the seeded user) — the household's own route."""
|
||||
import secrets
|
||||
|
||||
|
||||
def put(w, sub, t, name, body):
|
||||
rc, code, out = w.app_curl(sub, f"/remote.php/dav/files/{t['uid']}/{name}", "-u", f"{t['uid']}:{t['pw']}",
|
||||
"-H", "Content-Type: text/plain", method="PUT", data=body)
|
||||
return code in ("201", "204")
|
||||
|
||||
|
||||
def get(w, sub, t, name):
|
||||
rc, code, out = w.app_curl(sub, f"/remote.php/dav/files/{t['uid']}/{name}", "-u", f"{t['uid']}:{t['pw']}")
|
||||
return code, out
|
||||
|
||||
|
||||
def seed_files(w, sub, t, n=3, say=print):
|
||||
files = {}
|
||||
for i in range(n):
|
||||
name, body = f"felhom-seed-{i}-{secrets.token_hex(3)}.txt", "seed " + secrets.token_hex(16)
|
||||
ok = put(w, sub, t, name, body)
|
||||
say(f" nextcloud file PUT {name} ok={ok}")
|
||||
if ok:
|
||||
files[name] = body
|
||||
return files
|
||||
|
||||
|
||||
def verify_files(w, sub, t, files, say=print):
|
||||
res = {}
|
||||
for name, body in files.items():
|
||||
code, out = get(w, sub, t, name)
|
||||
res[name] = (code == "200" and out == body)
|
||||
neg_code, _ = get(w, sub, t, "felhom-never-" + secrets.token_hex(4) + ".txt")
|
||||
say(f" nextcloud files read back: {sum(res.values())}/{len(res)} (negative control http={neg_code})")
|
||||
return res, neg_code
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Point guest 9202 at the drill catalog (and a 90 s health timeout), or restore the saved config.
|
||||
|
||||
`09` §6.5: `git.repo_url` alone is INERT (R-615) — the cache dir must go too. The saved copy is
|
||||
`controller.yaml.pre-night0924` (NOT the older `.pre-28`, which a restore must never pick up).
|
||||
"""
|
||||
import re, sys, io
|
||||
sys.path.insert(0, '.')
|
||||
import walk as w
|
||||
|
||||
VOL = "/var/lib/docker/volumes/felhom-controller-data/_data"
|
||||
DRILL_REPO = "https://gitea.dooplex.hu/admin/app-catalog-drill.git"
|
||||
|
||||
|
||||
def creds():
|
||||
for l in io.open("/home/kisfenyo/.git-credentials").read().strip().split("\n"):
|
||||
m = re.match(r'https://(admin):([^@]+)@gitea\.dooplex\.hu', l)
|
||||
if m:
|
||||
return m.group(1), m.group(2)
|
||||
raise SystemExit("no admin credential")
|
||||
|
||||
|
||||
def to_drill():
|
||||
u, t = creds()
|
||||
print(w.guest(f"""
|
||||
set -e
|
||||
test -f {VOL}/controller.yaml.pre-night0924 || cp -p {VOL}/controller.yaml {VOL}/controller.yaml.pre-night0924
|
||||
python3 - <<'PY'
|
||||
import re
|
||||
p = "{VOL}/controller.yaml"
|
||||
s = open(p).read()
|
||||
s = re.sub(r'(^\\s+repo_url: ).*$', r'\\g<1>{DRILL_REPO}', s, count=1, flags=re.M)
|
||||
s = re.sub(r'(^git:(?:\\n\\s+.*)*?\\n\\s+token: ).*$', r'\\g<1>"{t}"', s, count=1, flags=re.M)
|
||||
s = re.sub(r'(^git:(?:\\n\\s+.*)*?\\n\\s+username: ).*$', r'\\g<1>"{u}"', s, count=1, flags=re.M)
|
||||
if not re.search(r'^update:', s, re.M):
|
||||
s += "update:\\n health_timeout: 90s\\n"
|
||||
open(p, "w").write(s)
|
||||
PY
|
||||
rm -rf {VOL}/catalog-cache {VOL}/data/catalog-cache
|
||||
docker restart felhom-controller >/dev/null
|
||||
sleep 15
|
||||
grep -A6 '^git:' {VOL}/controller.yaml | sed 's/token:.*/token: <redacted>/'
|
||||
grep -A2 '^update:' {VOL}/controller.yaml
|
||||
"""))
|
||||
|
||||
|
||||
def restore():
|
||||
print(w.guest(f"""
|
||||
set -e
|
||||
cp -p {VOL}/controller.yaml.pre-night0924 {VOL}/controller.yaml
|
||||
rm -rf {VOL}/catalog-cache {VOL}/data/catalog-cache
|
||||
docker restart felhom-controller >/dev/null
|
||||
sleep 15
|
||||
grep -A6 '^git:' {VOL}/controller.yaml | sed 's/token:.*/token: <redacted>/'
|
||||
grep -c '^update:' {VOL}/controller.yaml || true
|
||||
"""))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
to_drill() if sys.argv[1] == "drill" else restore()
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A1 spike: what does nextcloud's Tier-2 mirror hold, file by file, and what would a whole restore need?
|
||||
nextcloud installed from the drill (= live head), a user + 3 files through WebDAV, then an update whose tag
|
||||
does not exist: its backing-up leg runs (unit + Tier 2), the pull fails, nothing moves."""
|
||||
import json, re, sys, time
|
||||
sys.path.insert(0, ".")
|
||||
import walk as w, fixtures as fx, ncfiles as nf
|
||||
F, SUB, APP = fx.Nextcloud(), "cloud-a1", "nextcloud"
|
||||
CAT = f"{w.DRILL}/templates/{APP}"
|
||||
out = {}
|
||||
w.login()
|
||||
w.sync_rescan()
|
||||
out["deployed"] = w.deploy(APP, SUB)
|
||||
A = F.seed(w, SUB, w.say)
|
||||
files = nf.seed_files(w, SUB, A, 3, w.say)
|
||||
out["files_seeded"] = len(files)
|
||||
|
||||
|
||||
def drill(edit, msg):
|
||||
import fcntl
|
||||
with open(f"{w.SC}/drill.lock", "w") as lk:
|
||||
fcntl.flock(lk, fcntl.LOCK_EX)
|
||||
w.sh(["git", "-C", w.DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120)
|
||||
edit()
|
||||
w.sh(["git", "-C", w.DRILL, "commit", "-qam", "NIGHT-A1 " + msg])
|
||||
w.sh(["git", "-C", w.DRILL, "push", "-q", "origin", "main"], timeout=120)
|
||||
w.say("drill: " + msg)
|
||||
|
||||
|
||||
orig = open(f"{CAT}/docker-compose.yml").read()
|
||||
drill(lambda: open(f"{CAT}/docker-compose.yml", "w").write(orig.replace("nextcloud:34.0.4-apache", "nextcloud:34.0.99-notag")), "nextcloud 34.0.99-notag (backing-up runs, pull fails)")
|
||||
w.sync_rescan(APP, "nextcloud:34.0.99-notag", tries=60)
|
||||
out["press"] = w.press_update(APP)
|
||||
drill(lambda: open(f"{CAT}/docker-compose.yml", "w").write(orig), "nextcloud back to 34.0.4")
|
||||
w.sync_rescan()
|
||||
cfg = w.guest("python3 - <<'PY'\nimport json\nd=json.load(open('/var/lib/docker/volumes/felhom-controller-data/_data/data/settings.json'))\nprint(json.dumps((d.get('cross_drive') or d.get('crossdrive') or {}).get('nextcloud'),indent=1))\nPY")
|
||||
out["tier2_record"] = cfg
|
||||
w.say("Tier-2 record:\n" + cfg)
|
||||
lay = w.guest("""for b in /mnt/sys_drive/felhom-data/backups/secondary/nextcloud /mnt/felhom-drives/scratch_hdd/backups/secondary/nextcloud; do [ -d $b ] || continue; echo "== $b"; ls -la $b; du -sh $b/* 2>/dev/null; find $b -maxdepth 4 -type d | head -30; done
|
||||
echo "== live userdata"; ls -la /mnt/felhom-drives/scratch_hdd/userdata/nextcloud 2>&1 | head; find /mnt/felhom-drives/scratch_hdd/userdata/nextcloud -name 'felhom-seed-*' -printf '%p %s %TY-%Tm-%Td %TH:%TM %u:%g %m\\n' 2>/dev/null
|
||||
echo "== mirror copies of the seeds"; find /mnt/sys_drive/felhom-data/backups/secondary/nextcloud /mnt/felhom-drives/scratch_hdd/backups/secondary/nextcloud -name 'felhom-seed-*' -printf '%p %s %TY-%Tm-%Td %TH:%TM %u:%g %m\\n' 2>/dev/null""")
|
||||
out["layout"] = lay
|
||||
w.say("layout:\n" + lay)
|
||||
json.dump({"A": {"uid": A["uid"]}, "files": list(files)}, open("../A1/spike-state.json", "w"), indent=2)
|
||||
json.dump(out, open("../A1/00-spike.json", "w"), indent=2, ensure_ascii=False, default=str)
|
||||
open("../A1/00-spike.log", "w").write("\n".join(w.LOG) + "\n")
|
||||
json.dump({"A": A, "files": files}, open(f"{w.SC}/a1-secret-state.json", "w"))
|
||||
@@ -0,0 +1,507 @@
|
||||
#!/usr/bin/env python3
|
||||
"""walk.py — ONE app's full update walk on guest 9202, through the product's own endpoints.
|
||||
|
||||
EVIDENCE, NOT PRODUCT. It presses exactly the buttons a person presses:
|
||||
POST /api/stacks/<n>/deploy · POST /api/sync · POST /api/stacks/rescan
|
||||
POST /api/stacks/<n>/update · POST /api/stacks/<n>/remove
|
||||
and reads GET /api/stacks/<n>. No controller code exists for it.
|
||||
|
||||
The walk, per `09` §6.4 and the update-night brief §4:
|
||||
1 deploy from the DRILL catalog at the LIVE pin
|
||||
2 seed through the app's OWN front door (R-156: never a volume, never SQL)
|
||||
3 read the seed back <- control C1; a fixture that cannot prove itself proves nothing
|
||||
4 „Mentés most"
|
||||
5 commit the real one-step bump to the DRILL repo, sync, rescan, read the badge in BOTH languages
|
||||
6 press the guarded Update, record every phase with timestamps
|
||||
7 read the seed back through the front door
|
||||
8 the four version observables side by side
|
||||
9 write the verdict record in `09`'s JSON shape
|
||||
|
||||
`inconclusive` is a first-class verdict and is NEVER collapsed into `failed`.
|
||||
"""
|
||||
import argparse, json, os, re, subprocess, sys, time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
SC = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/25947a06-40b7-43dd-8334-2519e02142da/scratchpad"
|
||||
EV = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/audits/night-2026-09-24"
|
||||
DRILL = "/mnt/5_hdd/felhom.eu/drill/app-catalog-drill"
|
||||
# GUEST=9201 selects demo-hp's hub-enabled guest (the mail proof); default 9202, the scratch guest.
|
||||
GUEST = os.environ.get("GUEST", "9202")
|
||||
BASE = {"9202": "https://192.168.0.114", "9201": "https://192.168.0.138"}[GUEST]
|
||||
DOMAIN = os.environ.get("DOMAIN", "enkisfelhom.hu")
|
||||
HOSTHDR = f"Host: felhom.{DOMAIN}"
|
||||
HP = "demo-hp"
|
||||
|
||||
LOG = []
|
||||
|
||||
|
||||
def say(*a):
|
||||
line = " ".join(str(x) for x in a)
|
||||
ts = datetime.now().strftime("%H:%M:%S")
|
||||
print(f"{ts} {line}", flush=True)
|
||||
LOG.append(f"{ts} {line}")
|
||||
|
||||
|
||||
def sh(args, timeout=300, inp=None):
|
||||
try:
|
||||
return subprocess.run(args, capture_output=True, text=True, timeout=timeout, input=inp)
|
||||
except (subprocess.TimeoutExpired, OSError) as e:
|
||||
return subprocess.CompletedProcess(args, 124, "", f"{e}")
|
||||
|
||||
|
||||
def guest(script, timeout=600):
|
||||
"""Run a bash script inside guest 9202. Piped as a file — never as an argument (quoting)."""
|
||||
# ONE TEMP FILE PER CALL (night 2026-09-23): the shared /tmp/w<guest>.sh swapped scripts under
|
||||
# two concurrent walks (memory: guest-helper-shares-one-tmp-file).
|
||||
import secrets as _s
|
||||
t = f"/tmp/w{GUEST}-{os.getpid()}-{_s.token_hex(4)}.sh"
|
||||
r = sh(["ssh", "-o", "ConnectTimeout=20", "-o", "StrictHostKeyChecking=accept-new", HP,
|
||||
f"export LC_ALL=C; cat > {t}; pct push {GUEST} {t} {t} >/dev/null 2>&1; "
|
||||
f"pct exec {GUEST} -- bash {t}; pct exec {GUEST} -- rm -f {t}; rm -f {t}"],
|
||||
timeout=timeout, inp=script)
|
||||
return r.stdout or ""
|
||||
|
||||
|
||||
def login():
|
||||
pw = open(f"{SC}/.ctlpw").read().strip()
|
||||
sh(["curl", "-sk", "-D", f"{SC}/hdr{os.getpid()}.txt", "-o", "/dev/null", "-H", HOSTHDR,
|
||||
"-X", "POST", "--data-urlencode", f"password={pw}", f"{BASE}/login"])
|
||||
h = open(f"{SC}/hdr{os.getpid()}.txt").read()
|
||||
m = re.search(r"felhom_session=[A-Za-z0-9._-]+", h, re.I)
|
||||
if not m:
|
||||
sys.exit("login failed: no session cookie")
|
||||
open(f"{SC}/sess{os.getpid()}.txt", "w").write(m.group(0))
|
||||
r = sh(["curl", "-sk", "-L", "-H", HOSTHDR, "-H", f"Cookie: {m.group(0)}", f"{BASE}/"])
|
||||
c = re.search(r'<meta name="csrf-token" content="([^"]+)"', r.stdout or "")
|
||||
if not c:
|
||||
sys.exit("login failed: no csrf token")
|
||||
open(f"{SC}/csrf{os.getpid()}.txt", "w").write(c.group(1))
|
||||
|
||||
|
||||
def ctl(method, path, data=None, raw=False, tries=2):
|
||||
"""One controller API call. Re-logs in once on a 302/401 — the controller's session store is
|
||||
in memory, so any controller restart during the night invalidates it silently."""
|
||||
for attempt in range(tries):
|
||||
sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip()
|
||||
csrf = open(f"{SC}/csrf{os.getpid()}.txt").read().strip()
|
||||
args = ["curl", "-sk", "-H", HOSTHDR, "-H", f"Cookie: {sess}", "-w", "\n%{http_code}"]
|
||||
if method != "GET":
|
||||
args += ["-H", f"X-CSRF-Token: {csrf}", "-H", "Content-Type: application/json",
|
||||
"-X", method]
|
||||
if data is not None:
|
||||
args += ["--data", json.dumps(data)]
|
||||
args.append(f"{BASE}{path}")
|
||||
r = sh(args)
|
||||
body, _, code = (r.stdout or "").rpartition("\n")
|
||||
if code.strip() in ("302", "401") and attempt + 1 < tries:
|
||||
login()
|
||||
continue
|
||||
if raw:
|
||||
return code.strip(), body
|
||||
try:
|
||||
return code.strip(), json.loads(body)
|
||||
except Exception:
|
||||
return code.strip(), {"_raw": body[:600]}
|
||||
return code.strip(), {"_raw": body[:600]}
|
||||
|
||||
|
||||
def page(path):
|
||||
sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip()
|
||||
r = sh(["curl", "-sk", "-H", HOSTHDR, "-H", f"Cookie: {sess}", f"{BASE}{path}"])
|
||||
return r.stdout or ""
|
||||
|
||||
|
||||
def app_curl(sub, path, *extra, method=None, data=None, timeout=45):
|
||||
"""A call to the APP's own front door on 9202 — the household's route, not ours."""
|
||||
args = ["curl", "-sSk", "--max-time", str(timeout), "-H", f"Host: {sub}.{DOMAIN}",
|
||||
"-w", "\n%{http_code}"]
|
||||
if method:
|
||||
args += ["-X", method]
|
||||
if data is not None:
|
||||
args += ["--data-binary", "@-"]
|
||||
args += list(extra) + [f"{BASE}{path}"]
|
||||
r = sh(args, timeout=timeout + 30, inp=data)
|
||||
body, _, code = (r.stdout or "").rpartition("\n")
|
||||
return r.returncode, code.strip(), body
|
||||
|
||||
|
||||
def stack(name):
|
||||
_, d = ctl("GET", f"/api/stacks/{name}")
|
||||
return (d.get("data") or {}) if isinstance(d, dict) else {}
|
||||
|
||||
|
||||
def wait_app(sub, path="/", want=("200", "302", "303", "401", "403"), tries=60, delay=5):
|
||||
"""Settling says the container runs; this says the APP answers. Not the same thing."""
|
||||
last = None
|
||||
for _ in range(tries):
|
||||
rc, code, _ = app_curl(sub, path, timeout=15)
|
||||
last = (rc, code)
|
||||
if rc == 0 and code in want:
|
||||
return True
|
||||
time.sleep(delay)
|
||||
say(f" app never answered on {sub}{path} (last rc={last[0]} code={last[1]})")
|
||||
return False
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ the walk
|
||||
|
||||
|
||||
DRIVE = "/mnt/felhom-drives/scratch_hdd/userdata"
|
||||
|
||||
# What THIS run generated for a deploy, per app. Deploy secrets are ENCRYPTED AT REST in
|
||||
# `app.yaml` (`ENC:…`), which is right and which means a fixture cannot read an app's admin
|
||||
# password back off the box — the household sees it once. So the value the harness itself
|
||||
# generated is kept here for the life of the run, and nowhere else.
|
||||
GENERATED = {}
|
||||
|
||||
|
||||
def deploy_values(name, sub):
|
||||
"""Fill EVERY required deploy field the way the wizard would, by asking the box what this app
|
||||
asks for — `GET /api/stacks/<n>/deploy-fields` — instead of assuming DOMAIN+SUBDOMAIN.
|
||||
|
||||
Measured 2026-09-21: three apps in one batch refused at the deploy with a correct 400 because
|
||||
a required field was absent — `HDD_PATH` (navidrome, audiobookshelf) and an admin password
|
||||
(grafana). The refusals happen BEFORE anything is created (`deploy.go:324`), which is the only
|
||||
reason this was safe to discover by running it (live-probes rule).
|
||||
|
||||
A `path` field must name a directory that ALREADY EXISTS (`deploy.go:330`), so one is made on
|
||||
the scratch drive first — the same act the drive browser performs for a household.
|
||||
"""
|
||||
code, d = ctl("GET", f"/api/stacks/{name}/deploy-fields")
|
||||
fields = (((d.get("data") or {}).get("metadata") or {}).get("deploy_fields")) or []
|
||||
values = {"DOMAIN": DOMAIN, "SUBDOMAIN": sub}
|
||||
made = []
|
||||
for f in fields:
|
||||
ev, ty = f.get("env_var"), f.get("type")
|
||||
if ev in values:
|
||||
continue
|
||||
# `type: password` is MANDATORY whatever `required` says — `deploy.go:305-312` refuses
|
||||
# when the caller sends none, deliberately ("the user needs to know their password"),
|
||||
# while `.felhom.yml` declares `required: false` and the API serves that verbatim. A
|
||||
# caller that trusts the contract gets a 400. Measured tonight on grafana; filed.
|
||||
if not f.get("required") and ty != "password":
|
||||
continue # the controller generates the optional secrets itself
|
||||
if ty == "path":
|
||||
p = f"{DRIVE}/{name}"
|
||||
values[ev] = p
|
||||
made.append(p)
|
||||
elif ty in ("secret", "password"):
|
||||
import secrets as _s
|
||||
values[ev] = "Drill-" + _s.token_hex(12)
|
||||
GENERATED.setdefault(name, {})[ev] = values[ev]
|
||||
elif f.get("default"):
|
||||
values[ev] = f["default"]
|
||||
else:
|
||||
values[ev] = f"drill-{name}"
|
||||
if made:
|
||||
guest("mkdir -p " + " ".join(made) + "; ls -ld " + " ".join(made))
|
||||
say(f" [1] made the drive paths this app requires: {made}")
|
||||
extra = [k for k in values if k not in ("DOMAIN", "SUBDOMAIN")]
|
||||
if extra:
|
||||
say(f" [1] required fields filled beyond DOMAIN/SUBDOMAIN: {extra}")
|
||||
return values
|
||||
|
||||
|
||||
def deploy(name, sub, extra_values=None):
|
||||
st = stack(name)
|
||||
if st.get("deployed"):
|
||||
say(f" [1] {name} already deployed — reusing")
|
||||
return True
|
||||
values = deploy_values(name, sub)
|
||||
if extra_values:
|
||||
values.update(extra_values)
|
||||
code, d = ctl("POST", f"/api/stacks/{name}/deploy", {"values": values})
|
||||
say(f" [1] deploy -> {code} {str(d)[:120]}")
|
||||
if code != "202":
|
||||
return False
|
||||
# WAIT FOR `deployed`, NOT FOR `running`. Measured 2026-09-21 on tandoor: docker reported the
|
||||
# container `healthy` while the controller's own state read `unhealthy` — a gate on `running`
|
||||
# alone therefore times out on an app that is up. The state is RECORDED rather than required;
|
||||
# the real gate is the fixture's own `wait_app`, which asks whether the APP answers.
|
||||
seen = None
|
||||
for _ in range(90):
|
||||
time.sleep(5)
|
||||
st = stack(name)
|
||||
seen = st.get("state")
|
||||
# `deployed` alone is NOT enough and `state` alone is NOT right. Measured 2026-09-21:
|
||||
# tandoor reads `unhealthy` while serving (R-618), so gating on "running" hangs; and romm
|
||||
# read `deployed=True, state=degraded, pinned_images=None` twenty seconds in, i.e. the
|
||||
# deploy had not finished writing app.yaml. The PIN is the deploy's own completion mark
|
||||
# (`runComposeDeploy` writes it), so that is what to wait for.
|
||||
pins = (st.get("app_config") or {}).get("pinned_images")
|
||||
if st.get("deployed") and pins and seen in ("running", "unhealthy", "degraded"):
|
||||
say(f" [1] deployed, controller state={seen}, "
|
||||
f"pinned={(st.get('app_config') or {}).get('pinned_images')}")
|
||||
if seen != "running":
|
||||
say(f" [1] NOTE: the controller's own state is {seen!r}, not 'running' — recorded, "
|
||||
f"not treated as a failure; the fixture's front-door wait is the real gate")
|
||||
return True
|
||||
say(f" [1] never became deployed (last controller state={seen!r})")
|
||||
return False
|
||||
|
||||
|
||||
def backup_now(name):
|
||||
"""R-648 (2026-09-23): NO whole-box „Mentés most" from a drill, ever.
|
||||
|
||||
`POST /api/backup/run` is the only backup endpoint and it is WHOLE-BOX: on 9201 it stopped and
|
||||
restarted 9 of 10 standing apps twice, and on 9202 it broke a deploy in flight (R-634). The product
|
||||
has NO per-app backup endpoint (router.go: /backup/run, /backup/tier2 only); the per-app backup
|
||||
exists only inside the guarded update, whose `backing-up` phase calls RunAppBackupNow for the one
|
||||
app. So this presses nothing: the update takes the throwaway app's own backup, and says so in its
|
||||
phase list. A seed written "after the backup" is therefore written before the update's own backup
|
||||
— the undo's last-second copy is still the one that must bring it back."""
|
||||
say(f" [4] backup press SKIPPED for {name} (R-648: whole-box only; the update's backing-up phase backs up {name} alone)")
|
||||
return None
|
||||
|
||||
def drill_bump(app, frm, to, service_hint=None):
|
||||
"""Serialised across concurrent walks: one git working tree, one lock."""
|
||||
import fcntl
|
||||
with open(f"{SC}/drill.lock", "w") as lk:
|
||||
fcntl.flock(lk, fcntl.LOCK_EX)
|
||||
sh(["git", "-C", DRILL, "pull", "-q", "--rebase", "origin", "main"], timeout=120)
|
||||
return _drill_bump(app, frm, to, service_hint)
|
||||
|
||||
|
||||
def _drill_bump(app, frm, to, service_hint=None):
|
||||
"""Commit the edge to the DRILL repo. catalog_since set by hand (the drill repo has no gates).
|
||||
|
||||
`frm`/`to` may be comma-separated lists of the SAME length: an app whose own version lives in
|
||||
two images (adventurelog's backend and frontend) moves both in one edge, while its engine
|
||||
sidecar stays where it is — `09` §3b Q3's rule is per SERVICE, and an app-half edge must move
|
||||
every service that carries the app's own version and no others.
|
||||
"""
|
||||
comp = f"{DRILL}/templates/{app}/docker-compose.yml"
|
||||
fy = f"{DRILL}/templates/{app}/.felhom.yml"
|
||||
s = open(comp).read()
|
||||
froms = [x.strip() for x in frm.split(",") if x.strip()]
|
||||
tos = [x.strip() for x in to.split(",") if x.strip()]
|
||||
if len(froms) != len(tos):
|
||||
say(f" [5] from/to lists differ in length: {froms} vs {tos}")
|
||||
return None
|
||||
for f1, t1 in zip(froms, tos):
|
||||
if f"image: {f1}" not in s:
|
||||
say(f" [5] FROM ref not found in compose: {f1}")
|
||||
return None
|
||||
s = s.replace(f"image: {f1}", f"image: {t1}")
|
||||
open(comp, "w").write(s)
|
||||
f = open(fy).read()
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
f = re.sub(r'^catalog_since:.*$', f'catalog_since: "{today}"', f, count=1, flags=re.M)
|
||||
open(fy, "w").write(f)
|
||||
sh(["git", "-C", DRILL, "add", "-A"])
|
||||
sh(["git", "-C", DRILL, "commit", "-q", "-m", f"DRILL {app}: {frm} -> {to}"])
|
||||
r = sh(["git", "-C", DRILL, "push", "-q", "origin", "main"], timeout=120)
|
||||
h = sh(["git", "-C", DRILL, "rev-parse", "--short=12", "HEAD"]).stdout.strip()
|
||||
say(f" [5] drill commit {h}: {app} {frm} -> {to} (push rc={r.returncode})")
|
||||
return h
|
||||
|
||||
|
||||
def sync_rescan(expect_app=None, expect_ref=None, tries=12, delay=5):
|
||||
"""Sync, rescan, and — when told what to expect — WAIT FOR THE BADGE TO CATCH UP.
|
||||
|
||||
R-607: `POST /api/sync` answers "nincs valtozas" while the catalog HAS moved, and
|
||||
`catalog_images` stays stale until a separate rescan. Tonight showed the rescan alone is not
|
||||
enough either: mealie's badge read "Naprakesz" seconds after its bump was pushed, and the
|
||||
Update that followed moved nothing and still reported "Frissitve". So when the caller knows
|
||||
which reference should appear, this polls for it and SAYS HOW LONG IT TOOK — which is the
|
||||
NUMBER R-607 asks for and has never had.
|
||||
"""
|
||||
t0 = time.time()
|
||||
ctl("POST", "/api/sync")
|
||||
time.sleep(2)
|
||||
ctl("POST", "/api/stacks/rescan")
|
||||
time.sleep(2)
|
||||
if not expect_app or not expect_ref:
|
||||
return None
|
||||
for i in range(tries):
|
||||
cat = stack(expect_app).get("catalog_images") or {}
|
||||
if expect_ref in cat.values():
|
||||
waited = round(time.time() - t0, 1)
|
||||
if i:
|
||||
say(f" [sync] the badge needed {waited}s and {i+1} sync+rescan rounds to catch up "
|
||||
f"to {expect_ref} — R-607's window, measured")
|
||||
return waited
|
||||
time.sleep(delay)
|
||||
ctl("POST", "/api/sync")
|
||||
time.sleep(1)
|
||||
ctl("POST", "/api/stacks/rescan")
|
||||
say(f" [sync] the badge NEVER caught up to {expect_ref} in {round(time.time()-t0,1)}s — "
|
||||
f"catalog_images = {stack(expect_app).get('catalog_images')}")
|
||||
return None
|
||||
|
||||
|
||||
def badges(name):
|
||||
out = {}
|
||||
for lang, suffix in (("hu", ""), ("en", "?lang=en")):
|
||||
h = page(f"/apps/{name}{suffix}")
|
||||
m = re.findall(r'<span class="tag tag-[^"]*"[^>]*title="([^"]*)"[^>]*>([^<]*)<', h)
|
||||
out[lang] = [{"title": a.strip(), "text": b.strip()} for a, b in m][:3]
|
||||
return out
|
||||
|
||||
|
||||
def press_update(name, poll=1.0, cap_s=1800):
|
||||
code, d = ctl("POST", f"/api/stacks/{name}/update")
|
||||
say(f" [6] Update -> {code} {str(d)[:220]}")
|
||||
if code not in ("202", "200"):
|
||||
return {"accepted": False, "http": code, "refusal": d, "phases": [], "duration_s": 0}
|
||||
phases, seen, t0 = [], None, time.time()
|
||||
while time.time() - t0 < cap_s:
|
||||
st = stack(name)
|
||||
ph = st.get("update_phase")
|
||||
if ph != seen:
|
||||
seen = ph
|
||||
rec = {"t": round(time.time() - t0, 1), "phase": ph,
|
||||
"label": st.get("update_phase_label"), "updating": st.get("updating"),
|
||||
"error": st.get("update_error"), "hold": st.get("hold_reason")}
|
||||
phases.append(rec)
|
||||
say(f" +{rec['t']:>6.1f}s phase={ph} label={rec['label']} "
|
||||
f"err={rec['error']} hold={rec['hold']}")
|
||||
if not st.get("updating") and ph in ("done", "failed", "undone", None) and time.time() - t0 > 3:
|
||||
break
|
||||
time.sleep(poll)
|
||||
st = stack(name)
|
||||
return {"accepted": True, "http": code, "phases": phases,
|
||||
"duration_s": round(time.time() - t0, 1),
|
||||
"final_phase": st.get("update_phase"), "update_error": st.get("update_error"),
|
||||
"hold_reason": st.get("hold_reason"), "state": st.get("state")}
|
||||
|
||||
|
||||
def observables(name):
|
||||
st = stack(name)
|
||||
ac = st.get("app_config") or {}
|
||||
live = guest(f"""
|
||||
grep -E '^\\s+image:' /opt/docker/stacks/{name}/docker-compose.yml 2>/dev/null | sed 's/^ *//'
|
||||
echo '---inspect---'
|
||||
for c in $(docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'); do
|
||||
echo -n "$c "; docker inspect "$c" --format '{{{{.Config.Image}}}} running={{{{.State.Running}}}} restarts={{{{.RestartCount}}}}'
|
||||
done
|
||||
""")
|
||||
a, _, b = live.partition("---inspect---")
|
||||
return {
|
||||
"pinned_images": ac.get("pinned_images"),
|
||||
"installed_images": {k: (v.get("ref") if isinstance(v, dict) else v)
|
||||
for k, v in (ac.get("installed_images") or {}).items()},
|
||||
"catalog_images": st.get("catalog_images"),
|
||||
"live_compose_image_lines": [x for x in a.strip().splitlines() if x.strip()],
|
||||
"docker_inspect": [x for x in b.strip().splitlines() if x.strip()],
|
||||
}
|
||||
|
||||
|
||||
def app_logs(name, lines=400):
|
||||
"""The app's own container log, DECODED. The endpoint answers a JSON envelope whose `logs` is
|
||||
one string with escaped newlines — a scan over the envelope sees a single enormous line and
|
||||
finds nothing, which reads exactly like "the app printed no migration line" and is not. R-96
|
||||
rule 3 in a new place: an absent line is not evidence when the instrument cannot see lines."""
|
||||
code, d = ctl("GET", f"/api/stacks/{name}/logs?lines={lines}")
|
||||
if isinstance(d, dict):
|
||||
data = d.get("data")
|
||||
if isinstance(data, dict) and isinstance(data.get("logs"), str):
|
||||
return data["logs"]
|
||||
if isinstance(d.get("_raw"), str):
|
||||
return d["_raw"]
|
||||
return str(d)
|
||||
|
||||
|
||||
def write_verdict(rec, appdir):
|
||||
os.makedirs(appdir, exist_ok=True)
|
||||
p = os.path.join(appdir, "verdict.json")
|
||||
json.dump(rec, open(p, "w"), indent=2, ensure_ascii=False)
|
||||
say(f" [9] verdict {rec['verdict']} -> {p}")
|
||||
|
||||
|
||||
def remove(name):
|
||||
"""Remove through the PRODUCT, never `docker rm` (live-probes rule). The remove endpoint
|
||||
refuses a running stack — `409 still running` — so the stop is part of the act, not a tidy-up."""
|
||||
c1, d1 = ctl("POST", f"/api/stacks/{name}/stop")
|
||||
say(f" [X] stop -> {c1} {str(d1)[:100]}")
|
||||
for _ in range(24):
|
||||
time.sleep(5)
|
||||
if stack(name).get("state") != "running":
|
||||
break
|
||||
code, d = ctl("POST", f"/api/stacks/{name}/remove",
|
||||
{"remove_hdd_data": True, "remove_backups": True})
|
||||
say(f" [X] remove (with drive data) -> {code} {str(d)[:160]}")
|
||||
if code == "409":
|
||||
# R-442's fail-closed guard: when the storage subsystem cannot RESOLVE the app's drive
|
||||
# path, the removal is REFUSED and the app is kept rather than half-deleted. On guest 9202
|
||||
# `/api/disks` answers `agent not configured`, so every app deployed with an HDD_PATH hits
|
||||
# this. The household's other choice — remove the app, KEEP the data — is accepted, and the
|
||||
# harness takes it, then tidies its own directory by name at teardown.
|
||||
say(" [X] refused because the drive path cannot be resolved (R-442, fail-closed and right)"
|
||||
" — removing the app and KEEPING the drive data instead")
|
||||
code, d = ctl("POST", f"/api/stacks/{name}/remove",
|
||||
{"remove_hdd_data": False, "remove_backups": True})
|
||||
say(f" [X] remove (keeping drive data) -> {code} {str(d)[:160]}")
|
||||
time.sleep(5)
|
||||
st = stack(name)
|
||||
left = guest(f"ls -d /opt/docker/stacks/{name} 2>/dev/null; "
|
||||
f"docker ps -a --filter label=com.docker.compose.project={name} --format '{{{{.Names}}}}'")
|
||||
say(f" [X] after remove: deployed={st.get('deployed')} leftovers={left.strip()!r}")
|
||||
return code
|
||||
|
||||
|
||||
def app_env(name, key):
|
||||
"""Read one deploy value the CUSTOMER was given (e.g. the generated admin password) from the
|
||||
app's own `app.yaml`. This is not seeding — it is how the household logs in; the controller
|
||||
shows them the same value. Data still goes in through the app's own front door."""
|
||||
out = guest(f"grep -E '^\\s*{key}:' /opt/docker/stacks/{name}/app.yaml 2>/dev/null | head -1")
|
||||
if ":" in out:
|
||||
return out.split(":", 1)[1].strip().strip('"').strip("'")
|
||||
return ""
|
||||
|
||||
|
||||
def snapshots(name):
|
||||
"""The restorable copies the backups page offers for this app."""
|
||||
code, d = ctl("GET", f"/api/backup/snapshots?stack={name}")
|
||||
data = d.get("data") if isinstance(d, dict) else None
|
||||
if isinstance(data, dict):
|
||||
for k in ("snapshots", "items", "restore_points"):
|
||||
if isinstance(data.get(k), list):
|
||||
return data[k]
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
|
||||
def restore(name, snapshot_id=None, wait_s=1200):
|
||||
"""The household's own way out: the „Visszaállítás a mentésből" button on the backups page.
|
||||
|
||||
A FORM post, not an API call — `POST /backup/restore` with `_csrf`, `stack_name`,
|
||||
`snapshot_id` — because that is the button the sentence tells them to press.
|
||||
"""
|
||||
snaps = snapshots(name)
|
||||
if snapshot_id is None:
|
||||
if not snaps:
|
||||
say(f" [R] no restorable copy offered for {name}")
|
||||
return {"ok": False, "why": "no snapshot offered", "snapshots": snaps}
|
||||
first = snaps[0]
|
||||
snapshot_id = first.get("id") or first.get("snapshot_id") or first.get("short_id")
|
||||
say(f" [R] restoring {name} from snapshot {snapshot_id!r} (of {len(snaps)} offered)")
|
||||
sess = open(f"{SC}/sess{os.getpid()}.txt").read().strip()
|
||||
csrf = open(f"{SC}/csrf{os.getpid()}.txt").read().strip()
|
||||
r = sh(["curl", "-sk", "-D", "-", "-o", "/dev/null", "-H", HOSTHDR, "-H", f"Cookie: {sess}",
|
||||
"-X", "POST",
|
||||
"--data-urlencode", f"_csrf={csrf}",
|
||||
"--data-urlencode", f"stack_name={name}",
|
||||
"--data-urlencode", f"snapshot_id={snapshot_id}",
|
||||
f"{BASE}/backup/restore"], timeout=180)
|
||||
head = (r.stdout or "").split("\n")[0].strip()
|
||||
loc = [l for l in (r.stdout or "").split("\n") if l.lower().startswith("location:")]
|
||||
say(f" [R] POST /backup/restore -> {head} {loc[:1]}")
|
||||
t0 = time.time()
|
||||
last = None
|
||||
while time.time() - t0 < wait_s:
|
||||
code, d = ctl("GET", "/api/backup/restore-status")
|
||||
dd = d.get("data") or {}
|
||||
cur = (dd.get("running"), dd.get("phase") or dd.get("state"), dd.get("message"))
|
||||
if cur != last:
|
||||
say(f" +{round(time.time()-t0,1):>6.1f}s restore {cur}")
|
||||
last = cur
|
||||
if not dd.get("running", False) and time.time() - t0 > 5:
|
||||
break
|
||||
time.sleep(2)
|
||||
st = stack(name)
|
||||
say(f" [R] after restore: state={st.get('state')} hold={st.get('hold_reason')!r} "
|
||||
f"phase={st.get('update_phase')}")
|
||||
return {"ok": True, "snapshot_id": snapshot_id, "snapshots": snaps,
|
||||
"http": head, "location": loc[:1], "seconds": round(time.time() - t0, 1),
|
||||
"state_after": st.get("state"), "hold_after": st.get("hold_reason"),
|
||||
"observables_after": observables(name)}
|
||||
Reference in New Issue
Block a user