From 650aa0cf99cc3d36ce93f6f5a8b9e4c920994c8c Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 29 Oct 2026 10:12:40 +0100 Subject: [PATCH] Release run 2026-10-29: hub 0.175.0 ceremony + watch, agent 0.168.0 on the three test boxes, controller 0.347.0 walk + local restore + OOM + D62 end to end on Tester 1 (evidence); R-1341 filed; R-1339/R-1340 moved to their sections, headers recounted --- .../audits/d63-2026-10-29/9202-before.txt | 8 ++ .../audits/d63-2026-10-29/9202-rc.txt | 8 ++ .../d63-2026-10-29/agent-delivery-journal.txt | 28 +++++ .../audits/d63-2026-10-29/agent-delivery.txt | 28 +++++ .../audits/d63-2026-10-29/ceremony.log | 17 +++ .../audits/d63-2026-10-29/ceremony.py | 78 +++++++++++++ documentation/audits/d63-2026-10-29/floor.sh | 12 ++ .../audits/d63-2026-10-29/opsign-agent.sh | 12 ++ .../audits/d63-2026-10-29/opsign-agent.txt | 12 ++ .../audits/d63-2026-10-29/p4_live.py | 108 ++++++++++++++++++ .../audits/d63-2026-10-29/perbox-floors.txt | 9 ++ .../audits/d63-2026-10-29/run-ceremony.log | 11 ++ .../audits/d63-2026-10-29/run-p4-backup.log | 13 +++ .../audits/d63-2026-10-29/run-p4-deploy.log | 10 ++ .../audits/d63-2026-10-29/run-p4-oom.log | 21 ++++ .../audits/d63-2026-10-29/run-p4-restore.log | 30 +++++ .../audits/d63-2026-10-29/run-p4-teardown.log | 9 ++ .../audits/d63-2026-10-29/run-p4b-backup.log | 13 +++ .../audits/d63-2026-10-29/run-p4b-d62.log | 17 +++ .../audits/d63-2026-10-29/run-p4b-deploy.log | 10 ++ .../d63-2026-10-29/run-p4b-teardown.log | 9 ++ .../d63-2026-10-29/tester1-after-p4.txt | 19 +++ .../audits/d63-2026-10-29/walk-3boxes.txt | 58 ++++++++++ .../audits/d63-2026-10-29/walk-9202-rc1.txt | 7 ++ .../audits/d63-2026-10-29/walk_release.sh | 16 +++ documentation/backlog/OPEN-ITEMS.md | 11 +- 26 files changed, 569 insertions(+), 5 deletions(-) create mode 100644 documentation/audits/d63-2026-10-29/9202-before.txt create mode 100644 documentation/audits/d63-2026-10-29/9202-rc.txt create mode 100644 documentation/audits/d63-2026-10-29/agent-delivery-journal.txt create mode 100644 documentation/audits/d63-2026-10-29/agent-delivery.txt create mode 100644 documentation/audits/d63-2026-10-29/ceremony.log create mode 100644 documentation/audits/d63-2026-10-29/ceremony.py create mode 100644 documentation/audits/d63-2026-10-29/floor.sh create mode 100644 documentation/audits/d63-2026-10-29/opsign-agent.sh create mode 100644 documentation/audits/d63-2026-10-29/opsign-agent.txt create mode 100644 documentation/audits/d63-2026-10-29/p4_live.py create mode 100644 documentation/audits/d63-2026-10-29/perbox-floors.txt create mode 100644 documentation/audits/d63-2026-10-29/run-ceremony.log create mode 100644 documentation/audits/d63-2026-10-29/run-p4-backup.log create mode 100644 documentation/audits/d63-2026-10-29/run-p4-deploy.log create mode 100644 documentation/audits/d63-2026-10-29/run-p4-oom.log create mode 100644 documentation/audits/d63-2026-10-29/run-p4-restore.log create mode 100644 documentation/audits/d63-2026-10-29/run-p4-teardown.log create mode 100644 documentation/audits/d63-2026-10-29/run-p4b-backup.log create mode 100644 documentation/audits/d63-2026-10-29/run-p4b-d62.log create mode 100644 documentation/audits/d63-2026-10-29/run-p4b-deploy.log create mode 100644 documentation/audits/d63-2026-10-29/run-p4b-teardown.log create mode 100644 documentation/audits/d63-2026-10-29/tester1-after-p4.txt create mode 100644 documentation/audits/d63-2026-10-29/walk-3boxes.txt create mode 100644 documentation/audits/d63-2026-10-29/walk-9202-rc1.txt create mode 100644 documentation/audits/d63-2026-10-29/walk_release.sh diff --git a/documentation/audits/d63-2026-10-29/9202-before.txt b/documentation/audits/d63-2026-10-29/9202-before.txt new file mode 100644 index 00000000..98843b6f --- /dev/null +++ b/documentation/audits/d63-2026-10-29/9202-before.txt @@ -0,0 +1,8 @@ +gitea.dooplex.hu/admin/felhom-controller:0.346.1 +felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.346.1 Up 5 hours (healthy) +filebrowser gtstef/filebrowser:1.5.8-stable Up 5 hours (healthy) +paperless-postgres postgres:18-alpine Up 5 hours (healthy) +paperless-redis redis:7-alpine Up 5 hours (healthy) +paperless-webserver ghcr.io/paperless-ngx/paperless-ngx:2.20.15 Up 5 hours (healthy) +traefik traefik:v3.7.16 Up 5 hours +359 diff --git a/documentation/audits/d63-2026-10-29/9202-rc.txt b/documentation/audits/d63-2026-10-29/9202-rc.txt new file mode 100644 index 00000000..0388ef12 --- /dev/null +++ b/documentation/audits/d63-2026-10-29/9202-rc.txt @@ -0,0 +1,8 @@ +gitea.dooplex.hu/admin/felhom-controller:0.347.0-rc.1 +gitea.dooplex.hu/admin/felhom-controller:0.347.0-rc.1 Up 50 seconds (healthy) +sha256:2a73dd06cb0edba44e83e2a423738ef03ad1f6110faeeab57da9304509592140 +2026/10/29 08:53:39 undo.go:676: [INFO] [stacks] applied-meta backfill (R-646): recorded 0 []; skipped 0 [] — not current with the catalog, their pinned version's .felhom.yml is no longer on the box +2026/10/29 08:53:39 scheduler.go:120: [INFO] [scheduler] Registered periodic job: conversion-copy-release (every 1h0m0s) +2026/10/29 08:53:39 scheduler.go:85: [DEBUG] [scheduler] periodic job registered: name="conversion-copy-release" interval=1h0m0s totalJobs=14 +2026/10/29 08:53:39 scheduler.go:120: [INFO] [scheduler] Registered periodic job: restore-copy-sweep (every 6h0m0s) +2026/10/29 08:53:39 scheduler.go:85: [DEBUG] [scheduler] periodic job registered: name="restore-copy-sweep" interval=6h0m0s totalJobs=15 diff --git a/documentation/audits/d63-2026-10-29/agent-delivery-journal.txt b/documentation/audits/d63-2026-10-29/agent-delivery-journal.txt new file mode 100644 index 00000000..51f360aa --- /dev/null +++ b/documentation/audits/d63-2026-10-29/agent-delivery-journal.txt @@ -0,0 +1,28 @@ +09:04:08 +== root@192.168.0.154 felhom-agent 0.168.0 +Oct 29 09:56:27 felhom felhom-agent[1161]: time=2026-10-29T09:56:27.781+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/selfupda +Oct 29 09:56:28 felhom felhom-agent[1161]: time=2026-10-29T09:56:28.301+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\": {\" +Oct 29 09:56:28 felhom felhom-agent[1161]: time=2026-10-29T09:56:28.302+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=fdb69d53f8afa8d3 op=agent_update +Oct 29 09:56:31 felhom felhom-agent[424223]: time=2026-10-29T09:56:31.398+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0 +Oct 29 09:56:32 felhom felhom-agent[424223]: time=2026-10-29T09:56:32.081+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s +Oct 29 09:57:32 felhom sudo[426210]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit +Oct 29 09:57:32 felhom felhom-selfupdate-guarded[426218]: committed (pending cleared, .prev retained) +Oct 29 09:57:32 felhom felhom-agent[424223]: time=2026-10-29T09:57:32.122+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper="" +== hp felhom-agent 0.168.0 +Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.564+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/selfupd +Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.953+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\": {\ +Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.954+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=63ed679a243bcac2 op=agent_update +Oct 29 09:55:37 demo-hp felhom-agent[961306]: time=2026-10-29T09:55:37.136+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0 +Oct 29 09:55:38 demo-hp felhom-agent[961306]: time=2026-10-29T09:55:38.063+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s +Oct 29 09:56:38 demo-hp sudo[966235]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit +Oct 29 09:56:38 demo-hp felhom-selfupdate-guarded[966243]: committed (pending cleared, .prev retained) +Oct 29 09:56:38 demo-hp felhom-agent[961306]: time=2026-10-29T09:56:38.091+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper="" +== felhom-pve felhom-agent 0.168.0 +Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.145+01:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=95d5b5018bde3b37 op=agent_update key_id=felhom-op-1 nonce=7655701562a +Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.429+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/sel +Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.698+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\" +Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.698+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=95d5b5018bde3b37 op=agent_update +Oct 29 10:03:43 demo-felhom felhom-agent[357627]: time=2026-10-29T10:03:43.578+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0 +Oct 29 10:03:44 demo-felhom felhom-agent[357627]: time=2026-10-29T10:03:44.857+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s + +[exited with code 0] diff --git a/documentation/audits/d63-2026-10-29/agent-delivery.txt b/documentation/audits/d63-2026-10-29/agent-delivery.txt new file mode 100644 index 00000000..c38b74f1 --- /dev/null +++ b/documentation/audits/d63-2026-10-29/agent-delivery.txt @@ -0,0 +1,28 @@ +== root@192.168.0.154 felhom-agent 0.168.0 +Oct 29 09:56:27 felhom felhom-agent[1161]: time=2026-10-29T09:56:27.781+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/selfupda +Oct 29 09:56:28 felhom felhom-agent[1161]: time=2026-10-29T09:56:28.301+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\": {\" +Oct 29 09:56:28 felhom felhom-agent[1161]: time=2026-10-29T09:56:28.302+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=fdb69d53f8afa8d3 op=agent_update +Oct 29 09:56:31 felhom felhom-agent[424223]: time=2026-10-29T09:56:31.398+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0 +Oct 29 09:56:32 felhom felhom-agent[424223]: time=2026-10-29T09:56:32.081+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s +Oct 29 09:57:32 felhom sudo[426210]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit +Oct 29 09:57:32 felhom felhom-selfupdate-guarded[426218]: committed (pending cleared, .prev retained) +Oct 29 09:57:32 felhom felhom-agent[424223]: time=2026-10-29T09:57:32.122+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper="" +== hp felhom-agent 0.168.0 +Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.564+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/selfupd +Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.953+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\": {\ +Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.954+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=63ed679a243bcac2 op=agent_update +Oct 29 09:55:37 demo-hp felhom-agent[961306]: time=2026-10-29T09:55:37.136+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0 +Oct 29 09:55:38 demo-hp felhom-agent[961306]: time=2026-10-29T09:55:38.063+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s +Oct 29 09:56:38 demo-hp sudo[966235]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit +Oct 29 09:56:38 demo-hp felhom-selfupdate-guarded[966243]: committed (pending cleared, .prev retained) +Oct 29 09:56:38 demo-hp felhom-agent[961306]: time=2026-10-29T09:56:38.091+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper="" +== felhom-pve felhom-agent 0.168.0 +Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.145+01:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=95d5b5018bde3b37 op=agent_update key_id=felhom-op-1 nonce=7655701562a +Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.429+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/sel +Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.698+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\" +Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.698+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=95d5b5018bde3b37 op=agent_update +Oct 29 10:03:43 demo-felhom felhom-agent[357627]: time=2026-10-29T10:03:43.578+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0 +Oct 29 10:03:44 demo-felhom felhom-agent[357627]: time=2026-10-29T10:03:44.857+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s +Oct 29 10:04:44 demo-felhom felhom-agent[357627]: time=2026-10-29T10:04:44.877+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper="" +felhom-agent 0.168.0 +1e70196c51346cf10e3f04f51975de5899b5dc6baa5a45ec1d17bc248830d2ca /usr/local/bin/felhom-agent diff --git a/documentation/audits/d63-2026-10-29/ceremony.log b/documentation/audits/d63-2026-10-29/ceremony.log new file mode 100644 index 00000000..8534b1e5 --- /dev/null +++ b/documentation/audits/d63-2026-10-29/ceremony.log @@ -0,0 +1,17 @@ +09:40:09 preflight → 200 {"data": {"agent_supported": true, "escrow_state": "escrowed", "items": [{"id": "pbs_storage_id", "ok": true, "detail": "felhom-pbs"}, {"id": "dr_tier", "ok": true, "detail": "DR tier applied"}, {"id": "age_binary", "ok": true, "detail": "/usr/bin/age"}, {"id": "hub_upload", "ok": true, "detail": "hub upload target configured"}, {"id": "staged_secret", "ok": false, "detail": "no staged secret (informational \u2014 the controller re-stages when offsite is configured)"}, {"id": "sudo_grant", "ok": true, "detail": "sudo grant listed (list-mode)"}], "offbox_configured": true, "ok": true, "stale": +09:40:09 start → 200 {"data": {"job_id": "escrow-1793263209404621564", "phase": "running"}, "error": "", "ok": true} +09:40:12 status → 200 {"data": {"claim_expires_in_sec": 599, "claimable": true, "claimed": false, "detail": "", "entropy_bits": 129.24070185585344, "job_id": "escrow-1793263209404621564", "key_fingerprint": "82:c6:61:ba:ae:36:47:af:0f:dc:17:0b:0d:5c:5d:63:cd:e7:47:e6:fe:9a:14:fe:66:c1:f0:e6:8f:a1:44:3b", "phase": "done", "restic_pw_sealed": true, "uploaded": false}, "error": "", "ok": true} +09:40:12 claim → 200; code received: True (10 words); confirm_required=True positions=[3, 4] +09:40:12 confirm → 200 {"data": {"uploaded": true}, "error": "", "ok": true} +09:52:46 controller escrow lines: +2026/10/29 08:40:09 [INFO] [web] escrow readiness: READY (agent preflight ok) — the recovery-code reminder is shown +2026/10/29 08:40:09 [INFO] [web] escrow ceremony started via wizard (job escrow-1793263209404621564) +2026/10/29 08:40:12 [INFO] [web] escrow recovery code claimed (one-shot; not logged) — its package waits for the typed-back words +2026/10/29 08:40:12 [INFO] [web] escrow recovery code typed back — the new package is on the hub (R-1151) + +stale warning (nem fedi): False | control word Helyre present: False +escrow_state: escrowed +2026/10/29 08:40:09 [INFO] [web] escrow ceremony started via wizard (job escrow-1793263209404621564) +2026/10/29 08:40:12 [INFO] [web] escrow recovery code claimed (one-shot; not logged) — its package waits for the typed-back words +2026/10/29 08:40:12 [INFO] [web] escrow recovery code typed back — the new package is on the hub (R-1151) + diff --git a/documentation/audits/d63-2026-10-29/ceremony.py b/documentation/audits/d63-2026-10-29/ceremony.py new file mode 100644 index 00000000..126e9f6d --- /dev/null +++ b/documentation/audits/d63-2026-10-29/ceremony.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""ceremony.py — one escrow ceremony on Tester 1 through the wizard's own endpoints (hub 0.175.0 check, 2026-10-29). +The recovery code lives in this process's memory only: never printed, never written, dropped at the end. The dashboard +password is read from the 0600 file and handed to curl on stdin (never on a command line).""" +import json, os, subprocess, sys, time + +sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts") +SCR = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/0b76c4d7-9b39-4e4e-869d-d0d2c6006331/scratchpad" +os.environ.setdefault("SC", f"{SCR}/rt-tester1") +os.environ.setdefault("TARGET", "tester-1") +import box_walk as w + +w.login() +sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip() +csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip() +BASE_ARGS = ["curl", "-sk", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-H", f"X-CSRF-Token: {csrf}"] + + +def call(method, path, form_stdin=None, body=None): + args = BASE_ARGS + ["-X", method, "-w", "\n%{http_code}"] + inp = None + if form_stdin is not None: + args += ["--data-urlencode", "password@-"] + inp = form_stdin + if body is not None: + args += ["-H", "Content-Type: application/json", "--data-binary", "@-"] + inp = json.dumps(body) + r = subprocess.run(args + [f"{w.BASE}{path}"], input=inp, capture_output=True, text=True, timeout=120) + out, _, code = r.stdout.rpartition("\n") + try: + return code, json.loads(out) + except Exception: + return code, {"raw": out[:200]} + + +def safe(d): + """A copy with the code removed — the only form that is printed.""" + d = json.loads(json.dumps(d)) + if isinstance(d.get("data"), dict): + d["data"].pop("recovery_code", None) + return d + + +t0 = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) +code, d = call("GET", "/api/escrow/preflight") +w.say(f"preflight → {code} {json.dumps(d)[:600]}") +pw = open(f"{w.SC}/.ctlpw").read().strip() +code, d = call("POST", "/api/escrow/start", form_stdin=pw) +pw = None +w.say(f"start → {code} {json.dumps(safe(d))[:300]}") +if code != "200": + sys.exit(1) +for _ in range(120): + time.sleep(3) + code, d = call("GET", "/api/escrow/status") + ph = (d.get("data") or {}).get("phase") + if ph not in ("running", "queued", "starting", None) or (d.get("data") or {}).get("claimable"): + break +w.say(f"status → {code} {json.dumps(safe(d))[:400]}") +code, d = call("POST", "/api/escrow/claim") +rc = (d.get("data") or {}).get("recovery_code") or "" +pos = (d.get("data") or {}).get("verify_positions") or [] +w.say(f"claim → {code}; code received: {bool(rc)} ({len(rc.split('-'))} words); confirm_required={(d.get('data') or {}).get('confirm_required')} positions={pos}") +if rc and pos: + words = rc.split("-") + picked = [words[pos[0] - 1], words[pos[1] - 1]] + rc, words = None, None + code, d = call("POST", "/api/escrow/confirm", body={"words": picked}) + picked = None + w.say(f"confirm → {code} {json.dumps(safe(d))[:400]}") +rc = None +for _ in range(40): + time.sleep(15) + st = w.guest("docker logs --since " + t0 + " felhom-controller 2>&1 | grep -iE 'escrow' | grep -viE 'recovery_code' | tail -12") + if "escrowed" in st or "auto-confirm" in st.lower(): + break +w.say("controller escrow lines:\n" + st) +open(f"{SCR}/d63/ev/ceremony.log", "a").write("\n".join(w.LOG) + "\n") diff --git a/documentation/audits/d63-2026-10-29/floor.sh b/documentation/audits/d63-2026-10-29/floor.sh new file mode 100644 index 00000000..e55a2928 --- /dev/null +++ b/documentation/audits/d63-2026-10-29/floor.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# floor.sh — one floor save at the hub with CC's +# scoped token (decision 251). The token arrives through `sec run` under the NAME; it goes to curl through a header file +# in $XDG_RUNTIME_DIR (printf is a builtin, so the value is never on a command line) and is never printed. +# Prints the status line and the Location header only: 303 to ?flash=floor_set is success; any other flash is a refusal. +set -euo pipefail +NAME=$1; P=$2; VER=$3; MA=$4 +H=$(mktemp -p "${XDG_RUNTIME_DIR:-/tmp}") +printf "Authorization: Bearer %s\n" "${!NAME}" > "$H" +curl -s -o /dev/null -D - -H @"$H" --data-urlencode "min_controller_version=$VER" --data-urlencode "min_agent=$MA" \ + "http://10.43.52.34:8080$P" | grep -iE "^HTTP|^location" || true +: > "$H"; rm -f "$H" diff --git a/documentation/audits/d63-2026-10-29/opsign-agent.sh b/documentation/audits/d63-2026-10-29/opsign-agent.sh new file mode 100644 index 00000000..dec5b853 --- /dev/null +++ b/documentation/audits/d63-2026-10-29/opsign-agent.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# opsign-agent.sh ... — one signed agent_update job per box, queued at the hub. +# The hub key arrives through `sec run` under the NAME given; its value is never printed. +set -euo pipefail +KEYNAME=$1; VER=$2; SHA=$3; shift 3 +HUBKEY=${!KEYNAME} +D=/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/0b76c4d7-9b39-4e4e-869d-d0d2c6006331/scratchpad/d63 +for H in "$@"; do + echo "== $H" + "$D/felhom-opsign" -op agent_update -host "$H" -key-id felhom-op-1 -key /mnt/5_hdd/felhom.eu/felhom-op-operational \ + -agent-version "$VER" -sha256 "$SHA" -ttl 45m -upload http://10.43.52.34:8080 -hub-key "$HUBKEY" 2>&1 | tail -3 +done diff --git a/documentation/audits/d63-2026-10-29/opsign-agent.txt b/documentation/audits/d63-2026-10-29/opsign-agent.txt new file mode 100644 index 00000000..d3f722b7 --- /dev/null +++ b/documentation/audits/d63-2026-10-29/opsign-agent.txt @@ -0,0 +1,12 @@ +== tester-1-d70be4 +signed: op=agent_update host=tester-1-d70be4 guest="" key_id=felhom-op-1 nonce=d853bac716c5c4fbfb2b566436885b14 expires=2026-10-29T09:37:05Z +{"op_blob_b64":"eyJleHBpcmVzX2F0IjoiMjAyNi0xMC0yOVQwOTozNzowNVoiLCJpc3N1ZWRfYXQiOiIyMDI2LTEwLTI5VDA4OjUyOjA1WiIsImtleV9pZCI6ImZlbGhvbS1vcC0xIiwibm9uY2UiOiJkODUzYmFjNzE2YzVjNGZiZmIyYjU2NjQzNjg4NWIxNCIsIm9wIjoiYWdlbnRfdXBkYXRlIiwicGFyYW1zIjp7InNoYTI1NiI6IjFlNzAxOTZjNTEzNDZjZjEwZTNmMDRmNTE5NzVkZTU4OTliNWRjNmJhYTVhNDVlYzFkMTdiYzI0ODgzMGQyY2EiLCJ2ZXJzaW9uIjoiMC4xNjguMCJ9LCJ0YXJnZXQiOnsiZ3Vlc3RfaWQiOiIiLCJob3N0X2lkIjoidGVzdGVyLTEtZDcwYmU0In19","sig_armored":"-----BEGIN SSH SIGNATURE-----\nU1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgvzPSoI2ADfHbHEAHRCPmujzBoW\nMZnxNr+xYHre2UvD4AAAAMZmVsaG9tLW9wLXYxAAAAAAAAAAZzaGE1MTIAAABTAAAAC3Nz\naC1lZDI1NTE5AAAAQJsggYiigisd0vNWgbqm+IqEMkSJuWqcdR/HvrB1CFdYTALfaT7S8n\nsX2+4JNk/Qn95q08vRge98k/xgoqtuxQ8=\n-----END SSH SIGNATURE-----\n"} +uploaded signed op to the hub jobs queue +== demo-hp-bb76ea +signed: op=agent_update host=demo-hp-bb76ea guest="" key_id=felhom-op-1 nonce=4806149bb90d075b876a866744da475f expires=2026-10-29T09:37:05Z +{"op_blob_b64":"eyJleHBpcmVzX2F0IjoiMjAyNi0xMC0yOVQwOTozNzowNVoiLCJpc3N1ZWRfYXQiOiIyMDI2LTEwLTI5VDA4OjUyOjA1WiIsImtleV9pZCI6ImZlbGhvbS1vcC0xIiwibm9uY2UiOiI0ODA2MTQ5YmI5MGQwNzViODc2YTg2Njc0NGRhNDc1ZiIsIm9wIjoiYWdlbnRfdXBkYXRlIiwicGFyYW1zIjp7InNoYTI1NiI6IjFlNzAxOTZjNTEzNDZjZjEwZTNmMDRmNTE5NzVkZTU4OTliNWRjNmJhYTVhNDVlYzFkMTdiYzI0ODgzMGQyY2EiLCJ2ZXJzaW9uIjoiMC4xNjguMCJ9LCJ0YXJnZXQiOnsiZ3Vlc3RfaWQiOiIiLCJob3N0X2lkIjoiZGVtby1ocC1iYjc2ZWEifX0=","sig_armored":"-----BEGIN SSH SIGNATURE-----\nU1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgvzPSoI2ADfHbHEAHRCPmujzBoW\nMZnxNr+xYHre2UvD4AAAAMZmVsaG9tLW9wLXYxAAAAAAAAAAZzaGE1MTIAAABTAAAAC3Nz\naC1lZDI1NTE5AAAAQCTkF8R1B2TQsTUJwD1Wp8Bvut39A3AA1IOUiZY6kkUMJYU78i4Voa\nxYPt4TtoRtNKJH2lgX9+8ej2ndtarNpAI=\n-----END SSH SIGNATURE-----\n"} +uploaded signed op to the hub jobs queue +== demo-felhom-8363b5 +signed: op=agent_update host=demo-felhom-8363b5 guest="" key_id=felhom-op-1 nonce=7655701562aaeb40205877356e345792 expires=2026-10-29T09:37:05Z +{"op_blob_b64":"eyJleHBpcmVzX2F0IjoiMjAyNi0xMC0yOVQwOTozNzowNVoiLCJpc3N1ZWRfYXQiOiIyMDI2LTEwLTI5VDA4OjUyOjA1WiIsImtleV9pZCI6ImZlbGhvbS1vcC0xIiwibm9uY2UiOiI3NjU1NzAxNTYyYWFlYjQwMjA1ODc3MzU2ZTM0NTc5MiIsIm9wIjoiYWdlbnRfdXBkYXRlIiwicGFyYW1zIjp7InNoYTI1NiI6IjFlNzAxOTZjNTEzNDZjZjEwZTNmMDRmNTE5NzVkZTU4OTliNWRjNmJhYTVhNDVlYzFkMTdiYzI0ODgzMGQyY2EiLCJ2ZXJzaW9uIjoiMC4xNjguMCJ9LCJ0YXJnZXQiOnsiZ3Vlc3RfaWQiOiIiLCJob3N0X2lkIjoiZGVtby1mZWxob20tODM2M2I1In19","sig_armored":"-----BEGIN SSH SIGNATURE-----\nU1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgvzPSoI2ADfHbHEAHRCPmujzBoW\nMZnxNr+xYHre2UvD4AAAAMZmVsaG9tLW9wLXYxAAAAAAAAAAZzaGE1MTIAAABTAAAAC3Nz\naC1lZDI1NTE5AAAAQKk9J7puhHscGOLV/msgTtOaX+E/XGHlettw5r9xiu7MnvJ/wWXlUD\no2vrV5I0XtvqFInZY+0hfp8jnSjqtHXAI=\n-----END SSH SIGNATURE-----\n"} +uploaded signed op to the hub jobs queue diff --git a/documentation/audits/d63-2026-10-29/p4_live.py b/documentation/audits/d63-2026-10-29/p4_live.py new file mode 100644 index 00000000..9b1c4b16 --- /dev/null +++ b/documentation/audits/d63-2026-10-29/p4_live.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""p4_live.py — Tester 1 on the release build (controller 0.347.0, agent 0.168.0), 2026-10-29, attended. + deploy a throwaway Gokapi (catalog cap 128 MB). + backup the household's „Mentés most" (whole box; the only backup press there is, R-648) — makes Gokapi's unit. + restore one LOCAL restore of Gokapi from its unit through the restore page's form; the app must run after. + oom a memory hog inside Gokapi (`tail /dev/zero`, a child): the host counter rises; the controller logs the OOM + line ONCE with the app and the source, and pushes app_oom once. + teardown the app removed through the product.""" +import json, os, sys, time + +sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts") +SCR = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/0b76c4d7-9b39-4e4e-869d-d0d2c6006331/scratchpad" +os.environ.setdefault("SC", f"{SCR}/rt-tester1") +os.environ.setdefault("TARGET", "tester-1") +import box_walk as w + +EV = f"{SCR}/d63/ev" +APP, SUB = "gokapi", "gokapi" + + +def now(): + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def look(): + st = w.stack(APP) + w.say(f" app: state={st.get('state')} hold={st.get('hold_reason')!r} deployed={st.get('deployed')}") + w.say(" container: " + w.guest(f"docker ps -a --filter name=^/{APP}$ --format '{{{{.Image}}}} {{{{.Status}}}}'").strip()) + return st + + +w.login() +phase = sys.argv[1] +t0 = now() +w.say(f"=== phase {phase} at {t0}") +w.say("controller: " + w.guest("docker ps --filter name=^/felhom-controller$ --format '{{.Image}} {{.Status}}'").strip()) + +if phase == "deploy": + assert w.deploy(APP, SUB), "deploy failed" + w.say(" cap: " + w.guest(f"docker inspect -f '{{{{.HostConfig.Memory}}}}' {APP}; docker exec {APP} sh -c 'command -v tail; echo ok'").strip()) + look() + +elif phase == "backup": + c, d = w.ctl("POST", "/api/backup/run") + w.say(f" backup press → {c} {str(d)[:160]}") + for _ in range(240): + time.sleep(10) + lg = w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'App-data backup completed|Recovery unit (captured|NOT captured) for {APP}' | tail -3") + if "App-data backup completed" in lg: + break + w.say(" log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'App-data backup completed|{APP}' | tail -8")) + w.say(f" restore points: {json.dumps(w.snapshots(APP))[:300]}") + +elif phase == "restore": + look() + res = w.restore(APP) + w.say(f" result: {json.dumps(res)[:600]}") + code, d = w.ctl("GET", "/api/backup/restore-status") + w.say(f" restore record: {json.dumps(d)[:600]}") + w.say(" log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E '{APP}|Restore-from-unit|R-1270|restore copy' | tail -15")) + look() + w.say(" front door: " + str(w.app_curl(SUB, "/")[1])) + +elif phase == "oom": + cid = w.guest(f"docker inspect -f '{{{{.Id}}}}' {APP}").strip() + hostf = f"/sys/fs/cgroup/lxc/9201/ns/system.slice/docker-{cid}.scope/memory.events" + w.say(" host counter before: " + w.sh(w.ssh_args() + ["root@192.168.0.154", f"grep oom_kill {hostf}"]).stdout.strip()) + w.say(" hog: " + w.guest(f"docker exec {APP} sh -c 'tail /dev/zero' ; echo rc=$?", timeout=300).strip()) + w.say(" after: " + w.guest(f"docker inspect -f 'running={{{{.State.Running}}}} oomkilled={{{{.State.OOMKilled}}}} restarts={{{{.RestartCount}}}}' {APP}")) + w.say(" host counter after: " + w.sh(w.ssh_args() + ["root@192.168.0.154", f"grep oom_kill {hostf}"]).stdout.strip()) + time.sleep(75) + w.say(" OOM lines (first 2 scans): " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'oom' | head -10")) + time.sleep(65) + w.say(" app_oom events pushed: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -cE 'Event pushed: app_oom'")) + w.say(" OOM lines (after 4 scans): " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'oom' | head -10")) + +elif phase == "d62": + # D62 end to end on the release build and hub 0.175.0: restore, the copy listed, deleted through the form at once + import re, subprocess + res = w.restore(APP) + w.say(f" restore ok={res.get('state_after')} hold={res.get('hold_after')}") + page = w.page("/backups/restore") + rows = re.findall(r'
', page) + w.say(f" rows: {rows}") + m = re.search(r'data-restore-copies="true">(.*?)\s*