Release run 2026-10-29: hub 0.175.0 ceremony + watch, agent 0.168.0 on the three test boxes, controller 0.347.0 walk + local restore + OOM + D62 end to end on Tester 1 (evidence); R-1341 filed; R-1339/R-1340 moved to their sections, headers recounted
gates / gates (push) Successful in 1m46s
gates / gates (push) Successful in 1m46s
This commit is contained in:
admin committed
1 parent
d6352cd8bf
commit
650aa0cf99
26 files changed
+569
-5
No files matched your search
@@ -0,0 +1,8 @@
|
||||
gitea.dooplex.hu/admin/felhom-controller:0.346.1
|
||||
felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.346.1 Up 5 hours (healthy)
|
||||
filebrowser gtstef/filebrowser:1.5.8-stable Up 5 hours (healthy)
|
||||
paperless-postgres postgres:18-alpine Up 5 hours (healthy)
|
||||
paperless-redis redis:7-alpine Up 5 hours (healthy)
|
||||
paperless-webserver ghcr.io/paperless-ngx/paperless-ngx:2.20.15 Up 5 hours (healthy)
|
||||
traefik traefik:v3.7.16 Up 5 hours
|
||||
359
|
||||
@@ -0,0 +1,8 @@
|
||||
gitea.dooplex.hu/admin/felhom-controller:0.347.0-rc.1
|
||||
gitea.dooplex.hu/admin/felhom-controller:0.347.0-rc.1 Up 50 seconds (healthy)
|
||||
sha256:2a73dd06cb0edba44e83e2a423738ef03ad1f6110faeeab57da9304509592140
|
||||
2026/10/29 08:53:39 undo.go:676: [INFO] [stacks] applied-meta backfill (R-646): recorded 0 []; skipped 0 [] — not current with the catalog, their pinned version's .felhom.yml is no longer on the box
|
||||
2026/10/29 08:53:39 scheduler.go:120: [INFO] [scheduler] Registered periodic job: conversion-copy-release (every 1h0m0s)
|
||||
2026/10/29 08:53:39 scheduler.go:85: [DEBUG] [scheduler] periodic job registered: name="conversion-copy-release" interval=1h0m0s totalJobs=14
|
||||
2026/10/29 08:53:39 scheduler.go:120: [INFO] [scheduler] Registered periodic job: restore-copy-sweep (every 6h0m0s)
|
||||
2026/10/29 08:53:39 scheduler.go:85: [DEBUG] [scheduler] periodic job registered: name="restore-copy-sweep" interval=6h0m0s totalJobs=15
|
||||
@@ -0,0 +1,28 @@
|
||||
09:04:08
|
||||
== root@192.168.0.154 felhom-agent 0.168.0
|
||||
Oct 29 09:56:27 felhom felhom-agent[1161]: time=2026-10-29T09:56:27.781+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/selfupda
|
||||
Oct 29 09:56:28 felhom felhom-agent[1161]: time=2026-10-29T09:56:28.301+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\": {\"
|
||||
Oct 29 09:56:28 felhom felhom-agent[1161]: time=2026-10-29T09:56:28.302+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=fdb69d53f8afa8d3 op=agent_update
|
||||
Oct 29 09:56:31 felhom felhom-agent[424223]: time=2026-10-29T09:56:31.398+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0
|
||||
Oct 29 09:56:32 felhom felhom-agent[424223]: time=2026-10-29T09:56:32.081+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s
|
||||
Oct 29 09:57:32 felhom sudo[426210]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit
|
||||
Oct 29 09:57:32 felhom felhom-selfupdate-guarded[426218]: committed (pending cleared, .prev retained)
|
||||
Oct 29 09:57:32 felhom felhom-agent[424223]: time=2026-10-29T09:57:32.122+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper=""
|
||||
== hp felhom-agent 0.168.0
|
||||
Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.564+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/selfupd
|
||||
Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.953+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\": {\
|
||||
Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.954+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=63ed679a243bcac2 op=agent_update
|
||||
Oct 29 09:55:37 demo-hp felhom-agent[961306]: time=2026-10-29T09:55:37.136+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0
|
||||
Oct 29 09:55:38 demo-hp felhom-agent[961306]: time=2026-10-29T09:55:38.063+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s
|
||||
Oct 29 09:56:38 demo-hp sudo[966235]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit
|
||||
Oct 29 09:56:38 demo-hp felhom-selfupdate-guarded[966243]: committed (pending cleared, .prev retained)
|
||||
Oct 29 09:56:38 demo-hp felhom-agent[961306]: time=2026-10-29T09:56:38.091+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper=""
|
||||
== felhom-pve felhom-agent 0.168.0
|
||||
Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.145+01:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=95d5b5018bde3b37 op=agent_update key_id=felhom-op-1 nonce=7655701562a
|
||||
Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.429+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/sel
|
||||
Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.698+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\"
|
||||
Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.698+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=95d5b5018bde3b37 op=agent_update
|
||||
Oct 29 10:03:43 demo-felhom felhom-agent[357627]: time=2026-10-29T10:03:43.578+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0
|
||||
Oct 29 10:03:44 demo-felhom felhom-agent[357627]: time=2026-10-29T10:03:44.857+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s
|
||||
|
||||
[exited with code 0]
|
||||
@@ -0,0 +1,28 @@
|
||||
== root@192.168.0.154 felhom-agent 0.168.0
|
||||
Oct 29 09:56:27 felhom felhom-agent[1161]: time=2026-10-29T09:56:27.781+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/selfupda
|
||||
Oct 29 09:56:28 felhom felhom-agent[1161]: time=2026-10-29T09:56:28.301+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\": {\"
|
||||
Oct 29 09:56:28 felhom felhom-agent[1161]: time=2026-10-29T09:56:28.302+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=fdb69d53f8afa8d3 op=agent_update
|
||||
Oct 29 09:56:31 felhom felhom-agent[424223]: time=2026-10-29T09:56:31.398+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0
|
||||
Oct 29 09:56:32 felhom felhom-agent[424223]: time=2026-10-29T09:56:32.081+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s
|
||||
Oct 29 09:57:32 felhom sudo[426210]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit
|
||||
Oct 29 09:57:32 felhom felhom-selfupdate-guarded[426218]: committed (pending cleared, .prev retained)
|
||||
Oct 29 09:57:32 felhom felhom-agent[424223]: time=2026-10-29T09:57:32.122+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper=""
|
||||
== hp felhom-agent 0.168.0
|
||||
Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.564+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/selfupd
|
||||
Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.953+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\": {\
|
||||
Oct 29 09:55:33 demo-hp felhom-agent[1265]: time=2026-10-29T09:55:33.954+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=63ed679a243bcac2 op=agent_update
|
||||
Oct 29 09:55:37 demo-hp felhom-agent[961306]: time=2026-10-29T09:55:37.136+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0
|
||||
Oct 29 09:55:38 demo-hp felhom-agent[961306]: time=2026-10-29T09:55:38.063+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s
|
||||
Oct 29 09:56:38 demo-hp sudo[966235]: felhom-agent : PWD=/ ; USER=root ; COMMAND=/usr/local/sbin/felhom-selfupdate-guarded commit
|
||||
Oct 29 09:56:38 demo-hp felhom-selfupdate-guarded[966243]: committed (pending cleared, .prev retained)
|
||||
Oct 29 09:56:38 demo-hp felhom-agent[961306]: time=2026-10-29T09:56:38.091+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper=""
|
||||
== felhom-pve felhom-agent 0.168.0
|
||||
Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.145+01:00 level=WARN msg="signedjobs: AUTHORIZED signed op — executing" job=95d5b5018bde3b37 op=agent_update key_id=felhom-op-1 nonce=7655701562a
|
||||
Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.429+01:00 level=WARN msg="agent_update: handing the signed job to the root wrapper (felhom-os-apply agent_update)" staged=/var/lib/felhom-agent/sel
|
||||
Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.698+01:00 level=WARN msg="agent_update: signed update verified as root and handed off; restart scheduled" version=0.168.0 report="{\"agent_update\"
|
||||
Oct 29 10:03:40 demo-felhom felhom-agent[1194]: time=2026-10-29T10:03:40.698+01:00 level=WARN msg="signedjobs: signed op COMPLETED" job=95d5b5018bde3b37 op=agent_update
|
||||
Oct 29 10:03:43 demo-felhom felhom-agent[357627]: time=2026-10-29T10:03:43.578+01:00 level=INFO msg="capabilities self-check" ok=76 total=76 degraded=0 inactive=0
|
||||
Oct 29 10:03:44 demo-felhom felhom-agent[357627]: time=2026-10-29T10:03:44.857+01:00 level=WARN msg="selfupdate: new version running — dwelling before commit" version=0.168.0 prev=0.167.0 dwell=1m0s
|
||||
Oct 29 10:04:44 demo-felhom felhom-agent[357627]: time=2026-10-29T10:04:44.877+01:00 level=WARN msg="selfupdate: update committed" version=0.168.0 wrapper=""
|
||||
felhom-agent 0.168.0
|
||||
1e70196c51346cf10e3f04f51975de5899b5dc6baa5a45ec1d17bc248830d2ca /usr/local/bin/felhom-agent
|
||||
@@ -0,0 +1,17 @@
|
||||
09:40:09 preflight → 200 {"data": {"agent_supported": true, "escrow_state": "escrowed", "items": [{"id": "pbs_storage_id", "ok": true, "detail": "felhom-pbs"}, {"id": "dr_tier", "ok": true, "detail": "DR tier applied"}, {"id": "age_binary", "ok": true, "detail": "/usr/bin/age"}, {"id": "hub_upload", "ok": true, "detail": "hub upload target configured"}, {"id": "staged_secret", "ok": false, "detail": "no staged secret (informational \u2014 the controller re-stages when offsite is configured)"}, {"id": "sudo_grant", "ok": true, "detail": "sudo grant listed (list-mode)"}], "offbox_configured": true, "ok": true, "stale":
|
||||
09:40:09 start → 200 {"data": {"job_id": "escrow-1793263209404621564", "phase": "running"}, "error": "", "ok": true}
|
||||
09:40:12 status → 200 {"data": {"claim_expires_in_sec": 599, "claimable": true, "claimed": false, "detail": "", "entropy_bits": 129.24070185585344, "job_id": "escrow-1793263209404621564", "key_fingerprint": "82:c6:61:ba:ae:36:47:af:0f:dc:17:0b:0d:5c:5d:63:cd:e7:47:e6:fe:9a:14:fe:66:c1:f0:e6:8f:a1:44:3b", "phase": "done", "restic_pw_sealed": true, "uploaded": false}, "error": "", "ok": true}
|
||||
09:40:12 claim → 200; code received: True (10 words); confirm_required=True positions=[3, 4]
|
||||
09:40:12 confirm → 200 {"data": {"uploaded": true}, "error": "", "ok": true}
|
||||
09:52:46 controller escrow lines:
|
||||
2026/10/29 08:40:09 [INFO] [web] escrow readiness: READY (agent preflight ok) — the recovery-code reminder is shown
|
||||
2026/10/29 08:40:09 [INFO] [web] escrow ceremony started via wizard (job escrow-1793263209404621564)
|
||||
2026/10/29 08:40:12 [INFO] [web] escrow recovery code claimed (one-shot; not logged) — its package waits for the typed-back words
|
||||
2026/10/29 08:40:12 [INFO] [web] escrow recovery code typed back — the new package is on the hub (R-1151)
|
||||
|
||||
stale warning (nem fedi): False | control word Helyre present: False
|
||||
escrow_state: escrowed
|
||||
2026/10/29 08:40:09 [INFO] [web] escrow ceremony started via wizard (job escrow-1793263209404621564)
|
||||
2026/10/29 08:40:12 [INFO] [web] escrow recovery code claimed (one-shot; not logged) — its package waits for the typed-back words
|
||||
2026/10/29 08:40:12 [INFO] [web] escrow recovery code typed back — the new package is on the hub (R-1151)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ceremony.py — one escrow ceremony on Tester 1 through the wizard's own endpoints (hub 0.175.0 check, 2026-10-29).
|
||||
The recovery code lives in this process's memory only: never printed, never written, dropped at the end. The dashboard
|
||||
password is read from the 0600 file and handed to curl on stdin (never on a command line)."""
|
||||
import json, os, subprocess, sys, time
|
||||
|
||||
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
|
||||
SCR = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/0b76c4d7-9b39-4e4e-869d-d0d2c6006331/scratchpad"
|
||||
os.environ.setdefault("SC", f"{SCR}/rt-tester1")
|
||||
os.environ.setdefault("TARGET", "tester-1")
|
||||
import box_walk as w
|
||||
|
||||
w.login()
|
||||
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
||||
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
||||
BASE_ARGS = ["curl", "-sk", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-H", f"X-CSRF-Token: {csrf}"]
|
||||
|
||||
|
||||
def call(method, path, form_stdin=None, body=None):
|
||||
args = BASE_ARGS + ["-X", method, "-w", "\n%{http_code}"]
|
||||
inp = None
|
||||
if form_stdin is not None:
|
||||
args += ["--data-urlencode", "password@-"]
|
||||
inp = form_stdin
|
||||
if body is not None:
|
||||
args += ["-H", "Content-Type: application/json", "--data-binary", "@-"]
|
||||
inp = json.dumps(body)
|
||||
r = subprocess.run(args + [f"{w.BASE}{path}"], input=inp, capture_output=True, text=True, timeout=120)
|
||||
out, _, code = r.stdout.rpartition("\n")
|
||||
try:
|
||||
return code, json.loads(out)
|
||||
except Exception:
|
||||
return code, {"raw": out[:200]}
|
||||
|
||||
|
||||
def safe(d):
|
||||
"""A copy with the code removed — the only form that is printed."""
|
||||
d = json.loads(json.dumps(d))
|
||||
if isinstance(d.get("data"), dict):
|
||||
d["data"].pop("recovery_code", None)
|
||||
return d
|
||||
|
||||
|
||||
t0 = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
code, d = call("GET", "/api/escrow/preflight")
|
||||
w.say(f"preflight → {code} {json.dumps(d)[:600]}")
|
||||
pw = open(f"{w.SC}/.ctlpw").read().strip()
|
||||
code, d = call("POST", "/api/escrow/start", form_stdin=pw)
|
||||
pw = None
|
||||
w.say(f"start → {code} {json.dumps(safe(d))[:300]}")
|
||||
if code != "200":
|
||||
sys.exit(1)
|
||||
for _ in range(120):
|
||||
time.sleep(3)
|
||||
code, d = call("GET", "/api/escrow/status")
|
||||
ph = (d.get("data") or {}).get("phase")
|
||||
if ph not in ("running", "queued", "starting", None) or (d.get("data") or {}).get("claimable"):
|
||||
break
|
||||
w.say(f"status → {code} {json.dumps(safe(d))[:400]}")
|
||||
code, d = call("POST", "/api/escrow/claim")
|
||||
rc = (d.get("data") or {}).get("recovery_code") or ""
|
||||
pos = (d.get("data") or {}).get("verify_positions") or []
|
||||
w.say(f"claim → {code}; code received: {bool(rc)} ({len(rc.split('-'))} words); confirm_required={(d.get('data') or {}).get('confirm_required')} positions={pos}")
|
||||
if rc and pos:
|
||||
words = rc.split("-")
|
||||
picked = [words[pos[0] - 1], words[pos[1] - 1]]
|
||||
rc, words = None, None
|
||||
code, d = call("POST", "/api/escrow/confirm", body={"words": picked})
|
||||
picked = None
|
||||
w.say(f"confirm → {code} {json.dumps(safe(d))[:400]}")
|
||||
rc = None
|
||||
for _ in range(40):
|
||||
time.sleep(15)
|
||||
st = w.guest("docker logs --since " + t0 + " felhom-controller 2>&1 | grep -iE 'escrow' | grep -viE 'recovery_code' | tail -12")
|
||||
if "escrowed" in st or "auto-confirm" in st.lower():
|
||||
break
|
||||
w.say("controller escrow lines:\n" + st)
|
||||
open(f"{SCR}/d63/ev/ceremony.log", "a").write("\n".join(w.LOG) + "\n")
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# floor.sh <token NAME> <path> <min_controller_version or ""> <min_agent or ""> — one floor save at the hub with CC's
|
||||
# scoped token (decision 251). The token arrives through `sec run` under the NAME; it goes to curl through a header file
|
||||
# in $XDG_RUNTIME_DIR (printf is a builtin, so the value is never on a command line) and is never printed.
|
||||
# Prints the status line and the Location header only: 303 to ?flash=floor_set is success; any other flash is a refusal.
|
||||
set -euo pipefail
|
||||
NAME=$1; P=$2; VER=$3; MA=$4
|
||||
H=$(mktemp -p "${XDG_RUNTIME_DIR:-/tmp}")
|
||||
printf "Authorization: Bearer %s\n" "${!NAME}" > "$H"
|
||||
curl -s -o /dev/null -D - -H @"$H" --data-urlencode "min_controller_version=$VER" --data-urlencode "min_agent=$MA" \
|
||||
"http://10.43.52.34:8080$P" | grep -iE "^HTTP|^location" || true
|
||||
: > "$H"; rm -f "$H"
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# opsign-agent.sh <hub-key NAME> <version> <sha256> <host_id>... — one signed agent_update job per box, queued at the hub.
|
||||
# The hub key arrives through `sec run` under the NAME given; its value is never printed.
|
||||
set -euo pipefail
|
||||
KEYNAME=$1; VER=$2; SHA=$3; shift 3
|
||||
HUBKEY=${!KEYNAME}
|
||||
D=/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/0b76c4d7-9b39-4e4e-869d-d0d2c6006331/scratchpad/d63
|
||||
for H in "$@"; do
|
||||
echo "== $H"
|
||||
"$D/felhom-opsign" -op agent_update -host "$H" -key-id felhom-op-1 -key /mnt/5_hdd/felhom.eu/felhom-op-operational \
|
||||
-agent-version "$VER" -sha256 "$SHA" -ttl 45m -upload http://10.43.52.34:8080 -hub-key "$HUBKEY" 2>&1 | tail -3
|
||||
done
|
||||
@@ -0,0 +1,12 @@
|
||||
== tester-1-d70be4
|
||||
signed: op=agent_update host=tester-1-d70be4 guest="" key_id=felhom-op-1 nonce=d853bac716c5c4fbfb2b566436885b14 expires=2026-10-29T09:37:05Z
|
||||
{"op_blob_b64":"eyJleHBpcmVzX2F0IjoiMjAyNi0xMC0yOVQwOTozNzowNVoiLCJpc3N1ZWRfYXQiOiIyMDI2LTEwLTI5VDA4OjUyOjA1WiIsImtleV9pZCI6ImZlbGhvbS1vcC0xIiwibm9uY2UiOiJkODUzYmFjNzE2YzVjNGZiZmIyYjU2NjQzNjg4NWIxNCIsIm9wIjoiYWdlbnRfdXBkYXRlIiwicGFyYW1zIjp7InNoYTI1NiI6IjFlNzAxOTZjNTEzNDZjZjEwZTNmMDRmNTE5NzVkZTU4OTliNWRjNmJhYTVhNDVlYzFkMTdiYzI0ODgzMGQyY2EiLCJ2ZXJzaW9uIjoiMC4xNjguMCJ9LCJ0YXJnZXQiOnsiZ3Vlc3RfaWQiOiIiLCJob3N0X2lkIjoidGVzdGVyLTEtZDcwYmU0In19","sig_armored":"-----BEGIN SSH SIGNATURE-----\nU1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgvzPSoI2ADfHbHEAHRCPmujzBoW\nMZnxNr+xYHre2UvD4AAAAMZmVsaG9tLW9wLXYxAAAAAAAAAAZzaGE1MTIAAABTAAAAC3Nz\naC1lZDI1NTE5AAAAQJsggYiigisd0vNWgbqm+IqEMkSJuWqcdR/HvrB1CFdYTALfaT7S8n\nsX2+4JNk/Qn95q08vRge98k/xgoqtuxQ8=\n-----END SSH SIGNATURE-----\n"}
|
||||
uploaded signed op to the hub jobs queue
|
||||
== demo-hp-bb76ea
|
||||
signed: op=agent_update host=demo-hp-bb76ea guest="" key_id=felhom-op-1 nonce=4806149bb90d075b876a866744da475f expires=2026-10-29T09:37:05Z
|
||||
{"op_blob_b64":"eyJleHBpcmVzX2F0IjoiMjAyNi0xMC0yOVQwOTozNzowNVoiLCJpc3N1ZWRfYXQiOiIyMDI2LTEwLTI5VDA4OjUyOjA1WiIsImtleV9pZCI6ImZlbGhvbS1vcC0xIiwibm9uY2UiOiI0ODA2MTQ5YmI5MGQwNzViODc2YTg2Njc0NGRhNDc1ZiIsIm9wIjoiYWdlbnRfdXBkYXRlIiwicGFyYW1zIjp7InNoYTI1NiI6IjFlNzAxOTZjNTEzNDZjZjEwZTNmMDRmNTE5NzVkZTU4OTliNWRjNmJhYTVhNDVlYzFkMTdiYzI0ODgzMGQyY2EiLCJ2ZXJzaW9uIjoiMC4xNjguMCJ9LCJ0YXJnZXQiOnsiZ3Vlc3RfaWQiOiIiLCJob3N0X2lkIjoiZGVtby1ocC1iYjc2ZWEifX0=","sig_armored":"-----BEGIN SSH SIGNATURE-----\nU1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgvzPSoI2ADfHbHEAHRCPmujzBoW\nMZnxNr+xYHre2UvD4AAAAMZmVsaG9tLW9wLXYxAAAAAAAAAAZzaGE1MTIAAABTAAAAC3Nz\naC1lZDI1NTE5AAAAQCTkF8R1B2TQsTUJwD1Wp8Bvut39A3AA1IOUiZY6kkUMJYU78i4Voa\nxYPt4TtoRtNKJH2lgX9+8ej2ndtarNpAI=\n-----END SSH SIGNATURE-----\n"}
|
||||
uploaded signed op to the hub jobs queue
|
||||
== demo-felhom-8363b5
|
||||
signed: op=agent_update host=demo-felhom-8363b5 guest="" key_id=felhom-op-1 nonce=7655701562aaeb40205877356e345792 expires=2026-10-29T09:37:05Z
|
||||
{"op_blob_b64":"eyJleHBpcmVzX2F0IjoiMjAyNi0xMC0yOVQwOTozNzowNVoiLCJpc3N1ZWRfYXQiOiIyMDI2LTEwLTI5VDA4OjUyOjA1WiIsImtleV9pZCI6ImZlbGhvbS1vcC0xIiwibm9uY2UiOiI3NjU1NzAxNTYyYWFlYjQwMjA1ODc3MzU2ZTM0NTc5MiIsIm9wIjoiYWdlbnRfdXBkYXRlIiwicGFyYW1zIjp7InNoYTI1NiI6IjFlNzAxOTZjNTEzNDZjZjEwZTNmMDRmNTE5NzVkZTU4OTliNWRjNmJhYTVhNDVlYzFkMTdiYzI0ODgzMGQyY2EiLCJ2ZXJzaW9uIjoiMC4xNjguMCJ9LCJ0YXJnZXQiOnsiZ3Vlc3RfaWQiOiIiLCJob3N0X2lkIjoiZGVtby1mZWxob20tODM2M2I1In19","sig_armored":"-----BEGIN SSH SIGNATURE-----\nU1NIU0lHAAAAAQAAADMAAAALc3NoLWVkMjU1MTkAAAAgvzPSoI2ADfHbHEAHRCPmujzBoW\nMZnxNr+xYHre2UvD4AAAAMZmVsaG9tLW9wLXYxAAAAAAAAAAZzaGE1MTIAAABTAAAAC3Nz\naC1lZDI1NTE5AAAAQKk9J7puhHscGOLV/msgTtOaX+E/XGHlettw5r9xiu7MnvJ/wWXlUD\no2vrV5I0XtvqFInZY+0hfp8jnSjqtHXAI=\n-----END SSH SIGNATURE-----\n"}
|
||||
uploaded signed op to the hub jobs queue
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""p4_live.py <phase> — Tester 1 on the release build (controller 0.347.0, agent 0.168.0), 2026-10-29, attended.
|
||||
deploy a throwaway Gokapi (catalog cap 128 MB).
|
||||
backup the household's „Mentés most" (whole box; the only backup press there is, R-648) — makes Gokapi's unit.
|
||||
restore one LOCAL restore of Gokapi from its unit through the restore page's form; the app must run after.
|
||||
oom a memory hog inside Gokapi (`tail /dev/zero`, a child): the host counter rises; the controller logs the OOM
|
||||
line ONCE with the app and the source, and pushes app_oom once.
|
||||
teardown the app removed through the product."""
|
||||
import json, os, sys, time
|
||||
|
||||
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
|
||||
SCR = "/tmp/claude-1000/-mnt-5-hdd-felhom-eu-git/0b76c4d7-9b39-4e4e-869d-d0d2c6006331/scratchpad"
|
||||
os.environ.setdefault("SC", f"{SCR}/rt-tester1")
|
||||
os.environ.setdefault("TARGET", "tester-1")
|
||||
import box_walk as w
|
||||
|
||||
EV = f"{SCR}/d63/ev"
|
||||
APP, SUB = "gokapi", "gokapi"
|
||||
|
||||
|
||||
def now():
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
|
||||
def look():
|
||||
st = w.stack(APP)
|
||||
w.say(f" app: state={st.get('state')} hold={st.get('hold_reason')!r} deployed={st.get('deployed')}")
|
||||
w.say(" container: " + w.guest(f"docker ps -a --filter name=^/{APP}$ --format '{{{{.Image}}}} {{{{.Status}}}}'").strip())
|
||||
return st
|
||||
|
||||
|
||||
w.login()
|
||||
phase = sys.argv[1]
|
||||
t0 = now()
|
||||
w.say(f"=== phase {phase} at {t0}")
|
||||
w.say("controller: " + w.guest("docker ps --filter name=^/felhom-controller$ --format '{{.Image}} {{.Status}}'").strip())
|
||||
|
||||
if phase == "deploy":
|
||||
assert w.deploy(APP, SUB), "deploy failed"
|
||||
w.say(" cap: " + w.guest(f"docker inspect -f '{{{{.HostConfig.Memory}}}}' {APP}; docker exec {APP} sh -c 'command -v tail; echo ok'").strip())
|
||||
look()
|
||||
|
||||
elif phase == "backup":
|
||||
c, d = w.ctl("POST", "/api/backup/run")
|
||||
w.say(f" backup press → {c} {str(d)[:160]}")
|
||||
for _ in range(240):
|
||||
time.sleep(10)
|
||||
lg = w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'App-data backup completed|Recovery unit (captured|NOT captured) for {APP}' | tail -3")
|
||||
if "App-data backup completed" in lg:
|
||||
break
|
||||
w.say(" log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'App-data backup completed|{APP}' | tail -8"))
|
||||
w.say(f" restore points: {json.dumps(w.snapshots(APP))[:300]}")
|
||||
|
||||
elif phase == "restore":
|
||||
look()
|
||||
res = w.restore(APP)
|
||||
w.say(f" result: {json.dumps(res)[:600]}")
|
||||
code, d = w.ctl("GET", "/api/backup/restore-status")
|
||||
w.say(f" restore record: {json.dumps(d)[:600]}")
|
||||
w.say(" log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E '{APP}|Restore-from-unit|R-1270|restore copy' | tail -15"))
|
||||
look()
|
||||
w.say(" front door: " + str(w.app_curl(SUB, "/")[1]))
|
||||
|
||||
elif phase == "oom":
|
||||
cid = w.guest(f"docker inspect -f '{{{{.Id}}}}' {APP}").strip()
|
||||
hostf = f"/sys/fs/cgroup/lxc/9201/ns/system.slice/docker-{cid}.scope/memory.events"
|
||||
w.say(" host counter before: " + w.sh(w.ssh_args() + ["root@192.168.0.154", f"grep oom_kill {hostf}"]).stdout.strip())
|
||||
w.say(" hog: " + w.guest(f"docker exec {APP} sh -c 'tail /dev/zero' ; echo rc=$?", timeout=300).strip())
|
||||
w.say(" after: " + w.guest(f"docker inspect -f 'running={{{{.State.Running}}}} oomkilled={{{{.State.OOMKilled}}}} restarts={{{{.RestartCount}}}}' {APP}"))
|
||||
w.say(" host counter after: " + w.sh(w.ssh_args() + ["root@192.168.0.154", f"grep oom_kill {hostf}"]).stdout.strip())
|
||||
time.sleep(75)
|
||||
w.say(" OOM lines (first 2 scans): " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'oom' | head -10"))
|
||||
time.sleep(65)
|
||||
w.say(" app_oom events pushed: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -cE 'Event pushed: app_oom'"))
|
||||
w.say(" OOM lines (after 4 scans): " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -iE 'oom' | head -10"))
|
||||
|
||||
elif phase == "d62":
|
||||
# D62 end to end on the release build and hub 0.175.0: restore, the copy listed, deleted through the form at once
|
||||
import re, subprocess
|
||||
res = w.restore(APP)
|
||||
w.say(f" restore ok={res.get('state_after')} hold={res.get('hold_after')}")
|
||||
page = w.page("/backups/restore")
|
||||
rows = re.findall(r'<details class="kept-delete" data-restore-copy="([^"]+)">', page)
|
||||
w.say(f" rows: {rows}")
|
||||
m = re.search(r'data-restore-copies="true">(.*?)</div>\s*<script>', page, re.S)
|
||||
if m:
|
||||
w.say(" list text: " + re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", m.group(1))).strip()[:400])
|
||||
if rows:
|
||||
app, stamp = rows[0].split("/", 1)
|
||||
names = re.findall(r'<span class="app-row-name">([^<]+)</span>', page[:page.find('data-restore-copy="' + rows[0])])
|
||||
typed = names[-1].strip() if names else app
|
||||
sess = open(f"{w.SC}/sess{os.getpid()}.txt").read().strip()
|
||||
csrf = open(f"{w.SC}/csrf{os.getpid()}.txt").read().strip()
|
||||
r = subprocess.run(["curl", "-sk", "-w", "\n%{http_code}", "-H", w.HOSTHDR, "-H", f"Cookie: {sess}", "-H", f"X-CSRF-Token: {csrf}",
|
||||
"-H", "Accept: application/json", "-X", "POST", "--data-urlencode", f"_csrf={csrf}",
|
||||
"--data-urlencode", f"app={app}", "--data-urlencode", f"stamp={stamp}", "--data-urlencode", f"confirm={typed}",
|
||||
f"{w.BASE}/backup/restore-copy/delete"], capture_output=True, text=True, timeout=60)
|
||||
w.say(f" delete ({typed!r}) → {r.stdout.strip()[-300:]}")
|
||||
time.sleep(8)
|
||||
w.say(" log: " + w.guest(f"docker logs --since {t0} felhom-controller 2>&1 | grep -E 'restore copy DELETED|restore_copy_deleted|Event push' | tail -4"))
|
||||
w.say(" copies left: " + (w.guest("find /mnt/sys_drive/felhom-data/backups/restore-undo -mindepth 2 -maxdepth 2 2>/dev/null").strip() or "(none)"))
|
||||
|
||||
elif phase == "teardown":
|
||||
look()
|
||||
w.remove(APP)
|
||||
look()
|
||||
|
||||
open(os.path.join(EV, f"p4-{phase}.log"), "a").write("\n".join(w.LOG) + "\n")
|
||||
@@ -0,0 +1,9 @@
|
||||
== tester-1
|
||||
HTTP/1.1 303 See Other
|
||||
Location: /customers/tester-1?flash=floor_set
|
||||
== demo-hp
|
||||
HTTP/1.1 303 See Other
|
||||
Location: /customers/demo-hp?flash=floor_set
|
||||
== demo-felhom
|
||||
HTTP/1.1 303 See Other
|
||||
Location: /customers/demo-felhom?flash=floor_set
|
||||
@@ -0,0 +1,11 @@
|
||||
09:40:09 preflight → 200 {"data": {"agent_supported": true, "escrow_state": "escrowed", "items": [{"id": "pbs_storage_id", "ok": true, "detail": "felhom-pbs"}, {"id": "dr_tier", "ok": true, "detail": "DR tier applied"}, {"id": "age_binary", "ok": true, "detail": "/usr/bin/age"}, {"id": "hub_upload", "ok": true, "detail": "hub upload target configured"}, {"id": "staged_secret", "ok": false, "detail": "no staged secret (informational \u2014 the controller re-stages when offsite is configured)"}, {"id": "sudo_grant", "ok": true, "detail": "sudo grant listed (list-mode)"}], "offbox_configured": true, "ok": true, "stale":
|
||||
09:40:09 start → 200 {"data": {"job_id": "escrow-1793263209404621564", "phase": "running"}, "error": "", "ok": true}
|
||||
09:40:12 status → 200 {"data": {"claim_expires_in_sec": 599, "claimable": true, "claimed": false, "detail": "", "entropy_bits": 129.24070185585344, "job_id": "escrow-1793263209404621564", "key_fingerprint": "82:c6:61:ba:ae:36:47:af:0f:dc:17:0b:0d:5c:5d:63:cd:e7:47:e6:fe:9a:14:fe:66:c1:f0:e6:8f:a1:44:3b", "phase": "done", "restic_pw_sealed": true, "uploaded": false}, "error": "", "ok": true}
|
||||
09:40:12 claim → 200; code received: True (10 words); confirm_required=True positions=[3, 4]
|
||||
09:40:12 confirm → 200 {"data": {"uploaded": true}, "error": "", "ok": true}
|
||||
09:52:46 controller escrow lines:
|
||||
2026/10/29 08:40:09 [INFO] [web] escrow readiness: READY (agent preflight ok) — the recovery-code reminder is shown
|
||||
2026/10/29 08:40:09 [INFO] [web] escrow ceremony started via wizard (job escrow-1793263209404621564)
|
||||
2026/10/29 08:40:12 [INFO] [web] escrow recovery code claimed (one-shot; not logged) — its package waits for the typed-back words
|
||||
2026/10/29 08:40:12 [INFO] [web] escrow recovery code typed back — the new package is on the hub (R-1151)
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
09:59:44 === phase backup at 2026-10-29T08:59:44Z
|
||||
09:59:47 controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up About a minute (healthy)
|
||||
09:59:47 backup press → 200 {'ok': True, 'message': 'Mentés elindítva'}
|
||||
10:00:32 log: 2026/10/29 09:00:00 [INFO] [backup] Volume dump: gokapi/gokapi_gokapi_data → 43.0 KB
|
||||
2026/10/29 09:00:00 [INFO] [backup] Restarting gokapi after volume dump
|
||||
2026/10/29 09:00:00 [INFO] [stacks] Starting stack: gokapi
|
||||
2026/10/29 09:00:01 [INFO] [stacks] Stack gokapi started successfully (took 0.4s)
|
||||
2026/10/29 09:00:04 [INFO] [stacks] Stack gokapi post-start status:
|
||||
2026/10/29 09:00:04 [INFO] [stacks] gokapi f0rc3/gokapi:v1.9.6 running Up 3 seconds (health: starting)
|
||||
2026/10/29 09:00:23 [INFO] [backup] App-data backup completed: 2 databases (534.9 KB total), 4 volume dump(s) (35.888s)
|
||||
2026/10/29 09:00:23 [INFO] [backup] Recovery unit captured for gokapi → /mnt/sys_drive/felhom-data/backups/primary/gokapi (images=1, secrets-referenced=1, data_keys=0, portable-carried=0/0, withheld=1)
|
||||
|
||||
10:00:32 restore points: [{"time": "2026-10-29T09:00:00Z", "short_id": "helyi", "tier": 1, "drive_label": "Bels\u0151 SSD (rendszer)"}]
|
||||
@@ -0,0 +1,10 @@
|
||||
09:59:13 === phase deploy at 2026-10-29T08:59:13Z
|
||||
09:59:16 controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up About a minute (healthy)
|
||||
09:59:16 [1] required fields filled beyond DOMAIN/SUBDOMAIN: ['GOKAPI_PASSWORD']
|
||||
09:59:16 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'}
|
||||
09:59:36 [1] deployed, controller state=running, pinned={'gokapi': 'f0rc3/gokapi:v1.9.6'}
|
||||
09:59:39 cap: 134217728
|
||||
/usr/bin/tail
|
||||
ok
|
||||
09:59:39 app: state=running hold=None deployed=True
|
||||
09:59:42 container: f0rc3/gokapi:v1.9.6 Up 21 seconds (healthy)
|
||||
@@ -0,0 +1,21 @@
|
||||
10:02:07 === phase oom at 2026-10-29T09:02:07Z
|
||||
10:02:10 controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 4 minutes (healthy)
|
||||
10:02:13 host counter before:
|
||||
10:02:17 hog: rc=137
|
||||
10:02:20 after: running=true oomkilled=true restarts=0
|
||||
|
||||
10:02:20 host counter after:
|
||||
10:03:39 OOM lines (first 2 scans): 2026/10/29 09:02:32 [WARN] [deadapp] gokapi: container gokapi was OOM-killed (started 2026-10-29T09:00:44.706771082Z; kills 1, limit 128M, peak 128M; seen by docker,cgroup)
|
||||
2026/10/29 09:02:32 [INFO] Event pushed: app_oom (warning) [hu-only] — Alkalmazás memóriája elfogyott: gokapi (gokapi) — egy folyamatát a memóriakorlát leállította
|
||||
2026/10/29 09:03:02 [WARN] [deadapp] gokapi: container gokapi was OOM-killed (started 2026-10-29T09:00:44.706771082Z; kills 1, limit 128M, peak 128M; seen by docker,cgroup)
|
||||
2026/10/29 09:03:32 [WARN] [deadapp] gokapi: container gokapi was OOM-killed (started 2026-10-29T09:00:44.706771082Z; kills 1, limit 128M, peak 128M; seen by docker,cgroup)
|
||||
|
||||
10:04:48 app_oom events pushed: 1
|
||||
|
||||
10:04:51 OOM lines (after 4 scans): 2026/10/29 09:02:32 [WARN] [deadapp] gokapi: container gokapi was OOM-killed (started 2026-10-29T09:00:44.706771082Z; kills 1, limit 128M, peak 128M; seen by docker,cgroup)
|
||||
2026/10/29 09:02:32 [INFO] Event pushed: app_oom (warning) [hu-only] — Alkalmazás memóriája elfogyott: gokapi (gokapi) — egy folyamatát a memóriakorlát leállította
|
||||
2026/10/29 09:03:02 [WARN] [deadapp] gokapi: container gokapi was OOM-killed (started 2026-10-29T09:00:44.706771082Z; kills 1, limit 128M, peak 128M; seen by docker,cgroup)
|
||||
2026/10/29 09:03:32 [WARN] [deadapp] gokapi: container gokapi was OOM-killed (started 2026-10-29T09:00:44.706771082Z; kills 1, limit 128M, peak 128M; seen by docker,cgroup)
|
||||
2026/10/29 09:04:02 [WARN] [deadapp] gokapi: container gokapi was OOM-killed (started 2026-10-29T09:00:44.706771082Z; kills 1, limit 128M, peak 128M; seen by docker,cgroup)
|
||||
2026/10/29 09:04:32 [WARN] [deadapp] gokapi: container gokapi was OOM-killed (started 2026-10-29T09:00:44.706771082Z; kills 1, limit 128M, peak 128M; seen by docker,cgroup)
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
10:00:33 === phase restore at 2026-10-29T09:00:33Z
|
||||
10:00:37 controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 2 minutes (healthy)
|
||||
10:00:37 app: state=running hold=None deployed=True
|
||||
10:00:40 container: f0rc3/gokapi:v1.9.6 Up 38 seconds (healthy)
|
||||
10:00:40 [R] restoring gokapi from snapshot 'helyi' (of 1 offered)
|
||||
10:00:40 [R] POST /backup/restore -> HTTP/2 302 ['location: /backups/restore?flash=flash.restore.started']
|
||||
10:00:40 + 0.0s restore (True, None, None)
|
||||
10:01:45 + 64.9s restore (False, None, None)
|
||||
10:01:45 [R] after restore: state=running hold=None phase=None
|
||||
10:01:49 result: {"ok": true, "snapshot_id": "helyi", "snapshots": [{"time": "2026-10-29T09:00:00Z", "short_id": "helyi", "tier": 1, "drive_label": "Bels\u0151 SSD (rendszer)"}], "http": "HTTP/2 302", "location": ["location: /backups/restore?flash=flash.restore.started"], "seconds": 64.9, "state_after": "running", "hold_after": null, "observables_after": {"pinned_images": {"gokapi": "f0rc3/gokapi:v1.9.6"}, "installed_images": {"gokapi": "f0rc3/gokapi:v1.9.6"}, "catalog_images": {"gokapi": "f0rc3/gokapi:v1.9.6"}, "live_compose_image_lines": ["image: f0rc3/gokapi:v1.9.6"], "docker_inspect": ["gokapi f0rc3/gokap
|
||||
10:01:49 restore record: {"ok": true, "data": {"running": false, "op": "restore", "stack": "gokapi", "started_at": "2026-10-29T09:00:40.856391419Z", "last": {"op": "restore", "stack": "gokapi", "ok": true, "message": "A(z) gokapi: 2 adatk\u00f6tet vissza\u00e1ll\u00edtva \u2014 az alkalmaz\u00e1s \u00fajraindult.", "finished_at": "2026-10-29T09:01:43.903691761Z"}, "last_recent": true}}
|
||||
10:01:52 log: 2026/10/29 09:00:44 [INFO] [stacks] SaveAppConfig: saved config for gokapi
|
||||
2026/10/29 09:00:44 [INFO] [backup] Restored 2 Docker volume(s) for gokapi
|
||||
2026/10/29 09:00:44 [INFO] [stacks] Redeploying gokapi from recovery unit with 3 env vars
|
||||
2026/10/29 09:00:44 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/gokapi — 3 env vars, 0 encrypted, 1 sensitive fields
|
||||
2026/10/29 09:00:44 [INFO] [stacks] SaveAppConfig: saved config for gokapi
|
||||
2026/10/29 09:00:44 [INFO] [stacks] pin gokapi: gokapi=f0rc3/gokapi:v1.9.6
|
||||
2026/10/29 09:00:44 [INFO] [stacks] Starting stack: gokapi
|
||||
2026/10/29 09:00:44 [INFO] [stacks] Stack gokapi started successfully (took 0.5s)
|
||||
2026/10/29 09:00:45 [DEBUG] [stacks] SaveAppConfig: saving /opt/docker/stacks/gokapi — 3 env vars, 0 encrypted, 1 sensitive fields
|
||||
2026/10/29 09:00:45 [INFO] [stacks] SaveAppConfig: saved config for gokapi
|
||||
2026/10/29 09:00:45 [INFO] [stacks] installed-images gokapi: recorded 1 service(s) (gokapi=f0rc3/gokapi:v1.9.6 (sha256:ae9094a0ead8…))
|
||||
2026/10/29 09:00:48 [INFO] [stacks] Stack gokapi post-start status:
|
||||
2026/10/29 09:00:48 [INFO] [stacks] gokapi f0rc3/gokapi:v1.9.6 running Up 3 seconds (health: starting)
|
||||
2026/10/29 09:01:43 [INFO] [backup] Restore-from-unit completed: gokapi — 2 volume(s) of 2 listed, 0 database(s) of 0 listed
|
||||
2026/10/29 09:01:43 [INFO] [web] Restore completed (async): stack=gokapi in 1m3.046629586s (volumes 2/2, dbs 0/0)
|
||||
|
||||
10:01:52 app: state=running hold=None deployed=True
|
||||
10:01:55 container: f0rc3/gokapi:v1.9.6 Up About a minute (healthy)
|
||||
10:01:55 front door: 200
|
||||
@@ -0,0 +1,9 @@
|
||||
10:05:24 === phase teardown at 2026-10-29T09:05:24Z
|
||||
10:05:27 controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 7 minutes (healthy)
|
||||
10:05:27 app: state=running hold=None deployed=True
|
||||
10:05:30 container: f0rc3/gokapi:v1.9.6 Up 4 minutes (healthy)
|
||||
10:05:31 [X] stop -> 200 {'ok': True, 'message': 'Stack gokapi stop completed'}
|
||||
10:06:03 [X] remove (with drive data) -> 200 {'ok': True, 'data': {'removed': 'gokapi', 'volumes_removed': ['gokapi_gokapi_config', 'gokapi_gokapi_data'], 'hdd_paths_removed': [], 'hdd_paths_preserved': []
|
||||
10:06:11 [X] after remove: deployed=False leftovers=''
|
||||
10:06:11 app: state=not_deployed hold=None deployed=False
|
||||
10:06:15 container:
|
||||
@@ -0,0 +1,13 @@
|
||||
10:07:42 === phase backup at 2026-10-29T09:07:42Z
|
||||
10:07:45 controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 9 minutes (healthy)
|
||||
10:07:45 backup press → 200 {'ok': True, 'message': 'Mentés elindítva'}
|
||||
10:08:30 log: 2026/10/29 09:08:02 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/gokapi/docker-compose.yml
|
||||
2026/10/29 09:08:02 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/gokapi/docker-compose.yml
|
||||
2026/10/29 09:08:02 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/gokapi/docker-compose.yml
|
||||
2026/10/29 09:08:02 [INFO] [stacks] ParseComposeHDDMounts: found 0 HDD mounts for /opt/docker/stacks/gokapi/docker-compose.yml
|
||||
2026/10/29 09:08:02 [INFO] [backup] Recovery unit captured for gokapi → /mnt/sys_drive/felhom-data/backups/primary/gokapi (images=1, secrets-referenced=1, data_keys=0, portable-carried=0/0, withheld=1)
|
||||
2026/10/29 09:08:02 [INFO] [stacks] Stack gokapi post-start status:
|
||||
2026/10/29 09:08:02 [INFO] [stacks] gokapi f0rc3/gokapi:v1.9.6 running Up 3 seconds (health: starting)
|
||||
2026/10/29 09:08:21 [INFO] [backup] App-data backup completed: 2 databases (535.9 KB total), 4 volume dump(s) (35.82s)
|
||||
|
||||
10:08:30 restore points: [{"time": "2026-10-29T09:07:58Z", "short_id": "helyi", "tier": 1, "drive_label": "Bels\u0151 SSD (rendszer)"}]
|
||||
@@ -0,0 +1,17 @@
|
||||
10:08:31 === phase d62 at 2026-10-29T09:08:31Z
|
||||
10:08:35 controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 10 minutes (healthy)
|
||||
10:08:35 [R] restoring gokapi from snapshot 'helyi' (of 1 offered)
|
||||
10:08:35 [R] POST /backup/restore -> HTTP/2 302 ['location: /backups/restore?flash=flash.restore.started']
|
||||
10:08:35 + 0.0s restore (True, None, None)
|
||||
10:09:38 + 62.8s restore (False, None, None)
|
||||
10:09:38 [R] after restore: state=running hold=None phase=None
|
||||
10:09:42 restore ok=running hold=None
|
||||
10:10:30 rows: ['gokapi/20261029T090836Z']
|
||||
10:10:30 list text: Gokapi visszaállítás: 2026. 10. 29. 10:08 · 59,8 KB · automatikusan törlődik: 2026. 11. 28. Törlés most Ez a(z) Gokapi adatainak másolata a 2026. 10. 29-i visszaállítás előttről. Ha törlöd, a visszaállítás többé nem vonható vissza, és ez a másolat nem hozható vissza. A törléshez írd be: Gokapi Törlés most
|
||||
10:10:30 delete ('Gokapi') → {"error":"","ok":true,"redirect":"/backups/restore?fa=Gokapi\u0026flash=flash.restore_copy.deleted"}
|
||||
|
||||
200
|
||||
10:10:42 log: 2026/10/29 09:10:30 [WARN] [web] restore copy DELETED by hand (D62): app=gokapi restore_at=2026-10-29T09:08:36Z size=61264 path=/mnt/sys_drive/felhom-data/backups/restore-undo/gokapi/20261029T090836Z who=the household (dashboard sign-in) from 192.168.0.180
|
||||
2026/10/29 09:10:31 [INFO] Event pushed: restore_copy_deleted (info) [hu-only] — gokapi: the restore copy from before the restore of 2026-10-29T09:08:36Z (61264 bytes) was deleted by hand — the household (dashboard sign-in) from 192.168.0.180. That restore can no longer be undone from this box.
|
||||
|
||||
10:10:46 copies left: (none)
|
||||
@@ -0,0 +1,10 @@
|
||||
10:07:15 === phase deploy at 2026-10-29T09:07:15Z
|
||||
10:07:18 controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 9 minutes (healthy)
|
||||
10:07:18 [1] required fields filled beyond DOMAIN/SUBDOMAIN: ['GOKAPI_PASSWORD']
|
||||
10:07:18 [1] deploy -> 202 {'ok': True, 'message': 'Telepítés elindítva – az állapot a kártyán követhető'}
|
||||
10:07:33 [1] deployed, controller state=running, pinned={'gokapi': 'f0rc3/gokapi:v1.9.6'}
|
||||
10:07:37 cap: 134217728
|
||||
/usr/bin/tail
|
||||
ok
|
||||
10:07:37 app: state=running hold=None deployed=True
|
||||
10:07:41 container: f0rc3/gokapi:v1.9.6 Up 17 seconds (healthy)
|
||||
@@ -0,0 +1,9 @@
|
||||
10:10:58 === phase teardown at 2026-10-29T09:10:58Z
|
||||
10:11:01 controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 13 minutes (healthy)
|
||||
10:11:01 app: state=running hold=None deployed=True
|
||||
10:11:05 container: f0rc3/gokapi:v1.9.6 Up 2 minutes (healthy)
|
||||
10:11:05 [X] stop -> 200 {'ok': True, 'message': 'Stack gokapi stop completed'}
|
||||
10:11:37 [X] remove (with drive data) -> 200 {'ok': True, 'data': {'removed': 'gokapi', 'volumes_removed': ['gokapi_gokapi_config', 'gokapi_gokapi_data'], 'hdd_paths_removed': [], 'hdd_paths_preserved': []
|
||||
10:11:46 [X] after remove: deployed=False leftovers=''
|
||||
10:11:46 app: state=not_deployed hold=None deployed=False
|
||||
10:11:49 container:
|
||||
@@ -0,0 +1,19 @@
|
||||
3
|
||||
268
|
||||
/mnt/sys_drive/felhom-data/backups/restore-undo
|
||||
bookstack lscr.io/linuxserver/bookstack:26.09.2
|
||||
bookstack-db mariadb:12.3
|
||||
cloudflared cloudflare/cloudflared:2026.10.4
|
||||
felhom-controller gitea.dooplex.hu/admin/felhom-controller:0.347.0
|
||||
filebrowser gtstef/filebrowser:1.5.8-stable
|
||||
paperless-postgres postgres:18-alpine
|
||||
paperless-redis redis:7-alpine
|
||||
paperless-webserver ghcr.io/paperless-ngx/paperless-ngx:2.20.15
|
||||
privatebin privatebin/pdo:2.1.0
|
||||
traefik traefik:v3.7.16
|
||||
removed 3
|
||||
268
|
||||
primary
|
||||
secondary
|
||||
set-aside
|
||||
10
|
||||
@@ -0,0 +1,58 @@
|
||||
=== tester-1
|
||||
controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 52 seconds (healthy)
|
||||
containers: 10 total
|
||||
not running/unhealthy: none
|
||||
refusals since 2026-10-29T08:57:50Z: 0
|
||||
ERROR lines since: 0
|
||||
login page: 200
|
||||
file manager: 200
|
||||
=== demo-hp
|
||||
controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 53 seconds (healthy)
|
||||
containers: 26 total
|
||||
not running/unhealthy: none
|
||||
refusals since 2026-10-29T08:57:50Z: 0
|
||||
ERROR lines since: 0
|
||||
login page: 200
|
||||
file manager: 200
|
||||
=== demo-felhom
|
||||
controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0 Up 56 seconds (healthy)
|
||||
containers: 9 total
|
||||
not running/unhealthy: none
|
||||
refusals since 2026-10-29T08:57:50Z: 0
|
||||
ERROR lines since: 0
|
||||
login page: 200
|
||||
file manager: 200
|
||||
control NEGATIVE (a stub answering 200 over plain http): convicted, as it must (200 -)
|
||||
== tester-1 (enkicsifelhom.hu): 6 public name(s)
|
||||
OK control POSITIVE http://felhom.enkicsifelhom.hu/r1033-check?q=1 -> 301 https://felhom.enkicsifelhom.hu/r1033-check?q=1
|
||||
OK http://files.enkicsifelhom.hu/r1033-check?q=1 -> 301 https://files.enkicsifelhom.hu/r1033-check?q=1
|
||||
OK http://no-such-name-r1033.enkicsifelhom.hu/r1033-check?q=1 -> 301 https://no-such-name-r1033.enkicsifelhom.hu/r1033-check?q=1
|
||||
OK http://paperless.enkicsifelhom.hu/r1033-check?q=1 -> 301 https://paperless.enkicsifelhom.hu/r1033-check?q=1
|
||||
OK http://paste.enkicsifelhom.hu/r1033-check?q=1 -> 301 https://paste.enkicsifelhom.hu/r1033-check?q=1
|
||||
OK http://wiki.enkicsifelhom.hu/r1033-check?q=1 -> 301 https://wiki.enkicsifelhom.hu/r1033-check?q=1
|
||||
== demo-hp (enkisfelhom.hu): 15 public name(s)
|
||||
OK http://arcade.enkisfelhom.hu/r1033-check?q=1 -> 301 https://arcade.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://books.enkisfelhom.hu/r1033-check?q=1 -> 301 https://books.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://docs.enkisfelhom.hu/r1033-check?q=1 -> 301 https://docs.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://felhom.enkisfelhom.hu/r1033-check?q=1 -> 301 https://felhom.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://files.enkisfelhom.hu/r1033-check?q=1 -> 301 https://files.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://gist.enkisfelhom.hu/r1033-check?q=1 -> 301 https://gist.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://ha.enkisfelhom.hu/r1033-check?q=1 -> 301 https://ha.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://no-such-name-r1033.enkisfelhom.hu/r1033-check?q=1 -> 301 https://no-such-name-r1033.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://papir.enkisfelhom.hu/r1033-check?q=1 -> 301 https://papir.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://paste.enkisfelhom.hu/r1033-check?q=1 -> 301 https://paste.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://pdf.enkisfelhom.hu/r1033-check?q=1 -> 301 https://pdf.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://plex.enkisfelhom.hu/r1033-check?q=1 -> 301 https://plex.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://time.enkisfelhom.hu/r1033-check?q=1 -> 301 https://time.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://travel.enkisfelhom.hu/r1033-check?q=1 -> 301 https://travel.enkisfelhom.hu/r1033-check?q=1
|
||||
OK http://wiki.enkisfelhom.hu/r1033-check?q=1 -> 301 https://wiki.enkisfelhom.hu/r1033-check?q=1
|
||||
== demo-felhom (demo-felhom.eu): 8 public name(s)
|
||||
OK http://felhom.demo-felhom.eu/r1033-check?q=1 -> 301 https://felhom.demo-felhom.eu/r1033-check?q=1
|
||||
OK http://files.demo-felhom.eu/r1033-check?q=1 -> 301 https://files.demo-felhom.eu/r1033-check?q=1
|
||||
OK http://gist.demo-felhom.eu/r1033-check?q=1 -> 301 https://gist.demo-felhom.eu/r1033-check?q=1
|
||||
OK http://no-such-name-r1033.demo-felhom.eu/r1033-check?q=1 -> 301 https://no-such-name-r1033.demo-felhom.eu/r1033-check?q=1
|
||||
OK http://prowlarr.demo-felhom.eu/r1033-check?q=1 -> 301 https://prowlarr.demo-felhom.eu/r1033-check?q=1
|
||||
OK http://radarr.demo-felhom.eu/r1033-check?q=1 -> 301 https://radarr.demo-felhom.eu/r1033-check?q=1
|
||||
OK http://sonarr.demo-felhom.eu/r1033-check?q=1 -> 301 https://sonarr.demo-felhom.eu/r1033-check?q=1
|
||||
OK http://torrent.demo-felhom.eu/r1033-check?q=1 -> 301 https://torrent.demo-felhom.eu/r1033-check?q=1
|
||||
== every public name redirects http -> https
|
||||
@@ -0,0 +1,7 @@
|
||||
controller: gitea.dooplex.hu/admin/felhom-controller:0.347.0-rc.1 Up About a minute (healthy)
|
||||
containers: 6 total
|
||||
not running/unhealthy: none
|
||||
refusals since 2026-10-29T08:53:00Z: 0
|
||||
ERROR lines since: 0
|
||||
login page: 200
|
||||
file manager: 200
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# walk_release.sh <ssh-host> <vmid> <domain> <since RFC3339> — the release box walk, read-only, from inside the guest:
|
||||
# the controller's image and health; every container up and not unhealthy; the controller log since the switch (refusals,
|
||||
# errors); the dashboard login page and the file manager through the box's own proxy (GET only).
|
||||
H=$1; V=$2; D=$3; SINCE=$4
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=15 "$H" "pct exec $V -- bash -s" <<EOF
|
||||
echo "controller: \$(docker ps --filter name=^/felhom-controller\$ --format '{{.Image}} {{.Status}}')"
|
||||
echo "containers: \$(docker ps -a --format '{{.Names}}|{{.Status}}' | wc -l) total"
|
||||
bad=\$(docker ps -a --format '{{.Names}}|{{.Status}}' | grep -E 'unhealthy|Restarting|Exited|Dead|Created' || true)
|
||||
echo "not running/unhealthy: \${bad:-none}"
|
||||
echo "refusals since \$(echo $SINCE): \$(docker logs --since $SINCE felhom-controller 2>&1 | grep -ciE 'REFUSED a catalog definition|app_definition_refused|definition refused')"
|
||||
echo "ERROR lines since: \$(docker logs --since $SINCE felhom-controller 2>&1 | grep -c '\[ERROR\]')"
|
||||
docker logs --since $SINCE felhom-controller 2>&1 | grep '\[ERROR\]' | cut -c1-220 | tail -5
|
||||
echo "login page: \$(curl -sk -o /dev/null -w '%{http_code}' -H 'Host: felhom.$D' https://127.0.0.1/login)"
|
||||
echo "file manager: \$(curl -sk -o /dev/null -w '%{http_code}' -H 'Host: files.$D' https://127.0.0.1/)"
|
||||
EOF
|
||||
Reference in new issue
Block a user