admin/felhom.eu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Campaign 10: two run-2a violations were HARNESS defects, not product defects — fixed

Browse Source 
Run 2a hit its first two violations at cycle 10 and BOTH trace to my harness, not
the product. Recorded in full because a check that fails for the wrong reason is
as corrosive as one that passes for the wrong reason.

  HARD-RESET  VM returned=True canaries_intact=False
  I7          want=C10-C010-A-194530 got=C10-C009-A-192929 restore_ok=True

Root cause, evidenced: the cc_proof table's highest row is C10-C009-A — there is
NO C010-A row at all, so the seed never landed. The hard-reset atom ran earlier in
the same cycle and left rallly Exited(255); atom_restore_verify called seed() and
never checked its return value, so an unwritten generation became a fake stale
This commit is contained in:
authentik Default Admin
2026-08-01 19:51:57 +02:00
parent ac6c05bd7b
commit 9ca57e591b
8 changed files with 1973 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
documentation/tests/campaign10-evidence-2026-07-31/runner/c10lib.py
+39 -2
View File
@@ -96,8 +96,11 @@ def _psql(sql, timeout=180):
return rc, out.strip(), err.strip()
def seed(gen):
"""Write the generation marker into every app. Returns {app: ok}."""
def seed(gen, verify=True):
"""Write the generation marker into every app. Returns {app: ok}.
With verify=True the value is READ BACK — a write that reported success but did not
land is the failure mode that turned a dead app into a fake 'stale restore' in run 2a."""
res = {}
_psql("CREATE TABLE IF NOT EXISTS cc_proof (id serial primary key, gen text, at timestamptz default now())")
rc, out, err = _psql(f"INSERT INTO cc_proof (gen) VALUES ('{gen}') RETURNING gen")
@@ -109,6 +112,10 @@ def seed(gen):
rc, out, err = guest(
f"docker exec -u 0 {cont} sh -c {shq(f'echo {gen} > {d}/cc_proof.txt && cat {d}/cc_proof.txt')}", 180)
res[app] = (gen in out)
if verify:
back = read_canaries()
for a in list(res):
res[a] = res[a] and (back.get(a) == gen)
return res
@@ -124,6 +131,36 @@ def read_canaries():
return res
def apps_ready(names=None, timeout=600):
"""Wait until the named app containers are RUNNING (healthy where they report health).
Load-bearing: a canary lives inside an app container, so reading one before the app is
back does not measure the product — it measures the harness. Two false violations in
run 2a came from exactly that (a seed that never landed, then a 'stale' restore)."""
names = names or (ALL_APPS + [DRIVE_APP])
t0 = time.time()
while time.time() - t0 < timeout:
c = containers()
ok = True
for n in names:
probe = "rallly-postgres" if n == "rallly" else n
st = c.get(n) or ""
st2 = c.get(probe) or ""
if not st or "Up" not in st or "starting" in st:
ok = False
break
if n == "rallly" and ("Up" not in st2 or "starting" in st2):
ok = False
break
if ok:
return True
time.sleep(10)
return False
DRIVE_APP = "calibre-web"
def containers():
rc, out, _ = guest("docker ps --format '{{.Names}}|{{.Status}}'", 120)
d = {}
documentation/tests/campaign10-evidence-2026-07-31/runner/c10run.py
+161 -6
View File
@@ -118,12 +118,24 @@ def atom_backup():
def atom_restore_verify():
"""I7 — the whole point: a restore must return the discriminator asked for."""
app = "rallly" # the DB app; its canary is a real DB row
# PRECONDITION, not politeness: a canary lives inside an app container. Reading or writing
# one before the app is back measures the harness, not the product. Run 2a produced a fake
# "stale restore" this exact way, after a hard reset left rallly down.
if not L.apps_ready([app]):
rec("I7-SKIP", True, "apps not ready before seed — atom skipped, NOT counted as a violation")
return
genA = "C10-C%03d-A-%s" % (CYCLE, time.strftime("%H%M%S"))
L.seed(genA)
sa = L.seed(genA)
if not sa.get(app):
rec("I7-SKIP", True, "seed of %s did not land (app unhealthy) — atom skipped, not a violation" % genA)
return
if not atom_backup():
return
genB = "C10-C%03d-B-%s" % (CYCLE, time.strftime("%H%M%S"))
L.seed(genB) # must NOT survive the restore
sb = L.seed(genB) # must NOT survive the restore
if not sb.get(app):
rec("I7-SKIP", True, "seed of %s did not land — cannot distinguish stale from unwritten" % genB)
return
snaps = L.jparse(L.api("GET", "/api/backup/snapshots?stack=%s" % app)) or {}
pts = snaps.get("data") or []
if not pts:
@@ -216,6 +228,142 @@ def atom_redeploy_app():
rec("REDEPLOY", app in L.containers(), "%s redeployed" % app)
RAW_ADATOK = "/mnt/adatok"
ADATOK_UUID = "13a43656-b393-4879-8d05-37de560def5f"
def _adatok_row():
for d in disks():
if str(d.get("guest_path", "")).endswith("adatok"):
return d
return {}
def atom_abort_fs_inplace():
"""R-117 Q7 / I4 second clause — the filesystem dies WITHOUT the device disappearing.
Before agent v0.117.0 this state reported healthy and restarted apps onto it with
nothing emitted on any channel. Assert it SURFACES and does not silently retry."""
L.box("mount -o remount,abort %s" % RAW_ADATOK)
opts = L.box("grep ' %s ' /proc/mounts" % RAW_ADATOK)[1].strip()
if not ("abort" in opts or "emergency_ro" in opts):
rec("I4-abort", False, "could not abort the fs in place; /proc/mounts=%s" % opts[:120])
return
settle(150)
row = _adatok_row()
dev_present = "sd" in L.box("lsblk -dno NAME | tr '\\n' ' '")[1]
rec("I4-abort", row.get("bound_under_parent") is False and dev_present,
"aborted-in-place: bound=%s (want False), device still present=%s, opts=%s"
% (row.get("bound_under_parent"), dev_present, opts.split("ext4")[-1][:60]))
# the gate must STOP the app whose data is on the dead namespace, not restart it onto it
rec("I3-abort", DRIVE_APP not in L.containers(),
"%s must be stopped while its namespace is aborted; running=%s"
% (DRIVE_APP, DRIVE_APP in L.containers()))
# recovery needs a full device close — a remount lands on the same aborted superblock
L.hp("qm set 311 --delete %s" % NONTGT_DISK)
time.sleep(8)
L.hp("qm set 311 --%s %s" % (NONTGT_DISK, NONTGT_VOL))
settle(150)
row = _adatok_row()
rec("I4-abort-recover", row.get("bound_under_parent") is True and DRIVE_APP in L.containers(),
"after re-seat: bound=%s app_up=%s" % (row.get("bound_under_parent"), DRIVE_APP in L.containers()))
def atom_kill_agent_mid_backup():
"""Kill the AGENT while a backup runs, then prove the next backup still succeeds."""
L.api("POST", "/api/backup/run", {})
time.sleep(6)
L.box("systemctl kill -s KILL felhom-agent || true")
time.sleep(20)
L.box("systemctl start felhom-agent || true")
wait_backup()
time.sleep(20)
up = "active" in L.box("systemctl is-active felhom-agent")[1]
ok2 = atom_backup()
rec("KILL-AGENT", up and ok2, "agent active after kill=%s; subsequent backup ok=%s" % (up, ok2))
def atom_hard_reset_mid_write():
"""Hard-reset the VM mid-backup. Assert it returns AND the canaries survive intact."""
before = L.read_canaries()
L.api("POST", "/api/backup/run", {})
time.sleep(8)
L.hp("qm reset 311")
ok = False
for _ in range(40):
time.sleep(20)
if "felhom-controller" in L.containers():
ok = True
break
# the canaries live in the APP containers — wait for those, or this measures startup latency
ready = L.apps_ready() if ok else False
after = L.read_canaries() if ready else {}
intact = all(after.get(a) == before.get(a) for a in before) if ready else None
detail = "VM returned=%s apps_ready=%s canaries_intact=%s" % (ok, ready, intact)
if not ready:
rec("HARD-RESET", False, detail + " — apps did not return within 10 min (that IS the failure)")
else:
rec("HARD-RESET", bool(intact), detail + " (a hard reset mid-write must not corrupt them)")
def atom_reboot_vm():
L.hp("qm reboot 311")
ok = False
for _ in range(40):
time.sleep(20)
if "felhom-controller" in L.containers():
ok = True
break
rec("REBOOT-VM", ok, "controller back after full VM reboot")
def atom_concurrent_backup_restore():
"""The controller must REFUSE the second op, not run both over the same data."""
L.api("POST", "/api/backup/run", {})
time.sleep(3)
out = L.guest('/root/c10api.sh POST /backup/restore "stack_name=rallly&snapshot_id=helyi" 2>/dev/null')[1]
st = ((L.jparse(L.api("GET", "/api/backup/restore-status")) or {}).get("data") or {})
wait_backup(); wait_restore()
# either the restore was refused, or it queued behind the backup — never both mutating at once
rec("CONCURRENCY", True, "backup+restore raced; restore-status op=%s running=%s (recorded, not asserted)"
% (st.get("op"), st.get("running")))
def atom_concurrent_backup_detach():
"""Pull the backup TARGET out from under a running backup."""
L.api("POST", "/api/backup/run", {})
time.sleep(5)
L.hp("qm set 311 --delete %s" % TARGET_DISK)
settle()
evs = [e for e in hub_events("4m") if e[0] in ("backup_target_absent", "storage_disconnected")]
got = evs[-1][0] if evs else None
rec("I1-under-load", got == "backup_target_absent",
"target pulled DURING a backup -> got=%s (want backup_target_absent)" % got)
L.hp("qm set 311 --%s %s" % (TARGET_DISK, TARGET_VOL))
settle()
wait_backup()
ts = target_state()
rec("I1-under-load-recover", ts.get("degraded") is False,
"after return degraded=%s target=%s" % (ts.get("degraded"), ts.get("target")))
def atom_fill_drive():
"""Fill the backup target to near-full and back up. A refusal is a PASS; a silent
success that writes nothing is the failure this looks for."""
avail = L.box("df --output=avail -BM /mnt/mentes | tail -1")[1].strip().rstrip("M")
try:
keep = max(int(avail) - 300, 50)
except Exception:
keep = 50
L.box("fallocate -l %dM /mnt/mentes/c10-filler 2>&1 || dd if=/dev/zero of=/mnt/mentes/c10-filler bs=1M count=%d 2>/dev/null" % (keep, keep))
free_after = L.box("df -h /mnt/mentes | tail -1")[1].strip()
d = ((L.jparse(L.api("GET", "/api/backup/status")) or {}).get("data") or {})
L.api("POST", "/api/backup/run", {})
res = wait_backup()
L.box("rm -f /mnt/mentes/c10-filler")
ok = res.get("db_dump", {}).get("success") is not None
rec("FILL-DRIVE", ok, "near-full (%s) backup outcome recorded: %s" % (free_after[-24:], json.dumps(res)[:200]))
# ------------------------------------------------------------------ per-cycle invariants
def check_healthy_baseline():
@@ -285,9 +433,16 @@ def watchdog():
# ------------------------------------------------------------------ main
# LIGHT atoms run every cycle. HEAVY ones are slow (VM reboots, fills, aborts), so a
# rotating slice joins each cycle and is SHUFFLED IN with the rest — the v1 runner appended
# reboot after the shuffle, so it never interleaved with a detach. That is fixed here.
ATOMS = [atom_backup, atom_restore_verify, atom_detach_target,
atom_detach_nontarget, atom_kill_controller, atom_redeploy_app]
HEAVY = [atom_abort_fs_inplace, atom_kill_agent_mid_backup, atom_hard_reset_mid_write,
atom_reboot_vm, atom_reboot_guest, atom_concurrent_backup_restore,
atom_concurrent_backup_detach, atom_fill_drive]
def main():
global CYCLE
log("=== Campaign 10 Phase B start — max %d cycles, deadline %s ==="
@@ -302,10 +457,10 @@ def main():
if not watchdog():
break
order = ATOMS[:]
rng.shuffle(order) # permutation, not a fixed catalogue
# a reboot is expensive; fold it in every 7th cycle only
if CYCLE % 7 == 0:
order.append(atom_reboot_guest)
# ONE heavy atom per cycle: every heavy atom is seen every 8 cycles, and the cycle
# stays short enough to reach the tail the brief cares about (drift at ~38).
order += [HEAVY[CYCLE % len(HEAVY)]]
rng.shuffle(order) # permutation INCLUDING the heavy atoms
log("cycle %d: %s" % (CYCLE, " -> ".join(a.__name__.replace("atom_", "") for a in order)))
for a in order:
heartbeat(a.__name__)
documentation/tests/campaign10-evidence-2026-07-31/state/growth.tsv
+1521
View File
File diff suppressed because it is too large Load Diff
documentation/tests/campaign10-evidence-2026-07-31/state/journal-run2a.tsv
+217
View File
@@ -0,0 +1,217 @@
cycle utc invariant verdict detail
1 2026-08-01T15:13:03Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
1 2026-08-01T15:13:06Z I3 PASS adatok bound_under_parent=False while absent
1 2026-08-01T15:15:18Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
1 2026-08-01T15:15:21Z I4 PASS guest init boot before=1785586762 after=1785586762 (must be equal)
1 2026-08-01T15:16:45Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.552098165s", "last_run": "2026-08-01T15:16:45.729591527Z", "success": true}, "enabled": true, "running": false}
1 2026-08-01T15:16:45Z KILL-AGENT PASS agent active after kill=True; subsequent backup ok=True
1 2026-08-01T15:17:28Z BACKUP PASS {"db_dump": {"count": 1, "duration": "33.726608159s", "last_run": "2026-08-01T15:17:21.684974017Z", "success": true}, "enabled": true, "running": false}
1 2026-08-01T15:18:21Z BACKUP PASS {"db_dump": {"count": 1, "duration": "32.769576397s", "last_run": "2026-08-01T15:18:14.088210902Z", "success": true}, "enabled": true, "running": false}
1 2026-08-01T15:19:24Z I7 PASS app=rallly want=C10-C001-A-171728 got=C10-C001-A-171728 snap=helyi restore_ok=True rto=41.5s
1 2026-08-01T15:21:39Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
1 2026-08-01T15:21:42Z I3 PASS mentes bound_under_parent=False while absent
1 2026-08-01T15:23:55Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
1 2026-08-01T15:23:58Z I4 PASS guest init boot before=1785586762 after=1785586762 (must be equal)
1 2026-08-01T15:24:42Z REDEPLOY PASS grafana redeployed
1 2026-08-01T15:25:31Z KILL-CTRL PASS controller recovered after kill
1 2026-08-01T15:25:34Z I5 PASS degraded=False absent_events=0 target=felhom-backup
1 2026-08-01T15:25:34Z I6 PASS healthy state must carry no degraded message; got=None
1 2026-08-01T15:25:36Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
1 2026-08-01T15:25:36Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
1 2026-08-01T15:25:38Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
1 2026-08-01T15:25:40Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
1 2026-08-01T15:25:42Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
1 2026-08-01T15:25:44Z I11 PASS leaked=[]
2 2026-08-01T15:26:30Z REDEPLOY PASS homebox redeployed
2 2026-08-01T15:28:46Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
2 2026-08-01T15:28:49Z I3 PASS adatok bound_under_parent=False while absent
2 2026-08-01T15:31:01Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
2 2026-08-01T15:31:04Z I4 PASS guest init boot before=1785586762 after=1785586762 (must be equal)
2 2026-08-01T15:31:53Z KILL-CTRL PASS controller recovered after kill
2 2026-08-01T15:32:23Z BACKUP PASS {"db_dump": {"count": 1, "duration": "26.780169718s", "last_run": "2026-08-01T15:32:22.800970045Z", "success": true}, "enabled": true, "running": false}
2 2026-08-01T15:35:56Z HARD-RESET PASS VM returned=True canaries_intact=True (a hard reset mid-write must not corrupt them)
2 2026-08-01T15:36:49Z BACKUP PASS {"db_dump": {"count": 1, "duration": "35.398749738s", "last_run": "2026-08-01T15:36:44.432418682Z", "success": true}, "enabled": true, "running": false}
2 2026-08-01T15:37:51Z I7 PASS app=rallly want=C10-C002-A-173556 got=C10-C002-A-173556 snap=helyi restore_ok=True rto=41.8s
2 2026-08-01T15:40:07Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
2 2026-08-01T15:40:10Z I3 PASS mentes bound_under_parent=False while absent
2 2026-08-01T15:42:23Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
2 2026-08-01T15:42:25Z I4 PASS guest init boot before=1785598524 after=1785598524 (must be equal)
2 2026-08-01T15:42:27Z I5 PASS degraded=False absent_events=0 target=felhom-backup
2 2026-08-01T15:42:27Z I6 PASS healthy state must carry no degraded message; got=None
2 2026-08-01T15:42:29Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
2 2026-08-01T15:42:29Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
2 2026-08-01T15:42:31Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
2 2026-08-01T15:42:33Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
2 2026-08-01T15:42:35Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
2 2026-08-01T15:42:37Z I11 PASS leaked=[]
3 2026-08-01T15:43:09Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.204135618s", "last_run": "2026-08-01T15:43:08.819591677Z", "success": true}, "enabled": true, "running": false}
3 2026-08-01T15:43:53Z REDEPLOY PASS grafana redeployed
3 2026-08-01T15:46:09Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
3 2026-08-01T15:46:12Z I3 PASS mentes bound_under_parent=False while absent
3 2026-08-01T15:48:25Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
3 2026-08-01T15:48:27Z I4 PASS guest init boot before=1785598524 after=1785598524 (must be equal)
3 2026-08-01T15:49:46Z REBOOT-VM PASS controller back after full VM reboot
3 2026-08-01T15:52:03Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
3 2026-08-01T15:52:05Z I3 PASS adatok bound_under_parent=False while absent
3 2026-08-01T15:54:18Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
3 2026-08-01T15:54:20Z I4 PASS guest init boot before=1785599371 after=1785599371 (must be equal)
3 2026-08-01T15:55:09Z KILL-CTRL PASS controller recovered after kill
3 2026-08-01T15:56:01Z BACKUP PASS {"db_dump": {"count": 1, "duration": "28.761489809s", "last_run": "2026-08-01T15:55:50.887539787Z", "success": true}, "enabled": true, "running": false}
3 2026-08-01T15:56:52Z I7 PASS app=rallly want=C10-C003-A-175509 got=C10-C003-A-175509 snap=helyi restore_ok=True rto=29.4s
3 2026-08-01T15:56:55Z I5 PASS degraded=False absent_events=0 target=felhom-backup
3 2026-08-01T15:56:55Z I6 PASS healthy state must carry no degraded message; got=None
3 2026-08-01T15:56:56Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
3 2026-08-01T15:56:56Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
3 2026-08-01T15:56:58Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
3 2026-08-01T15:57:00Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
3 2026-08-01T15:57:03Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
3 2026-08-01T15:57:05Z I11 PASS leaked=[]
4 2026-08-01T15:59:23Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
4 2026-08-01T15:59:26Z I3 PASS adatok bound_under_parent=False while absent
4 2026-08-01T16:01:39Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
4 2026-08-01T16:01:42Z I4 PASS guest init boot before=1785599371 after=1785599371 (must be equal)
4 2026-08-01T16:02:25Z REDEPLOY PASS homebox redeployed
4 2026-08-01T16:02:55Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.332969204s", "last_run": "2026-08-01T16:02:54.940636984Z", "success": true}, "enabled": true, "running": false}
4 2026-08-01T16:03:33Z REBOOT PASS controller back after guest reboot
4 2026-08-01T16:04:27Z BACKUP PASS {"db_dump": {"count": 1, "duration": "35.523721413s", "last_run": "2026-08-01T16:04:23.027197293Z", "success": true}, "enabled": true, "running": false}
4 2026-08-01T16:05:30Z I7 PASS app=rallly want=C10-C004-A-180333 got=C10-C004-A-180333 snap=helyi restore_ok=True rto=41.2s
4 2026-08-01T16:07:46Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
4 2026-08-01T16:07:49Z I3 PASS mentes bound_under_parent=False while absent
4 2026-08-01T16:10:02Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
4 2026-08-01T16:10:05Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
4 2026-08-01T16:10:55Z KILL-CTRL PASS controller recovered after kill
4 2026-08-01T16:10:57Z I5 PASS degraded=False absent_events=0 target=felhom-backup
4 2026-08-01T16:10:57Z I6 PASS healthy state must carry no degraded message; got=None
4 2026-08-01T16:10:59Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
4 2026-08-01T16:10:59Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
4 2026-08-01T16:11:01Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
4 2026-08-01T16:11:03Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
4 2026-08-01T16:11:05Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
4 2026-08-01T16:11:07Z I11 PASS leaked=[]
5 2026-08-01T16:13:25Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
5 2026-08-01T16:13:28Z I3 PASS adatok bound_under_parent=False while absent
5 2026-08-01T16:15:41Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
5 2026-08-01T16:15:44Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
5 2026-08-01T16:16:14Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.569317719s", "last_run": "2026-08-01T16:16:14.83566107Z", "success": true}, "enabled": true, "running": false}
5 2026-08-01T16:16:55Z CONCURRENCY PASS backup+restore raced; restore-status op=None running=False (recorded, not asserted)
5 2026-08-01T16:17:39Z REDEPLOY PASS grafana redeployed
5 2026-08-01T16:18:29Z KILL-CTRL PASS controller recovered after kill
5 2026-08-01T16:19:09Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.097117609s", "last_run": "2026-08-01T16:19:08.609967049Z", "success": true}, "enabled": true, "running": false}
5 2026-08-01T16:20:14Z I7 PASS app=rallly want=C10-C005-A-181829 got=C10-C005-A-181829 snap=helyi restore_ok=True rto=43.7s
5 2026-08-01T16:22:30Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
5 2026-08-01T16:22:33Z I3 PASS mentes bound_under_parent=False while absent
5 2026-08-01T16:24:46Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
5 2026-08-01T16:24:48Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
5 2026-08-01T16:24:51Z I5 PASS degraded=False absent_events=0 target=felhom-backup
5 2026-08-01T16:24:51Z I6 PASS healthy state must carry no degraded message; got=None
5 2026-08-01T16:24:53Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
5 2026-08-01T16:24:53Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
5 2026-08-01T16:24:56Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
5 2026-08-01T16:24:58Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
5 2026-08-01T16:25:00Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
5 2026-08-01T16:25:02Z I11 PASS leaked=[]
6 2026-08-01T16:25:45Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.112658529s", "last_run": "2026-08-01T16:25:44.15087597Z", "success": true}, "enabled": true, "running": false}
6 2026-08-01T16:26:50Z I7 PASS app=rallly want=C10-C006-A-182504 got=C10-C006-A-182504 snap=helyi restore_ok=True rto=41.9s
6 2026-08-01T16:29:05Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
6 2026-08-01T16:29:07Z I3 PASS mentes bound_under_parent=False while absent
6 2026-08-01T16:31:20Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
6 2026-08-01T16:31:23Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
6 2026-08-01T16:31:55Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.235289153s", "last_run": "2026-08-01T16:31:54.107505287Z", "success": true}, "enabled": true, "running": false}
6 2026-08-01T16:34:16Z I1-under-load PASS target pulled DURING a backup -> got=backup_target_absent (want backup_target_absent)
6 2026-08-01T16:36:33Z I1-under-load-recover PASS after return degraded=False target=felhom-backup
6 2026-08-01T16:37:23Z KILL-CTRL PASS controller recovered after kill
6 2026-08-01T16:38:08Z REDEPLOY PASS homebox redeployed
6 2026-08-01T16:40:25Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
6 2026-08-01T16:40:28Z I3 PASS adatok bound_under_parent=False while absent
6 2026-08-01T16:42:40Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
6 2026-08-01T16:42:43Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
6 2026-08-01T16:42:46Z I5 PASS degraded=False absent_events=0 target=felhom-backup
6 2026-08-01T16:42:46Z I6 PASS healthy state must carry no degraded message; got=None
6 2026-08-01T16:42:48Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
6 2026-08-01T16:42:48Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
6 2026-08-01T16:42:50Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
6 2026-08-01T16:42:52Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
6 2026-08-01T16:42:55Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
6 2026-08-01T16:42:58Z I11 PASS leaked=[]
7 2026-08-01T16:43:50Z REDEPLOY PASS grafana redeployed
7 2026-08-01T16:44:40Z KILL-CTRL PASS controller recovered after kill
7 2026-08-01T16:45:27Z BACKUP PASS {"db_dump": {"count": 1, "duration": "26.964029153s", "last_run": "2026-08-01T16:45:24.923590721Z", "success": true}, "enabled": true, "running": false}
7 2026-08-01T16:46:32Z I7 PASS app=rallly want=C10-C007-A-184440 got=C10-C007-A-184440 snap=helyi restore_ok=True rto=33.4s
7 2026-08-01T16:47:12Z FILL-DRIVE PASS near-full (G 300M 100% /mnt/mentes) backup outcome recorded: {"db_dump": {"count": 1, "duration": "27.484620261s", "last_run": "2026-08-01T16:47:10.818330567Z", "success": true}, "enabled": true, "running": false}
7 2026-08-01T16:49:29Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
7 2026-08-01T16:49:33Z I3 PASS mentes bound_under_parent=False while absent
7 2026-08-01T16:51:46Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
7 2026-08-01T16:51:49Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
7 2026-08-01T16:54:06Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
7 2026-08-01T16:54:09Z I3 PASS adatok bound_under_parent=False while absent
7 2026-08-01T16:56:23Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
7 2026-08-01T16:56:28Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
7 2026-08-01T16:57:05Z BACKUP PASS {"db_dump": {"count": 1, "duration": "26.9201682s", "last_run": "2026-08-01T16:56:59.62688952Z", "success": true}, "enabled": true, "running": false}
7 2026-08-01T16:57:08Z I5 PASS degraded=False absent_events=0 target=felhom-backup
7 2026-08-01T16:57:08Z I6 PASS healthy state must carry no degraded message; got=None
7 2026-08-01T16:57:11Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
7 2026-08-01T16:57:11Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
7 2026-08-01T16:57:13Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
7 2026-08-01T16:57:17Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
7 2026-08-01T16:57:22Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
7 2026-08-01T16:57:24Z I11 PASS leaked=[]
8 2026-08-01T16:58:14Z REDEPLOY PASS homebox redeployed
8 2026-08-01T16:59:05Z KILL-CTRL PASS controller recovered after kill
8 2026-08-01T16:59:55Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.413465748s", "last_run": "2026-08-01T16:59:52.777435129Z", "success": true}, "enabled": true, "running": false}
8 2026-08-01T17:01:12Z I7 PASS app=rallly want=C10-C008-A-185905 got=C10-C008-A-185905 snap=helyi restore_ok=True rto=46.6s
8 2026-08-01T17:01:53Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.089524734s", "last_run": "2026-08-01T17:01:43.166606797Z", "success": true}, "enabled": true, "running": false}
8 2026-08-01T17:04:16Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
8 2026-08-01T17:04:25Z I3 PASS mentes bound_under_parent=False while absent
8 2026-08-01T17:06:41Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
8 2026-08-01T17:06:45Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
8 2026-08-01T17:09:24Z I4-abort PASS aborted-in-place: bound=False (want False), device still present=True, opts= rw,relatime,abort,emergency_ro 0 0
8 2026-08-01T17:09:31Z I3-abort PASS calibre-web must be stopped while its namespace is aborted; running=False
8 2026-08-01T17:12:29Z I4-abort-recover PASS after re-seat: bound=True app_up=True
8 2026-08-01T17:14:47Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
8 2026-08-01T17:14:51Z I3 PASS adatok bound_under_parent=False while absent
8 2026-08-01T17:17:03Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
8 2026-08-01T17:17:06Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
8 2026-08-01T17:17:08Z I5 PASS degraded=False absent_events=0 target=felhom-backup
8 2026-08-01T17:17:08Z I6 PASS healthy state must carry no degraded message; got=None
8 2026-08-01T17:17:10Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
8 2026-08-01T17:17:10Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
8 2026-08-01T17:17:12Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
8 2026-08-01T17:17:15Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
8 2026-08-01T17:17:17Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
8 2026-08-01T17:17:19Z I11 PASS leaked=[]
9 2026-08-01T17:18:05Z REDEPLOY PASS grafana redeployed
9 2026-08-01T17:18:56Z KILL-CTRL PASS controller recovered after kill
9 2026-08-01T17:20:21Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.162532868s", "last_run": "2026-08-01T17:20:21.182603993Z", "success": true}, "enabled": true, "running": false}
9 2026-08-01T17:20:21Z KILL-AGENT PASS agent active after kill=True; subsequent backup ok=True
9 2026-08-01T17:22:37Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
9 2026-08-01T17:22:40Z I3 PASS adatok bound_under_parent=False while absent
9 2026-08-01T17:24:53Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
9 2026-08-01T17:24:56Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
9 2026-08-01T17:27:12Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
9 2026-08-01T17:27:14Z I3 PASS mentes bound_under_parent=False while absent
9 2026-08-01T17:29:27Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
9 2026-08-01T17:29:29Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
9 2026-08-01T17:30:08Z BACKUP PASS {"db_dump": {"count": 1, "duration": "26.973542618s", "last_run": "2026-08-01T17:30:08.443533111Z", "success": true}, "enabled": true, "running": false}
9 2026-08-01T17:31:11Z I7 PASS app=rallly want=C10-C009-A-192929 got=C10-C009-A-192929 snap=helyi restore_ok=True rto=41.1s
9 2026-08-01T17:31:52Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.327749164s", "last_run": "2026-08-01T17:31:40.462537043Z", "success": true}, "enabled": true, "running": false}
9 2026-08-01T17:31:55Z I5 PASS degraded=False absent_events=0 target=felhom-backup
9 2026-08-01T17:31:55Z I6 PASS healthy state must carry no degraded message; got=None
9 2026-08-01T17:31:57Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
9 2026-08-01T17:31:57Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
9 2026-08-01T17:31:59Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
9 2026-08-01T17:32:00Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
9 2026-08-01T17:32:02Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
9 2026-08-01T17:32:05Z I11 PASS leaked=[]
10 2026-08-01T17:34:22Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
10 2026-08-01T17:34:25Z I3 PASS mentes bound_under_parent=False while absent
10 2026-08-01T17:36:38Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
10 2026-08-01T17:36:41Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
10 2026-08-01T17:38:57Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
10 2026-08-01T17:38:59Z I3 PASS adatok bound_under_parent=False while absent
10 2026-08-01T17:41:12Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
10 2026-08-01T17:41:14Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
10 2026-08-01T17:44:47Z HARD-RESET VIOLATION VM returned=True canaries_intact=False (a hard reset mid-write must not corrupt them)
10 2026-08-01T17:45:30Z REDEPLOY PASS homebox redeployed
10 2026-08-01T17:46:09Z BACKUP PASS {"db_dump": {"count": 1, "duration": "19.517467558s", "last_run": "2026-08-01T17:46:02.348224814Z", "success": true}, "enabled": true, "running": false}
10 2026-08-01T17:47:12Z I7 VIOLATION app=rallly want=C10-C010-A-194530 got=C10-C009-A-192929 snap=helyi restore_ok=True rto=41.0s
10 2026-08-01T17:47:52Z BACKUP PASS {"db_dump": {"count": 1, "duration": "28.898010904s", "last_run": "2026-08-01T17:47:42.890391148Z", "success": true}, "enabled": true, "running": false}
Can't render this file because it contains an unexpected character in line 6 and column 37.
documentation/tests/campaign10-evidence-2026-07-31/state/rto-run2a.tsv
+10
View File
@@ -0,0 +1,10 @@
1 2026-08-01T15:19:24Z rallly tier1 41.5 True
2 2026-08-01T15:37:51Z rallly tier1 41.8 True
3 2026-08-01T15:56:52Z rallly tier1 29.4 True
4 2026-08-01T16:05:30Z rallly tier1 41.2 True
5 2026-08-01T16:20:14Z rallly tier1 43.7 True
6 2026-08-01T16:26:50Z rallly tier1 41.9 True
7 2026-08-01T16:46:32Z rallly tier1 33.4 True
8 2026-08-01T17:01:12Z rallly tier1 46.6 True
9 2026-08-01T17:31:11Z rallly tier1 41.1 True
10 2026-08-01T17:47:12Z rallly tier1 41.0 False
1 1 2026-08-01T15:19:24Z rallly tier1 41.5 True
2 2 2026-08-01T15:37:51Z rallly tier1 41.8 True
3 3 2026-08-01T15:56:52Z rallly tier1 29.4 True
4 4 2026-08-01T16:05:30Z rallly tier1 41.2 True
5 5 2026-08-01T16:20:14Z rallly tier1 43.7 True
6 6 2026-08-01T16:26:50Z rallly tier1 41.9 True
7 7 2026-08-01T16:46:32Z rallly tier1 33.4 True
8 8 2026-08-01T17:01:12Z rallly tier1 46.6 True
9 9 2026-08-01T17:31:11Z rallly tier1 41.1 True
10 10 2026-08-01T17:47:12Z rallly tier1 41.0 False
documentation/tests/campaign10-evidence-2026-07-31/state/runner-run2a.log
+22
View File
@@ -0,0 +1,22 @@
17:10:45 === Campaign 10 Phase B start — max 45 cycles, deadline 07:10 ===
17:10:47 cycle 1: detach_nontarget -> kill_agent_mid_backup -> backup -> restore_verify -> detach_target -> redeploy_app -> kill_controller
17:25:44 cycle 1 done — violations so far: 0
17:25:46 cycle 2: redeploy_app -> detach_nontarget -> kill_controller -> backup -> hard_reset_mid_write -> restore_verify -> detach_target
17:42:37 cycle 2 done — violations so far: 0
17:42:39 cycle 3: backup -> redeploy_app -> detach_target -> reboot_vm -> detach_nontarget -> kill_controller -> restore_verify
17:57:05 cycle 3 done — violations so far: 0
17:57:07 cycle 4: detach_nontarget -> redeploy_app -> backup -> reboot_guest -> restore_verify -> detach_target -> kill_controller
18:11:07 cycle 4 done — violations so far: 0
18:11:10 cycle 5: detach_nontarget -> backup -> concurrent_backup_restore -> redeploy_app -> kill_controller -> restore_verify -> detach_target
18:25:02 cycle 5 done — violations so far: 0
18:25:04 cycle 6: restore_verify -> detach_target -> backup -> concurrent_backup_detach -> kill_controller -> redeploy_app -> detach_nontarget
18:42:58 cycle 6 done — violations so far: 0
18:43:00 cycle 7: redeploy_app -> kill_controller -> restore_verify -> fill_drive -> detach_target -> detach_nontarget -> backup
18:57:24 cycle 7 done — violations so far: 0
18:57:28 cycle 8: redeploy_app -> kill_controller -> restore_verify -> backup -> detach_target -> abort_fs_inplace -> detach_nontarget
19:17:19 cycle 8 done — violations so far: 0
19:17:21 cycle 9: redeploy_app -> kill_controller -> kill_agent_mid_backup -> detach_nontarget -> detach_target -> restore_verify -> backup
19:32:05 cycle 9 done — violations so far: 0
19:32:07 cycle 10: detach_target -> detach_nontarget -> hard_reset_mid_write -> redeploy_app -> restore_verify -> backup -> kill_controller
19:44:47 !! HARD-RESET VIOLATION: VM returned=True canaries_intact=False (a hard reset mid-write must not corrupt them)
19:47:12 !! I7 VIOLATION: app=rallly want=C10-C010-A-194530 got=C10-C009-A-192929 snap=helyi restore_ok=True rto=41.0s
documentation/tests/campaign10-evidence-2026-07-31/state/runner.log
+2 -2
View File
@@ -1,2 +1,2 @@
17:10:45 === Campaign 10 Phase B start — max 45 cycles, deadline 07:10 ===
17:10:47 cycle 1: detach_nontarget -> kill_agent_mid_backup -> backup -> restore_verify -> detach_target -> redeploy_app -> kill_controller
19:51:13 === Campaign 10 Phase B start — max 45 cycles, deadline 06:51 ===
19:51:15 cycle 1: detach_nontarget -> kill_agent_mid_backup -> backup -> restore_verify -> detach_target -> redeploy_app -> kill_controller
documentation/tests/campaign10-evidence-2026-07-31/state/status.txt
+1 -1
View File
@@ -1,4 +1,4 @@
cycle=1
phase=atom_detach_nontarget
utc=2026-08-01T15:10:47Z
utc=2026-08-01T17:51:15Z
violations=0