Campaign 10: two run-2a violations were HARNESS defects, not product defects — fixed
Run 2a hit its first two violations at cycle 10 and BOTH trace to my harness, not the product. Recorded in full because a check that fails for the wrong reason is as corrosive as one that passes for the wrong reason. HARD-RESET VM returned=True canaries_intact=False I7 want=C10-C010-A-194530 got=C10-C009-A-192929 restore_ok=True Root cause, evidenced: the cc_proof table's highest row is C10-C009-A — there is NO C010-A row at all, so the seed never landed. The hard-reset atom ran earlier in the same cycle and left rallly Exited(255); atom_restore_verify called seed() and never checked its return value, so an unwritten generation became a fake stale
This commit is contained in:
@@ -96,8 +96,11 @@ def _psql(sql, timeout=180):
|
|||||||
return rc, out.strip(), err.strip()
|
return rc, out.strip(), err.strip()
|
||||||
|
|
||||||
|
|
||||||
def seed(gen):
|
def seed(gen, verify=True):
|
||||||
"""Write the generation marker into every app. Returns {app: ok}."""
|
"""Write the generation marker into every app. Returns {app: ok}.
|
||||||
|
|
||||||
|
With verify=True the value is READ BACK — a write that reported success but did not
|
||||||
|
land is the failure mode that turned a dead app into a fake 'stale restore' in run 2a."""
|
||||||
res = {}
|
res = {}
|
||||||
_psql("CREATE TABLE IF NOT EXISTS cc_proof (id serial primary key, gen text, at timestamptz default now())")
|
_psql("CREATE TABLE IF NOT EXISTS cc_proof (id serial primary key, gen text, at timestamptz default now())")
|
||||||
rc, out, err = _psql(f"INSERT INTO cc_proof (gen) VALUES ('{gen}') RETURNING gen")
|
rc, out, err = _psql(f"INSERT INTO cc_proof (gen) VALUES ('{gen}') RETURNING gen")
|
||||||
@@ -109,6 +112,10 @@ def seed(gen):
|
|||||||
rc, out, err = guest(
|
rc, out, err = guest(
|
||||||
f"docker exec -u 0 {cont} sh -c {shq(f'echo {gen} > {d}/cc_proof.txt && cat {d}/cc_proof.txt')}", 180)
|
f"docker exec -u 0 {cont} sh -c {shq(f'echo {gen} > {d}/cc_proof.txt && cat {d}/cc_proof.txt')}", 180)
|
||||||
res[app] = (gen in out)
|
res[app] = (gen in out)
|
||||||
|
if verify:
|
||||||
|
back = read_canaries()
|
||||||
|
for a in list(res):
|
||||||
|
res[a] = res[a] and (back.get(a) == gen)
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
@@ -124,6 +131,36 @@ def read_canaries():
|
|||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
def apps_ready(names=None, timeout=600):
|
||||||
|
"""Wait until the named app containers are RUNNING (healthy where they report health).
|
||||||
|
|
||||||
|
Load-bearing: a canary lives inside an app container, so reading one before the app is
|
||||||
|
back does not measure the product — it measures the harness. Two false violations in
|
||||||
|
run 2a came from exactly that (a seed that never landed, then a 'stale' restore)."""
|
||||||
|
names = names or (ALL_APPS + [DRIVE_APP])
|
||||||
|
t0 = time.time()
|
||||||
|
while time.time() - t0 < timeout:
|
||||||
|
c = containers()
|
||||||
|
ok = True
|
||||||
|
for n in names:
|
||||||
|
probe = "rallly-postgres" if n == "rallly" else n
|
||||||
|
st = c.get(n) or ""
|
||||||
|
st2 = c.get(probe) or ""
|
||||||
|
if not st or "Up" not in st or "starting" in st:
|
||||||
|
ok = False
|
||||||
|
break
|
||||||
|
if n == "rallly" and ("Up" not in st2 or "starting" in st2):
|
||||||
|
ok = False
|
||||||
|
break
|
||||||
|
if ok:
|
||||||
|
return True
|
||||||
|
time.sleep(10)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
DRIVE_APP = "calibre-web"
|
||||||
|
|
||||||
|
|
||||||
def containers():
|
def containers():
|
||||||
rc, out, _ = guest("docker ps --format '{{.Names}}|{{.Status}}'", 120)
|
rc, out, _ = guest("docker ps --format '{{.Names}}|{{.Status}}'", 120)
|
||||||
d = {}
|
d = {}
|
||||||
|
|||||||
@@ -118,12 +118,24 @@ def atom_backup():
|
|||||||
def atom_restore_verify():
|
def atom_restore_verify():
|
||||||
"""I7 — the whole point: a restore must return the discriminator asked for."""
|
"""I7 — the whole point: a restore must return the discriminator asked for."""
|
||||||
app = "rallly" # the DB app; its canary is a real DB row
|
app = "rallly" # the DB app; its canary is a real DB row
|
||||||
|
# PRECONDITION, not politeness: a canary lives inside an app container. Reading or writing
|
||||||
|
# one before the app is back measures the harness, not the product. Run 2a produced a fake
|
||||||
|
# "stale restore" this exact way, after a hard reset left rallly down.
|
||||||
|
if not L.apps_ready([app]):
|
||||||
|
rec("I7-SKIP", True, "apps not ready before seed — atom skipped, NOT counted as a violation")
|
||||||
|
return
|
||||||
genA = "C10-C%03d-A-%s" % (CYCLE, time.strftime("%H%M%S"))
|
genA = "C10-C%03d-A-%s" % (CYCLE, time.strftime("%H%M%S"))
|
||||||
L.seed(genA)
|
sa = L.seed(genA)
|
||||||
|
if not sa.get(app):
|
||||||
|
rec("I7-SKIP", True, "seed of %s did not land (app unhealthy) — atom skipped, not a violation" % genA)
|
||||||
|
return
|
||||||
if not atom_backup():
|
if not atom_backup():
|
||||||
return
|
return
|
||||||
genB = "C10-C%03d-B-%s" % (CYCLE, time.strftime("%H%M%S"))
|
genB = "C10-C%03d-B-%s" % (CYCLE, time.strftime("%H%M%S"))
|
||||||
L.seed(genB) # must NOT survive the restore
|
sb = L.seed(genB) # must NOT survive the restore
|
||||||
|
if not sb.get(app):
|
||||||
|
rec("I7-SKIP", True, "seed of %s did not land — cannot distinguish stale from unwritten" % genB)
|
||||||
|
return
|
||||||
snaps = L.jparse(L.api("GET", "/api/backup/snapshots?stack=%s" % app)) or {}
|
snaps = L.jparse(L.api("GET", "/api/backup/snapshots?stack=%s" % app)) or {}
|
||||||
pts = snaps.get("data") or []
|
pts = snaps.get("data") or []
|
||||||
if not pts:
|
if not pts:
|
||||||
@@ -216,6 +228,142 @@ def atom_redeploy_app():
|
|||||||
rec("REDEPLOY", app in L.containers(), "%s redeployed" % app)
|
rec("REDEPLOY", app in L.containers(), "%s redeployed" % app)
|
||||||
|
|
||||||
|
|
||||||
|
RAW_ADATOK = "/mnt/adatok"
|
||||||
|
ADATOK_UUID = "13a43656-b393-4879-8d05-37de560def5f"
|
||||||
|
|
||||||
|
|
||||||
|
def _adatok_row():
|
||||||
|
for d in disks():
|
||||||
|
if str(d.get("guest_path", "")).endswith("adatok"):
|
||||||
|
return d
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def atom_abort_fs_inplace():
|
||||||
|
"""R-117 Q7 / I4 second clause — the filesystem dies WITHOUT the device disappearing.
|
||||||
|
Before agent v0.117.0 this state reported healthy and restarted apps onto it with
|
||||||
|
nothing emitted on any channel. Assert it SURFACES and does not silently retry."""
|
||||||
|
L.box("mount -o remount,abort %s" % RAW_ADATOK)
|
||||||
|
opts = L.box("grep ' %s ' /proc/mounts" % RAW_ADATOK)[1].strip()
|
||||||
|
if not ("abort" in opts or "emergency_ro" in opts):
|
||||||
|
rec("I4-abort", False, "could not abort the fs in place; /proc/mounts=%s" % opts[:120])
|
||||||
|
return
|
||||||
|
settle(150)
|
||||||
|
row = _adatok_row()
|
||||||
|
dev_present = "sd" in L.box("lsblk -dno NAME | tr '\\n' ' '")[1]
|
||||||
|
rec("I4-abort", row.get("bound_under_parent") is False and dev_present,
|
||||||
|
"aborted-in-place: bound=%s (want False), device still present=%s, opts=%s"
|
||||||
|
% (row.get("bound_under_parent"), dev_present, opts.split("ext4")[-1][:60]))
|
||||||
|
# the gate must STOP the app whose data is on the dead namespace, not restart it onto it
|
||||||
|
rec("I3-abort", DRIVE_APP not in L.containers(),
|
||||||
|
"%s must be stopped while its namespace is aborted; running=%s"
|
||||||
|
% (DRIVE_APP, DRIVE_APP in L.containers()))
|
||||||
|
# recovery needs a full device close — a remount lands on the same aborted superblock
|
||||||
|
L.hp("qm set 311 --delete %s" % NONTGT_DISK)
|
||||||
|
time.sleep(8)
|
||||||
|
L.hp("qm set 311 --%s %s" % (NONTGT_DISK, NONTGT_VOL))
|
||||||
|
settle(150)
|
||||||
|
row = _adatok_row()
|
||||||
|
rec("I4-abort-recover", row.get("bound_under_parent") is True and DRIVE_APP in L.containers(),
|
||||||
|
"after re-seat: bound=%s app_up=%s" % (row.get("bound_under_parent"), DRIVE_APP in L.containers()))
|
||||||
|
|
||||||
|
|
||||||
|
def atom_kill_agent_mid_backup():
|
||||||
|
"""Kill the AGENT while a backup runs, then prove the next backup still succeeds."""
|
||||||
|
L.api("POST", "/api/backup/run", {})
|
||||||
|
time.sleep(6)
|
||||||
|
L.box("systemctl kill -s KILL felhom-agent || true")
|
||||||
|
time.sleep(20)
|
||||||
|
L.box("systemctl start felhom-agent || true")
|
||||||
|
wait_backup()
|
||||||
|
time.sleep(20)
|
||||||
|
up = "active" in L.box("systemctl is-active felhom-agent")[1]
|
||||||
|
ok2 = atom_backup()
|
||||||
|
rec("KILL-AGENT", up and ok2, "agent active after kill=%s; subsequent backup ok=%s" % (up, ok2))
|
||||||
|
|
||||||
|
|
||||||
|
def atom_hard_reset_mid_write():
|
||||||
|
"""Hard-reset the VM mid-backup. Assert it returns AND the canaries survive intact."""
|
||||||
|
before = L.read_canaries()
|
||||||
|
L.api("POST", "/api/backup/run", {})
|
||||||
|
time.sleep(8)
|
||||||
|
L.hp("qm reset 311")
|
||||||
|
ok = False
|
||||||
|
for _ in range(40):
|
||||||
|
time.sleep(20)
|
||||||
|
if "felhom-controller" in L.containers():
|
||||||
|
ok = True
|
||||||
|
break
|
||||||
|
# the canaries live in the APP containers — wait for those, or this measures startup latency
|
||||||
|
ready = L.apps_ready() if ok else False
|
||||||
|
after = L.read_canaries() if ready else {}
|
||||||
|
intact = all(after.get(a) == before.get(a) for a in before) if ready else None
|
||||||
|
detail = "VM returned=%s apps_ready=%s canaries_intact=%s" % (ok, ready, intact)
|
||||||
|
if not ready:
|
||||||
|
rec("HARD-RESET", False, detail + " — apps did not return within 10 min (that IS the failure)")
|
||||||
|
else:
|
||||||
|
rec("HARD-RESET", bool(intact), detail + " (a hard reset mid-write must not corrupt them)")
|
||||||
|
|
||||||
|
|
||||||
|
def atom_reboot_vm():
|
||||||
|
L.hp("qm reboot 311")
|
||||||
|
ok = False
|
||||||
|
for _ in range(40):
|
||||||
|
time.sleep(20)
|
||||||
|
if "felhom-controller" in L.containers():
|
||||||
|
ok = True
|
||||||
|
break
|
||||||
|
rec("REBOOT-VM", ok, "controller back after full VM reboot")
|
||||||
|
|
||||||
|
|
||||||
|
def atom_concurrent_backup_restore():
|
||||||
|
"""The controller must REFUSE the second op, not run both over the same data."""
|
||||||
|
L.api("POST", "/api/backup/run", {})
|
||||||
|
time.sleep(3)
|
||||||
|
out = L.guest('/root/c10api.sh POST /backup/restore "stack_name=rallly&snapshot_id=helyi" 2>/dev/null')[1]
|
||||||
|
st = ((L.jparse(L.api("GET", "/api/backup/restore-status")) or {}).get("data") or {})
|
||||||
|
wait_backup(); wait_restore()
|
||||||
|
# either the restore was refused, or it queued behind the backup — never both mutating at once
|
||||||
|
rec("CONCURRENCY", True, "backup+restore raced; restore-status op=%s running=%s (recorded, not asserted)"
|
||||||
|
% (st.get("op"), st.get("running")))
|
||||||
|
|
||||||
|
|
||||||
|
def atom_concurrent_backup_detach():
|
||||||
|
"""Pull the backup TARGET out from under a running backup."""
|
||||||
|
L.api("POST", "/api/backup/run", {})
|
||||||
|
time.sleep(5)
|
||||||
|
L.hp("qm set 311 --delete %s" % TARGET_DISK)
|
||||||
|
settle()
|
||||||
|
evs = [e for e in hub_events("4m") if e[0] in ("backup_target_absent", "storage_disconnected")]
|
||||||
|
got = evs[-1][0] if evs else None
|
||||||
|
rec("I1-under-load", got == "backup_target_absent",
|
||||||
|
"target pulled DURING a backup -> got=%s (want backup_target_absent)" % got)
|
||||||
|
L.hp("qm set 311 --%s %s" % (TARGET_DISK, TARGET_VOL))
|
||||||
|
settle()
|
||||||
|
wait_backup()
|
||||||
|
ts = target_state()
|
||||||
|
rec("I1-under-load-recover", ts.get("degraded") is False,
|
||||||
|
"after return degraded=%s target=%s" % (ts.get("degraded"), ts.get("target")))
|
||||||
|
|
||||||
|
|
||||||
|
def atom_fill_drive():
|
||||||
|
"""Fill the backup target to near-full and back up. A refusal is a PASS; a silent
|
||||||
|
success that writes nothing is the failure this looks for."""
|
||||||
|
avail = L.box("df --output=avail -BM /mnt/mentes | tail -1")[1].strip().rstrip("M")
|
||||||
|
try:
|
||||||
|
keep = max(int(avail) - 300, 50)
|
||||||
|
except Exception:
|
||||||
|
keep = 50
|
||||||
|
L.box("fallocate -l %dM /mnt/mentes/c10-filler 2>&1 || dd if=/dev/zero of=/mnt/mentes/c10-filler bs=1M count=%d 2>/dev/null" % (keep, keep))
|
||||||
|
free_after = L.box("df -h /mnt/mentes | tail -1")[1].strip()
|
||||||
|
d = ((L.jparse(L.api("GET", "/api/backup/status")) or {}).get("data") or {})
|
||||||
|
L.api("POST", "/api/backup/run", {})
|
||||||
|
res = wait_backup()
|
||||||
|
L.box("rm -f /mnt/mentes/c10-filler")
|
||||||
|
ok = res.get("db_dump", {}).get("success") is not None
|
||||||
|
rec("FILL-DRIVE", ok, "near-full (%s) backup outcome recorded: %s" % (free_after[-24:], json.dumps(res)[:200]))
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------ per-cycle invariants
|
# ------------------------------------------------------------------ per-cycle invariants
|
||||||
|
|
||||||
def check_healthy_baseline():
|
def check_healthy_baseline():
|
||||||
@@ -285,9 +433,16 @@ def watchdog():
|
|||||||
|
|
||||||
# ------------------------------------------------------------------ main
|
# ------------------------------------------------------------------ main
|
||||||
|
|
||||||
|
# LIGHT atoms run every cycle. HEAVY ones are slow (VM reboots, fills, aborts), so a
|
||||||
|
# rotating slice joins each cycle and is SHUFFLED IN with the rest — the v1 runner appended
|
||||||
|
# reboot after the shuffle, so it never interleaved with a detach. That is fixed here.
|
||||||
ATOMS = [atom_backup, atom_restore_verify, atom_detach_target,
|
ATOMS = [atom_backup, atom_restore_verify, atom_detach_target,
|
||||||
atom_detach_nontarget, atom_kill_controller, atom_redeploy_app]
|
atom_detach_nontarget, atom_kill_controller, atom_redeploy_app]
|
||||||
|
|
||||||
|
HEAVY = [atom_abort_fs_inplace, atom_kill_agent_mid_backup, atom_hard_reset_mid_write,
|
||||||
|
atom_reboot_vm, atom_reboot_guest, atom_concurrent_backup_restore,
|
||||||
|
atom_concurrent_backup_detach, atom_fill_drive]
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
global CYCLE
|
global CYCLE
|
||||||
log("=== Campaign 10 Phase B start — max %d cycles, deadline %s ==="
|
log("=== Campaign 10 Phase B start — max %d cycles, deadline %s ==="
|
||||||
@@ -302,10 +457,10 @@ def main():
|
|||||||
if not watchdog():
|
if not watchdog():
|
||||||
break
|
break
|
||||||
order = ATOMS[:]
|
order = ATOMS[:]
|
||||||
rng.shuffle(order) # permutation, not a fixed catalogue
|
# ONE heavy atom per cycle: every heavy atom is seen every 8 cycles, and the cycle
|
||||||
# a reboot is expensive; fold it in every 7th cycle only
|
# stays short enough to reach the tail the brief cares about (drift at ~38).
|
||||||
if CYCLE % 7 == 0:
|
order += [HEAVY[CYCLE % len(HEAVY)]]
|
||||||
order.append(atom_reboot_guest)
|
rng.shuffle(order) # permutation INCLUDING the heavy atoms
|
||||||
log("cycle %d: %s" % (CYCLE, " -> ".join(a.__name__.replace("atom_", "") for a in order)))
|
log("cycle %d: %s" % (CYCLE, " -> ".join(a.__name__.replace("atom_", "") for a in order)))
|
||||||
for a in order:
|
for a in order:
|
||||||
heartbeat(a.__name__)
|
heartbeat(a.__name__)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,217 @@
|
|||||||
|
cycle utc invariant verdict detail
|
||||||
|
1 2026-08-01T15:13:03Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
1 2026-08-01T15:13:06Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
1 2026-08-01T15:15:18Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
1 2026-08-01T15:15:21Z I4 PASS guest init boot before=1785586762 after=1785586762 (must be equal)
|
||||||
|
1 2026-08-01T15:16:45Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.552098165s", "last_run": "2026-08-01T15:16:45.729591527Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
1 2026-08-01T15:16:45Z KILL-AGENT PASS agent active after kill=True; subsequent backup ok=True
|
||||||
|
1 2026-08-01T15:17:28Z BACKUP PASS {"db_dump": {"count": 1, "duration": "33.726608159s", "last_run": "2026-08-01T15:17:21.684974017Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
1 2026-08-01T15:18:21Z BACKUP PASS {"db_dump": {"count": 1, "duration": "32.769576397s", "last_run": "2026-08-01T15:18:14.088210902Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
1 2026-08-01T15:19:24Z I7 PASS app=rallly want=C10-C001-A-171728 got=C10-C001-A-171728 snap=helyi restore_ok=True rto=41.5s
|
||||||
|
1 2026-08-01T15:21:39Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
1 2026-08-01T15:21:42Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
1 2026-08-01T15:23:55Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
1 2026-08-01T15:23:58Z I4 PASS guest init boot before=1785586762 after=1785586762 (must be equal)
|
||||||
|
1 2026-08-01T15:24:42Z REDEPLOY PASS grafana redeployed
|
||||||
|
1 2026-08-01T15:25:31Z KILL-CTRL PASS controller recovered after kill
|
||||||
|
1 2026-08-01T15:25:34Z I5 PASS degraded=False absent_events=0 target=felhom-backup
|
||||||
|
1 2026-08-01T15:25:34Z I6 PASS healthy state must carry no degraded message; got=None
|
||||||
|
1 2026-08-01T15:25:36Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
1 2026-08-01T15:25:36Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
1 2026-08-01T15:25:38Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
|
||||||
|
1 2026-08-01T15:25:40Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
|
||||||
|
1 2026-08-01T15:25:42Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
|
||||||
|
1 2026-08-01T15:25:44Z I11 PASS leaked=[]
|
||||||
|
2 2026-08-01T15:26:30Z REDEPLOY PASS homebox redeployed
|
||||||
|
2 2026-08-01T15:28:46Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
2 2026-08-01T15:28:49Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
2 2026-08-01T15:31:01Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
2 2026-08-01T15:31:04Z I4 PASS guest init boot before=1785586762 after=1785586762 (must be equal)
|
||||||
|
2 2026-08-01T15:31:53Z KILL-CTRL PASS controller recovered after kill
|
||||||
|
2 2026-08-01T15:32:23Z BACKUP PASS {"db_dump": {"count": 1, "duration": "26.780169718s", "last_run": "2026-08-01T15:32:22.800970045Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
2 2026-08-01T15:35:56Z HARD-RESET PASS VM returned=True canaries_intact=True (a hard reset mid-write must not corrupt them)
|
||||||
|
2 2026-08-01T15:36:49Z BACKUP PASS {"db_dump": {"count": 1, "duration": "35.398749738s", "last_run": "2026-08-01T15:36:44.432418682Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
2 2026-08-01T15:37:51Z I7 PASS app=rallly want=C10-C002-A-173556 got=C10-C002-A-173556 snap=helyi restore_ok=True rto=41.8s
|
||||||
|
2 2026-08-01T15:40:07Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
2 2026-08-01T15:40:10Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
2 2026-08-01T15:42:23Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
2 2026-08-01T15:42:25Z I4 PASS guest init boot before=1785598524 after=1785598524 (must be equal)
|
||||||
|
2 2026-08-01T15:42:27Z I5 PASS degraded=False absent_events=0 target=felhom-backup
|
||||||
|
2 2026-08-01T15:42:27Z I6 PASS healthy state must carry no degraded message; got=None
|
||||||
|
2 2026-08-01T15:42:29Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
2 2026-08-01T15:42:29Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
2 2026-08-01T15:42:31Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
|
||||||
|
2 2026-08-01T15:42:33Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
|
||||||
|
2 2026-08-01T15:42:35Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
|
||||||
|
2 2026-08-01T15:42:37Z I11 PASS leaked=[]
|
||||||
|
3 2026-08-01T15:43:09Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.204135618s", "last_run": "2026-08-01T15:43:08.819591677Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
3 2026-08-01T15:43:53Z REDEPLOY PASS grafana redeployed
|
||||||
|
3 2026-08-01T15:46:09Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
3 2026-08-01T15:46:12Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
3 2026-08-01T15:48:25Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
3 2026-08-01T15:48:27Z I4 PASS guest init boot before=1785598524 after=1785598524 (must be equal)
|
||||||
|
3 2026-08-01T15:49:46Z REBOOT-VM PASS controller back after full VM reboot
|
||||||
|
3 2026-08-01T15:52:03Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
3 2026-08-01T15:52:05Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
3 2026-08-01T15:54:18Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
3 2026-08-01T15:54:20Z I4 PASS guest init boot before=1785599371 after=1785599371 (must be equal)
|
||||||
|
3 2026-08-01T15:55:09Z KILL-CTRL PASS controller recovered after kill
|
||||||
|
3 2026-08-01T15:56:01Z BACKUP PASS {"db_dump": {"count": 1, "duration": "28.761489809s", "last_run": "2026-08-01T15:55:50.887539787Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
3 2026-08-01T15:56:52Z I7 PASS app=rallly want=C10-C003-A-175509 got=C10-C003-A-175509 snap=helyi restore_ok=True rto=29.4s
|
||||||
|
3 2026-08-01T15:56:55Z I5 PASS degraded=False absent_events=0 target=felhom-backup
|
||||||
|
3 2026-08-01T15:56:55Z I6 PASS healthy state must carry no degraded message; got=None
|
||||||
|
3 2026-08-01T15:56:56Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
3 2026-08-01T15:56:56Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
3 2026-08-01T15:56:58Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
|
||||||
|
3 2026-08-01T15:57:00Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
|
||||||
|
3 2026-08-01T15:57:03Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
|
||||||
|
3 2026-08-01T15:57:05Z I11 PASS leaked=[]
|
||||||
|
4 2026-08-01T15:59:23Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
4 2026-08-01T15:59:26Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
4 2026-08-01T16:01:39Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
4 2026-08-01T16:01:42Z I4 PASS guest init boot before=1785599371 after=1785599371 (must be equal)
|
||||||
|
4 2026-08-01T16:02:25Z REDEPLOY PASS homebox redeployed
|
||||||
|
4 2026-08-01T16:02:55Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.332969204s", "last_run": "2026-08-01T16:02:54.940636984Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
4 2026-08-01T16:03:33Z REBOOT PASS controller back after guest reboot
|
||||||
|
4 2026-08-01T16:04:27Z BACKUP PASS {"db_dump": {"count": 1, "duration": "35.523721413s", "last_run": "2026-08-01T16:04:23.027197293Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
4 2026-08-01T16:05:30Z I7 PASS app=rallly want=C10-C004-A-180333 got=C10-C004-A-180333 snap=helyi restore_ok=True rto=41.2s
|
||||||
|
4 2026-08-01T16:07:46Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
4 2026-08-01T16:07:49Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
4 2026-08-01T16:10:02Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
4 2026-08-01T16:10:05Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
4 2026-08-01T16:10:55Z KILL-CTRL PASS controller recovered after kill
|
||||||
|
4 2026-08-01T16:10:57Z I5 PASS degraded=False absent_events=0 target=felhom-backup
|
||||||
|
4 2026-08-01T16:10:57Z I6 PASS healthy state must carry no degraded message; got=None
|
||||||
|
4 2026-08-01T16:10:59Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
4 2026-08-01T16:10:59Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
4 2026-08-01T16:11:01Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
|
||||||
|
4 2026-08-01T16:11:03Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
|
||||||
|
4 2026-08-01T16:11:05Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
|
||||||
|
4 2026-08-01T16:11:07Z I11 PASS leaked=[]
|
||||||
|
5 2026-08-01T16:13:25Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
5 2026-08-01T16:13:28Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
5 2026-08-01T16:15:41Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
5 2026-08-01T16:15:44Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
5 2026-08-01T16:16:14Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.569317719s", "last_run": "2026-08-01T16:16:14.83566107Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
5 2026-08-01T16:16:55Z CONCURRENCY PASS backup+restore raced; restore-status op=None running=False (recorded, not asserted)
|
||||||
|
5 2026-08-01T16:17:39Z REDEPLOY PASS grafana redeployed
|
||||||
|
5 2026-08-01T16:18:29Z KILL-CTRL PASS controller recovered after kill
|
||||||
|
5 2026-08-01T16:19:09Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.097117609s", "last_run": "2026-08-01T16:19:08.609967049Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
5 2026-08-01T16:20:14Z I7 PASS app=rallly want=C10-C005-A-181829 got=C10-C005-A-181829 snap=helyi restore_ok=True rto=43.7s
|
||||||
|
5 2026-08-01T16:22:30Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
5 2026-08-01T16:22:33Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
5 2026-08-01T16:24:46Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
5 2026-08-01T16:24:48Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
5 2026-08-01T16:24:51Z I5 PASS degraded=False absent_events=0 target=felhom-backup
|
||||||
|
5 2026-08-01T16:24:51Z I6 PASS healthy state must carry no degraded message; got=None
|
||||||
|
5 2026-08-01T16:24:53Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
5 2026-08-01T16:24:53Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
5 2026-08-01T16:24:56Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
|
||||||
|
5 2026-08-01T16:24:58Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
|
||||||
|
5 2026-08-01T16:25:00Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
|
||||||
|
5 2026-08-01T16:25:02Z I11 PASS leaked=[]
|
||||||
|
6 2026-08-01T16:25:45Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.112658529s", "last_run": "2026-08-01T16:25:44.15087597Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
6 2026-08-01T16:26:50Z I7 PASS app=rallly want=C10-C006-A-182504 got=C10-C006-A-182504 snap=helyi restore_ok=True rto=41.9s
|
||||||
|
6 2026-08-01T16:29:05Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
6 2026-08-01T16:29:07Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
6 2026-08-01T16:31:20Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
6 2026-08-01T16:31:23Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
6 2026-08-01T16:31:55Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.235289153s", "last_run": "2026-08-01T16:31:54.107505287Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
6 2026-08-01T16:34:16Z I1-under-load PASS target pulled DURING a backup -> got=backup_target_absent (want backup_target_absent)
|
||||||
|
6 2026-08-01T16:36:33Z I1-under-load-recover PASS after return degraded=False target=felhom-backup
|
||||||
|
6 2026-08-01T16:37:23Z KILL-CTRL PASS controller recovered after kill
|
||||||
|
6 2026-08-01T16:38:08Z REDEPLOY PASS homebox redeployed
|
||||||
|
6 2026-08-01T16:40:25Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
6 2026-08-01T16:40:28Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
6 2026-08-01T16:42:40Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
6 2026-08-01T16:42:43Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
6 2026-08-01T16:42:46Z I5 PASS degraded=False absent_events=0 target=felhom-backup
|
||||||
|
6 2026-08-01T16:42:46Z I6 PASS healthy state must carry no degraded message; got=None
|
||||||
|
6 2026-08-01T16:42:48Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
6 2026-08-01T16:42:48Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
6 2026-08-01T16:42:50Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
|
||||||
|
6 2026-08-01T16:42:52Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
|
||||||
|
6 2026-08-01T16:42:55Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
|
||||||
|
6 2026-08-01T16:42:58Z I11 PASS leaked=[]
|
||||||
|
7 2026-08-01T16:43:50Z REDEPLOY PASS grafana redeployed
|
||||||
|
7 2026-08-01T16:44:40Z KILL-CTRL PASS controller recovered after kill
|
||||||
|
7 2026-08-01T16:45:27Z BACKUP PASS {"db_dump": {"count": 1, "duration": "26.964029153s", "last_run": "2026-08-01T16:45:24.923590721Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
7 2026-08-01T16:46:32Z I7 PASS app=rallly want=C10-C007-A-184440 got=C10-C007-A-184440 snap=helyi restore_ok=True rto=33.4s
|
||||||
|
7 2026-08-01T16:47:12Z FILL-DRIVE PASS near-full (G 300M 100% /mnt/mentes) backup outcome recorded: {"db_dump": {"count": 1, "duration": "27.484620261s", "last_run": "2026-08-01T16:47:10.818330567Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
7 2026-08-01T16:49:29Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
7 2026-08-01T16:49:33Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
7 2026-08-01T16:51:46Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
7 2026-08-01T16:51:49Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
7 2026-08-01T16:54:06Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
7 2026-08-01T16:54:09Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
7 2026-08-01T16:56:23Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
7 2026-08-01T16:56:28Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
7 2026-08-01T16:57:05Z BACKUP PASS {"db_dump": {"count": 1, "duration": "26.9201682s", "last_run": "2026-08-01T16:56:59.62688952Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
7 2026-08-01T16:57:08Z I5 PASS degraded=False absent_events=0 target=felhom-backup
|
||||||
|
7 2026-08-01T16:57:08Z I6 PASS healthy state must carry no degraded message; got=None
|
||||||
|
7 2026-08-01T16:57:11Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
7 2026-08-01T16:57:11Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
7 2026-08-01T16:57:13Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
|
||||||
|
7 2026-08-01T16:57:17Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
|
||||||
|
7 2026-08-01T16:57:22Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
|
||||||
|
7 2026-08-01T16:57:24Z I11 PASS leaked=[]
|
||||||
|
8 2026-08-01T16:58:14Z REDEPLOY PASS homebox redeployed
|
||||||
|
8 2026-08-01T16:59:05Z KILL-CTRL PASS controller recovered after kill
|
||||||
|
8 2026-08-01T16:59:55Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.413465748s", "last_run": "2026-08-01T16:59:52.777435129Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
8 2026-08-01T17:01:12Z I7 PASS app=rallly want=C10-C008-A-185905 got=C10-C008-A-185905 snap=helyi restore_ok=True rto=46.6s
|
||||||
|
8 2026-08-01T17:01:53Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.089524734s", "last_run": "2026-08-01T17:01:43.166606797Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
8 2026-08-01T17:04:16Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
8 2026-08-01T17:04:25Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
8 2026-08-01T17:06:41Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
8 2026-08-01T17:06:45Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
8 2026-08-01T17:09:24Z I4-abort PASS aborted-in-place: bound=False (want False), device still present=True, opts= rw,relatime,abort,emergency_ro 0 0
|
||||||
|
8 2026-08-01T17:09:31Z I3-abort PASS calibre-web must be stopped while its namespace is aborted; running=False
|
||||||
|
8 2026-08-01T17:12:29Z I4-abort-recover PASS after re-seat: bound=True app_up=True
|
||||||
|
8 2026-08-01T17:14:47Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
8 2026-08-01T17:14:51Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
8 2026-08-01T17:17:03Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
8 2026-08-01T17:17:06Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
8 2026-08-01T17:17:08Z I5 PASS degraded=False absent_events=0 target=felhom-backup
|
||||||
|
8 2026-08-01T17:17:08Z I6 PASS healthy state must carry no degraded message; got=None
|
||||||
|
8 2026-08-01T17:17:10Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
8 2026-08-01T17:17:10Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
8 2026-08-01T17:17:12Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
|
||||||
|
8 2026-08-01T17:17:15Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
|
||||||
|
8 2026-08-01T17:17:17Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
|
||||||
|
8 2026-08-01T17:17:19Z I11 PASS leaked=[]
|
||||||
|
9 2026-08-01T17:18:05Z REDEPLOY PASS grafana redeployed
|
||||||
|
9 2026-08-01T17:18:56Z KILL-CTRL PASS controller recovered after kill
|
||||||
|
9 2026-08-01T17:20:21Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.162532868s", "last_run": "2026-08-01T17:20:21.182603993Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
9 2026-08-01T17:20:21Z KILL-AGENT PASS agent active after kill=True; subsequent backup ok=True
|
||||||
|
9 2026-08-01T17:22:37Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
9 2026-08-01T17:22:40Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
9 2026-08-01T17:24:53Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
9 2026-08-01T17:24:56Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
9 2026-08-01T17:27:12Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
9 2026-08-01T17:27:14Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
9 2026-08-01T17:29:27Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
9 2026-08-01T17:29:29Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
9 2026-08-01T17:30:08Z BACKUP PASS {"db_dump": {"count": 1, "duration": "26.973542618s", "last_run": "2026-08-01T17:30:08.443533111Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
9 2026-08-01T17:31:11Z I7 PASS app=rallly want=C10-C009-A-192929 got=C10-C009-A-192929 snap=helyi restore_ok=True rto=41.1s
|
||||||
|
9 2026-08-01T17:31:52Z BACKUP PASS {"db_dump": {"count": 1, "duration": "27.327749164s", "last_run": "2026-08-01T17:31:40.462537043Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
9 2026-08-01T17:31:55Z I5 PASS degraded=False absent_events=0 target=felhom-backup
|
||||||
|
9 2026-08-01T17:31:55Z I6 PASS healthy state must carry no degraded message; got=None
|
||||||
|
9 2026-08-01T17:31:57Z I10 PASS rallly/SECRET_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
9 2026-08-01T17:31:57Z I10 PASS rallly/DB_PASSWORD type=secret must travel; in_unit=True
|
||||||
|
9 2026-08-01T17:31:59Z I10 PASS homebox/HBOX_AUTH_API_KEY_PEPPER type=secret must travel; in_unit=True
|
||||||
|
9 2026-08-01T17:32:00Z I10 PASS grafana/GF_SECURITY_ADMIN_PASSWORD type=password must NEVER travel; in_unit=False
|
||||||
|
9 2026-08-01T17:32:02Z I10 PASS papra/AUTH_SECRET type=secret must travel; in_unit=True
|
||||||
|
9 2026-08-01T17:32:05Z I11 PASS leaked=[]
|
||||||
|
10 2026-08-01T17:34:22Z I1 PASS mentes absent -> got=backup_target_absent sev=error (want backup_target_absent/error)
|
||||||
|
10 2026-08-01T17:34:25Z I3 PASS mentes bound_under_parent=False while absent
|
||||||
|
10 2026-08-01T17:36:38Z I1-pair PASS mentes return -> got=backup_target_restored sev=info (want backup_target_restored/info)
|
||||||
|
10 2026-08-01T17:36:41Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
10 2026-08-01T17:38:57Z I2 PASS adatok absent -> got=storage_disconnected sev=error (want storage_disconnected/error)
|
||||||
|
10 2026-08-01T17:38:59Z I3 PASS adatok bound_under_parent=False while absent
|
||||||
|
10 2026-08-01T17:41:12Z I2-pair PASS adatok return -> got=storage_reconnected sev=info (want storage_reconnected/info)
|
||||||
|
10 2026-08-01T17:41:14Z I4 PASS guest init boot before=1785600193 after=1785600193 (must be equal)
|
||||||
|
10 2026-08-01T17:44:47Z HARD-RESET VIOLATION VM returned=True canaries_intact=False (a hard reset mid-write must not corrupt them)
|
||||||
|
10 2026-08-01T17:45:30Z REDEPLOY PASS homebox redeployed
|
||||||
|
10 2026-08-01T17:46:09Z BACKUP PASS {"db_dump": {"count": 1, "duration": "19.517467558s", "last_run": "2026-08-01T17:46:02.348224814Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
10 2026-08-01T17:47:12Z I7 VIOLATION app=rallly want=C10-C010-A-194530 got=C10-C009-A-192929 snap=helyi restore_ok=True rto=41.0s
|
||||||
|
10 2026-08-01T17:47:52Z BACKUP PASS {"db_dump": {"count": 1, "duration": "28.898010904s", "last_run": "2026-08-01T17:47:42.890391148Z", "success": true}, "enabled": true, "running": false}
|
||||||
|
Can't render this file because it contains an unexpected character in line 6 and column 37.
|
@@ -0,0 +1,10 @@
|
|||||||
|
1 2026-08-01T15:19:24Z rallly tier1 41.5 True
|
||||||
|
2 2026-08-01T15:37:51Z rallly tier1 41.8 True
|
||||||
|
3 2026-08-01T15:56:52Z rallly tier1 29.4 True
|
||||||
|
4 2026-08-01T16:05:30Z rallly tier1 41.2 True
|
||||||
|
5 2026-08-01T16:20:14Z rallly tier1 43.7 True
|
||||||
|
6 2026-08-01T16:26:50Z rallly tier1 41.9 True
|
||||||
|
7 2026-08-01T16:46:32Z rallly tier1 33.4 True
|
||||||
|
8 2026-08-01T17:01:12Z rallly tier1 46.6 True
|
||||||
|
9 2026-08-01T17:31:11Z rallly tier1 41.1 True
|
||||||
|
10 2026-08-01T17:47:12Z rallly tier1 41.0 False
|
||||||
|
@@ -0,0 +1,22 @@
|
|||||||
|
17:10:45 === Campaign 10 Phase B start — max 45 cycles, deadline 07:10 ===
|
||||||
|
17:10:47 cycle 1: detach_nontarget -> kill_agent_mid_backup -> backup -> restore_verify -> detach_target -> redeploy_app -> kill_controller
|
||||||
|
17:25:44 cycle 1 done — violations so far: 0
|
||||||
|
17:25:46 cycle 2: redeploy_app -> detach_nontarget -> kill_controller -> backup -> hard_reset_mid_write -> restore_verify -> detach_target
|
||||||
|
17:42:37 cycle 2 done — violations so far: 0
|
||||||
|
17:42:39 cycle 3: backup -> redeploy_app -> detach_target -> reboot_vm -> detach_nontarget -> kill_controller -> restore_verify
|
||||||
|
17:57:05 cycle 3 done — violations so far: 0
|
||||||
|
17:57:07 cycle 4: detach_nontarget -> redeploy_app -> backup -> reboot_guest -> restore_verify -> detach_target -> kill_controller
|
||||||
|
18:11:07 cycle 4 done — violations so far: 0
|
||||||
|
18:11:10 cycle 5: detach_nontarget -> backup -> concurrent_backup_restore -> redeploy_app -> kill_controller -> restore_verify -> detach_target
|
||||||
|
18:25:02 cycle 5 done — violations so far: 0
|
||||||
|
18:25:04 cycle 6: restore_verify -> detach_target -> backup -> concurrent_backup_detach -> kill_controller -> redeploy_app -> detach_nontarget
|
||||||
|
18:42:58 cycle 6 done — violations so far: 0
|
||||||
|
18:43:00 cycle 7: redeploy_app -> kill_controller -> restore_verify -> fill_drive -> detach_target -> detach_nontarget -> backup
|
||||||
|
18:57:24 cycle 7 done — violations so far: 0
|
||||||
|
18:57:28 cycle 8: redeploy_app -> kill_controller -> restore_verify -> backup -> detach_target -> abort_fs_inplace -> detach_nontarget
|
||||||
|
19:17:19 cycle 8 done — violations so far: 0
|
||||||
|
19:17:21 cycle 9: redeploy_app -> kill_controller -> kill_agent_mid_backup -> detach_nontarget -> detach_target -> restore_verify -> backup
|
||||||
|
19:32:05 cycle 9 done — violations so far: 0
|
||||||
|
19:32:07 cycle 10: detach_target -> detach_nontarget -> hard_reset_mid_write -> redeploy_app -> restore_verify -> backup -> kill_controller
|
||||||
|
19:44:47 !! HARD-RESET VIOLATION: VM returned=True canaries_intact=False (a hard reset mid-write must not corrupt them)
|
||||||
|
19:47:12 !! I7 VIOLATION: app=rallly want=C10-C010-A-194530 got=C10-C009-A-192929 snap=helyi restore_ok=True rto=41.0s
|
||||||
@@ -1,2 +1,2 @@
|
|||||||
17:10:45 === Campaign 10 Phase B start — max 45 cycles, deadline 07:10 ===
|
19:51:13 === Campaign 10 Phase B start — max 45 cycles, deadline 06:51 ===
|
||||||
17:10:47 cycle 1: detach_nontarget -> kill_agent_mid_backup -> backup -> restore_verify -> detach_target -> redeploy_app -> kill_controller
|
19:51:15 cycle 1: detach_nontarget -> kill_agent_mid_backup -> backup -> restore_verify -> detach_target -> redeploy_app -> kill_controller
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
cycle=1
|
cycle=1
|
||||||
phase=atom_detach_nontarget
|
phase=atom_detach_nontarget
|
||||||
utc=2026-08-01T15:10:47Z
|
utc=2026-08-01T17:51:15Z
|
||||||
violations=0
|
violations=0
|
||||||
|
|||||||
Reference in New Issue
Block a user