felhom.eu/documentation/audits/catalog-sweep-2026-10-24/tools/file_rows.py

#!/usr/bin/env python3
"""file_rows.py [--apply] — the sweep's register rows A–I into OPEN-ITEMS.md, each in its category section in
severity order, IDs from the highest R-number in BOTH registers + 1, section heading counts updated; a note on R-739.
Prints the letter → ID map (README, table.py and the report use it). Without --apply it changes nothing."""
import json, os, re, sys

FE = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../.."))
OPEN = os.path.join(FE, "documentation/backlog/OPEN-ITEMS.md")
CLOSED = os.path.join(FE, "documentation/backlog/CLOSED-ITEMS.md")
EV = "`audits/catalog-sweep-2026-10-24/"

ROWS = [
    ("A", "Backup & restore", "P2",
     "**A reinstall's own capture rewrites the unit that „remove — keep backups” kept, so restoring that unit breaks the app and the household's item is gone.** "
     f"Catalog sweep 2026-10-24, reproduced on its own ({EV}_unitprobe/tandoor/`): install Tandoor, one account, „Mentés most”, remove keeping data and backups, "
     "install again (a volume app: no kept-data question) → 19:50:32 „Recovery unit captured for tandoor” rewrote ONLY the unit's `app.yaml` (checksum `0f366c97…` → `4438c1c5…`, "
     "`created_at` 19:45:26 → 19:50:32) beside the OLD data → „Visszaállítás” paired the NEW install's database password with the OLD data: „password authentication failed” ×42 (app) / ×22 (postgres), "
     "state `starting`, the account not there. Seen three times (the sweep's Tandoor cycle, this probe, AdventureLog's re-run 20:03:18 → 20:05:32 → restore); AdventureLog's first run worked only because no capture fell between. "
     "The comment at `felhom-controller/controller/internal/backup/recovery_unit.go:101-103` says the refresh never rewrites the definition away from its data. Household order: „I reinstalled, my recipes are gone — restore the backup”. "
     "The data itself is not deleted (the old volumes are in the unit).",
     "OPEN", "—", "Design + fix in the controller: a capture of a NEW install must not adopt a unit whose data belongs to an earlier install (a new unit, or the old one set aside); a test in this order that fails today; then `tools/unitprobe.py` again", "CC"),
    ("B", "Backup & restore", "P2",
     "**A unit restore reports success while the app never came up.** Catalog sweep 2026-10-24: the controller logged „tandoor restored but health check failed: stack tandoor did not reach running state within 1m30s after restore”, "
     "and the status the restore page reads said `ok: true` — „A(z) tandoor: 3 adatkötet és az adatbázis visszaállítva — az alkalmazás újraindult.” (17:32:41; again in the unitprobe, 19:52:38). "
     f"The household is told it worked ({EV}tandoor/`, `_unitprobe/tandoor/record.json`).",
     "OPEN", "—", "The restore's verdict consults the health result; the message says the app did not start and what to do; a test that fails today", "CC"),
    ("C", "Backup & restore", "P2",
     "**„Use my kept data” loaded ANOTHER box's database — a future-dated off-site copy of the rehearsal box — under today's kept files.** Catalog sweep 2026-10-24: Tester4's off-site store holds copies from the 2026-10-20 rehearsal, "
     "whose box clock was set to 2026-10-28. On the new box the question said „Az adatbázist innen töltjük vissza: távoli mentés, 2026-10-28 07:39” for Paperless-ngx, Nextcloud and Immich, and loaded it: the seeded account / user / album "
     "was not there, and Immich never answered again (404 for 20 min). RomM, whose newest copy was its own („saját mentés, 2026-10-24 19:17”), got its item back. "
     "`07` §6.5 says the load is refused when the unit „was taken of another drive” — the rehearsal box's drive was another disk, so that check did not hold here (which field it compares: not measured). "
     f"The household's own local unit stays (nothing deleted). Same future-dated copies as R-1206 (that row: retention only). Evidence {EV}paperless-ngx/`, `nextcloud/`, `immich/`.",
     "OPEN", "—", "Read which drive identity the load compares and why the rehearsal unit passed; refuse a unit of another drive or one dated after now (or say so); a test with both; then the sweep's kept-data cycle on one app", "CC"),
    ("D", "Apps & catalog", "P3",
     "**Homepage answers a stranger with no gate at all**, while Glance — the same class of dashboard — is behind the family gate (decision 221, S26). A stranger who reaches `homepage.<domain>` sees the box's CPU, memory and disk figures "
     f"and, once the family edits it, their own links ({EV}homepage/`; the stranger request → 200).",
     "WAITING-ON-OPERATOR", "the operator's word", "Decide: `family_gate: true` like Glance (CC's pick: one line in the template, proven on the drill first) or leave it open", "operator"),
    ("E", "Install & onboarding", "P3",
     "**An install that fails on the image registry's rate limit tells the household only „exit code 1”.** Catalog sweep 2026-10-24, code-server 16:01:37: `lscr.io … toomanyrequests` → the product cleaned up right (R-649) but the event and the card say "
     f"„Alkalmazás telepítése nem sikerült: Code-Server — exit code 1”; nothing says the download server is busy and a later try works. The same install went through later ({EV}code-server/`).",
     "OPEN", "—", "Recognise `toomanyrequests` (and a pull timeout) in the deploy error and say „the app's download server is busy — try again in an hour”; a test with the stderr line", "CC"),
    ("F", "Backup & restore", "P3",
     "**The restore refusal tells the household to „turn on remote backup, or connect a second drive” when the box has both and their first copy has not run yet.** Catalog sweep 2026-10-24, Grimmory and Calibre-Web: „Mentés most” makes the local copy only "
     "(`07` §8, by design), so the drive files had no copy yet; the refusal's sentence is `restore.refuse.files.none` (`felhom-controller/controller/internal/web/handlers.go:1835`) whatever the box has. "
     "The Hungarian sentence is frozen by `i18n_go_parity`, so the fix is a NEW key („the first copy is made in tonight's backup”) chosen when a tier is set up but holds nothing yet.",
     "OPEN", "—", "The new key + the branch + a test", "CC"),
    ("G", "Storage & devices", "P4",
     "**An English household sees the drive names in Hungarian** on the install page (and wherever a storage label shows): „Belső tárhely — 52.0 GB free (default)”, „Külső tárhely (hdd_1)” (catalog sweep 2026-10-24, "
     f"{EV}navidrome/page-install-en.txt`). The label is stored once, in Hungarian, when the drive is found (`felhom-controller/controller/internal/settings/settings.go` `InferStorageLabel`); a household may rename it.",
     "OPEN", "—", "Render a label that still equals the inferred default in the reader's language; a test per language", "CC"),
    ("H", "Storage & devices", "P4",
     "**„Remove with data” leaves the app's empty parent folder** (`<drive>/appdata/<app>`) when the template binds a subfolder: Grimmory, RomM, Paperless-ngx (catalog sweep 2026-10-24; 4 KB each; the bound folders were removed). "
     "Harmless (empty, not counted as old data), untidy for a household that browses the drive.",
     "OPEN", "—", "Remove the parent when it is empty after the bound folders go; a test", "CC"),
    ("I", "Process & tooling", "P4",
     "**The catalog's box fixtures drifted from the apps they test** — fixed only in the sweep's copy "
     f"({EV}tools/upgrade_fixtures_box.py`, marked SWEEP COPY): OpenGist 1.15 signs up at `/-/register`; Calibre-Web's admin is the generated ADMIN_USER; Mealie's after_install replaces the factory password (the seed raced it); "
     "Papra 26.6 needs an Origin header; Rallly 4.15 closed password sign-up (no fixture possible by password); Vaultwarden 1.37.4 answers 404 on `POST /identity/accounts/register` (the fixture was written on 1.36; the admin invite works, the invited sign-up has no route — the newer clients' register/finish flow);the read-back through a family gate needs a fresh cookie after the setup gate opens (Grimmory stayed inconclusive). "
     "New fixtures for Homebox, Immich, Komga, Jellyfin, Ghost, Radarr, Sonarr in `tools/sweep_fixtures.py`; `box_walk.py` gained a `sweep` target.",
     "OPEN", "—", "Port them into `app-catalog-felhom.eu/scripts/` with the bench tests", "CC"),
]
R739_NOTE = (" **2026-10-24 (catalog sweep):** seen again on a box without DNS — Wanderer unhealthy and 404 for this reason "
             "(`audits/catalog-sweep-2026-10-24/wanderer/`).")
R1267_NOTE = (" **2026-10-24 (catalog sweep):** the same class for OnlyOffice and RomM (RomM's image volume `/romm` holds only the "
              "empty mount points of its binds); after every catalog app had been removed WITH data, the sweep box still held 52 "
              "such volumes (PrivateBin 21, OnlyOffice 8, RomM 8, Immich 3, and 12 from 12:50–12:52 before the batch, holding the same nginx folders as PrivateBin's) — remove-with-data "
              "does not see them (no compose project label). All small (4–72 KB), no household data "
              "(`audits/catalog-sweep-2026-10-24/_box/anon-volumes.txt`).")
SEV_ORDER = {"P1": 1, "P2": 2, "P3": 3, "P4": 4}


def main(apply):
    text = open(OPEN).read()
    ids = [int(x) for x in re.findall(r"\bR-(\d+)\b", text + open(CLOSED).read())]
    nxt = max(ids) + 1
    lines = text.split("\n")
    idmap = {}
    for letter, cat, sev, what, state, blocked, action, owner in ROWS:
        rid = f"R-{nxt}"; nxt += 1
        idmap[letter] = rid
        row = f"| **{rid}** | {cat} | {sev} | {what} | {state} | {blocked} | {action} | {owner} |"
        h = next(i for i, l in enumerate(lines) if re.match(rf"^## {re.escape(cat)} — \d+ rows", l))
        end = next((i for i in range(h + 1, len(lines)) if lines[i].startswith("## ")), len(lines))
        rows_at = [i for i in range(h, end) if lines[i].startswith("| **R-")]
        pos = None
        for i in rows_at:
            s = lines[i].split(" | ")[2].strip()
            if SEV_ORDER.get(s, 9) > SEV_ORDER[sev]:
                pos = i; break
        if pos is None:
            pos = rows_at[-1] + 1 if rows_at else h + 4
        lines.insert(pos, row)
        # heading: total + per-sev counts, recomputed from the rows of THIS category in the section
        m = re.match(r"^(## .+? — )(\d+) rows \((.*)\)$", lines[h])
        counts = {}
        for p in re.findall(r"(P\d) (\d+)", m.group(3)):
            counts[p[0]] = int(p[1])
        counts[sev] = counts.get(sev, 0) + 1
        lines[h] = f"{m.group(1)}{int(m.group(2)) + 1} rows ({', '.join(f'{k} {counts[k]}' for k in sorted(counts))})"
    for rid, note in (("R-739", R739_NOTE), ("R-1267", R1267_NOTE)):
        i = next(i for i, l in enumerate(lines) if l.startswith(f"| **{rid}** |"))
        cells = lines[i].split(" | ")
        assert len(cells) == 8, (rid, len(cells))
        if "catalog sweep" not in lines[i]:
            cells[4] = cells[4] + note  # the State cell, dated
            lines[i] = " | ".join(cells)
    print(json.dumps(idmap))
    if apply:
        open(OPEN, "w").write("\n".join(lines))


if __name__ == "__main__":
    main("--apply" in sys.argv)