diff --git a/scripts/test_upgrade_bench.py b/scripts/test_upgrade_bench.py index 1088e2e..926cc28 100644 --- a/scripts/test_upgrade_bench.py +++ b/scripts/test_upgrade_bench.py @@ -81,5 +81,70 @@ class ChangedFiles(unittest.TestCase): self.assertEqual(ut.changed_files(before, before), []) +class MarkerIgnore(unittest.TestCase): + """R-734 (`09` §3 decision 145): app-owned marker files do not make the `files_may_change` mark; any other + changed file still does. The data is the bench's own measurement (audits/more-night-apps-2026-09-30/D/evidence/ + D-immich-0b82-768/files-{before,after}-detail.json). RED-PROOF (REPORT): make classify_changes ignore every + changed file of an app with a list — test_an_unlisted_changed_file_still_marks fails.""" + + SRC = "/srv/felhom-gate/hdd/appdata/immich" + BEFORE = {SRC: {"profile/.immich": "13:7cb1f8a19d944d7e", "upload/.immich": "13:67848ba791bd7ef6", + "encoded-video/.immich": "13:68bdf71e87fabadc", "backups/.immich": "13:7cb1f8a19d944d7e", + "thumbs/.immich": "13:7cb1f8a19d944d7e", "library/.immich": "13:67848ba791bd7ef6"}, + "/srv/felhom-gate/hdd/userdata/media/photos": {}} + AFTER = {SRC: {"profile/.immich": "13:0bacc6528e07d8cb", "upload/.immich": "13:a68db79a3fbe6725", + "encoded-video/.immich": "13:2cb994a10bfe243c", "backups/.immich": "13:648fe4b9664fadff", + "thumbs/.immich": "13:0bacc6528e07d8cb", "library/.immich": "13:a68db79a3fbe6725"}, + "/srv/felhom-gate/hdd/userdata/media/photos": {}} + + def mark(self, app, before, after): + counted, ignored = ut.classify_changes(app, before, after) + tree = sorted({c.split(" (")[0].rsplit("/", 2)[0] for c in counted + [i["file"] for i in ignored]}) + return ut.files_may_change(tree, counted, ignored), counted, ignored + + def test_the_six_measured_markers_do_not_mark_and_each_carries_a_reason(self): + marked, counted, ignored = self.mark("immich", self.BEFORE, self.AFTER) + self.assertFalse(marked) + self.assertEqual(counted, []) + self.assertEqual(len(ignored), 6) + self.assertTrue(all(i["reason"] for i in ignored)) + + def test_an_unlisted_changed_file_still_marks(self): + after = {k: dict(v) for k, v in self.AFTER.items()} + after[self.SRC]["library/admin/2026/IMG_0001.jpg"] = "2048:abcd" # added: a household photo + before = {k: dict(v) for k, v in self.BEFORE.items()} + before[self.SRC]["upload/upload/x.jpg"] = "10:aa" + after[self.SRC]["upload/upload/x.jpg"] = "10:bb" # changed, same size as a marker + marked, counted, ignored = self.mark("immich", before, after) + self.assertTrue(marked) + self.assertIn(self.SRC + "/library/admin/2026/IMG_0001.jpg (added)", counted) + self.assertIn(self.SRC + "/upload/upload/x.jpg (changed)", counted) + self.assertEqual(len(ignored), 6) + + def test_a_listed_name_that_grew_or_vanished_still_marks(self): + after = {k: dict(v) for k, v in self.AFTER.items()} + after[self.SRC]["thumbs/.immich"] = "4096:ffff" # no longer a 13-byte marker + del after[self.SRC]["upload/.immich"] # removed + marked, counted, _ = self.mark("immich", self.BEFORE, after) + self.assertTrue(marked) + self.assertIn(self.SRC + "/thumbs/.immich (changed)", counted) + self.assertIn(self.SRC + "/upload/.immich (removed)", counted) + + def test_the_list_belongs_to_its_app(self): + # the same files under another app (and another bind) count: the list is per app + marked, counted, ignored = self.mark("calibre-web", self.BEFORE, self.AFTER) + self.assertTrue(marked) + self.assertEqual((len(counted), ignored), (6, [])) + + def test_a_moved_tree_the_file_walk_cannot_name_keeps_the_mark(self): + self.assertTrue(ut.files_may_change(["/d"], [], [])) + self.assertFalse(ut.files_may_change([], [], [])) + + def test_every_entry_has_a_reason(self): + for app, rules in ut.MARKER_IGNORE.items(): + for suffix, path, max_size, reason in rules: + self.assertTrue(suffix and path and max_size > 0 and len(reason) > 40, (app, path)) + + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/scripts/upgrade-test.py b/scripts/upgrade-test.py index b930e99..19f914f 100755 --- a/scripts/upgrade-test.py +++ b/scripts/upgrade-test.py @@ -51,8 +51,9 @@ import importlib.util, json, os, re, shutil, subprocess, sys, time from datetime import datetime, timezone from pathlib import Path -HARNESS_VERSION = 4 # 2: the memory watch (R-635); 3: box fixtures on the bench + files_may_change (2026-09-23 night); +HARNESS_VERSION = 5 # 2: the memory watch (R-635); 3: box fixtures on the bench + files_may_change (2026-09-23 night); # 4: a PostgreSQL major is CONVERTED on the bench, and the ladder carries the mark (`09` §6.4 part 10) + # 5: files_may_change no longer counts an app's listed MARKER files (MARKER_IGNORE, R-734) ROOT = Path("/opt/upg") TEMPLATES = ROOT / "templates" EVIDENCE = ROOT / "evidence" @@ -628,6 +629,62 @@ def bind_tree_files(project: str, workdir: Path) -> dict: return out +# R-734 (`09` §3 decision 145): app-owned MARKER files the `files_may_change` mark does not count. Each entry is +# (bind-source suffix, path inside that bind, largest size in bytes it may have, the reason). An entry ignores a file +# only when it was CHANGED or ADDED and is still that small — a removed marker, a grown one, or any file not listed +# here still counts, so the list can only take away what was measured to be noise. Pinned by +# scripts/test_upgrade_bench.py (MarkerIgnore: an unlisted changed file still marks the step). +_IMMICH_MARKER_REASON = ("immich rewrites this 13-byte folder marker at every start (its storage check writes a new " + "random token); measured on the bench 2026-09-30, v3.2.2 -> v3.2.4 and the 0b82 re-proof: " + "the only changed files were these six markers, no household file (R-734)") +MARKER_IGNORE = { + "immich": [("appdata/immich", "%s/.immich" % d, 64, _IMMICH_MARKER_REASON) + for d in ("encoded-video", "library", "backups", "profile", "thumbs", "upload")], +} + + +def _size_of(entry) -> int: + """The size from a bind_tree_files value (":"); -1 when it cannot be read.""" + try: + return int(str(entry).split(":", 1)[0]) + except ValueError: + return -1 + + +def classify_changes(app: str, before: dict, after: dict): + """(counted, ignored) between two bind_tree_files results. `counted` is changed_files' list without the + files MARKER_IGNORE lists for `app`; `ignored` is [{"file", "reason"}] for the ones it took away.""" + rules = MARKER_IGNORE.get(app) or [] + counted, ignored = [], [] + for src in sorted(set(before) | set(after)): + b, a = before.get(src) or {}, after.get(src) or {} + for rel in sorted(set(b) | set(a)): + if b.get(rel) == a.get(rel): + continue + kind = "added" if rel not in b else "removed" if rel not in a else "changed" + line = f"{src}/{rel} ({kind})" + hit = None + if kind != "removed": + for suffix, path, max_size, reason in rules: + if (src.rstrip("/") == suffix or src.rstrip("/").endswith("/" + suffix)) and rel == path \ + and 0 <= _size_of(a.get(rel)) <= max_size: + hit = reason + break + if hit: + ignored.append({"file": line, "reason": hit}) + else: + counted.append(line) + return counted, ignored + + +def files_may_change(tree_changed: list, counted: list, ignored: list) -> bool: + """The mark. The tree hash moved and at least one changed file is NOT on the app's ignore list — or the tree + hash moved and the per-file walk could not name any file (the two walks disagree: keep the honest mark).""" + if not tree_changed: + return False + return bool(counted) or not ignored + + def changed_files(before: dict, after: dict) -> list: """["/ (added|removed|changed)"] between two bind_tree_files results.""" out = [] @@ -927,10 +984,16 @@ def run_edge(edge_id: str) -> dict: detail_after = bind_tree_files(project, workdir) (ev / "files-after-detail.json").write_text(json.dumps(detail_after, indent=2)) rec["files_changed_detail"] = changed_files(detail_before, detail_after)[:200] - if rec["files_changed"]: + counted, ignored = classify_changes(app, detail_before, detail_after) + rec["files_ignored"] = ignored[:200] # R-734: app-owned markers taken away, each with its reason + if ignored: + say(f"files ignored as app markers (MARKER_IGNORE, R-734): {[i['file'] for i in ignored][:20]}") + if files_may_change(rec["files_changed"], counted, ignored): rec["marks"] = sorted(set(rec["marks"]) | {"files_may_change"}) - say(f"files_may_change — the files: {rec['files_changed_detail'][:20]}") + say(f"files_may_change — the files: {(counted or rec['files_changed_detail'])[:20]}") say(f"files_may_change: the bind-mounted tree changed under {rec['files_changed']}") + elif rec["files_changed"]: + say("files_may_change NOT set: every changed file is an app marker on the ignore list") # --- 5b. the MEMORY WATCH — only for an edge that just read back; a failed one is decided --- if rec["verdict"] == "proven" and SOAK_SECONDS > 0: