diff --git a/scripts/generate_site.py b/scripts/generate_site.py index a7fc9aae..1a6ef12c 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1304,17 +1304,20 @@ def collection_dat(db: dict, generated_at: str) -> str: import xml.etree.ElementTree as ET # noqa: PLC0415 games: dict[str, list[dict]] = {} - for entry in db.get("files", {}).values(): - parts = str(entry.get("path", "")).split("/") - if len(parts) < 3 or parts[0] != "bios": - continue - folder = parts[1:3] if len(parts) > 3 else parts[1:2] - # A backslash is how a Logiqx set name writes a subfolder: named - # "Sony - PlayStation", the set was looked for beside Sony/, and an - # audit of a copy of bios/ found 330 sets missing. - games.setdefault("\\".join(folder), []).append( - {**entry, "_rom": "\\".join(parts[1 + len(folder):])} - ) + files = db.get("files", {}) + # Every copy, not the one path the database keeps: an audit of bios/ + # read the 2,502 others as unknown files, and a fix moved them away + # from the engines that read them by path. + for sha1, paths in collection_paths(db).items(): + for path in sorted(paths): + parts = path.split("/") + folder = parts[1:3] if len(parts) > 3 else parts[1:2] + # A backslash is how a Logiqx set name writes a subfolder: named + # "Sony - PlayStation", the set was looked for beside Sony/, and an + # audit of a copy of bios/ found 330 sets missing. + games.setdefault("\\".join(folder), []).append( + {**files[sha1], "_rom": "\\".join(parts[1 + len(folder):])} + ) root = ET.Element("datafile") header = ET.SubElement(root, "header") version = generated_at[:10].replace("-", "") @@ -2097,6 +2100,28 @@ def file_location_items( return sorted(items, key=lambda item: item["sha1"]) +def collection_paths(db: dict) -> dict[str, set[str]]: + """Every path under bios/ that holds each file, by sha1. + + The database keeps one path per content. The others are the full tails + by_path_suffix records that name a file on disk: RPG Maker, engine + trees and the copies filed under several consoles keep 2,502 of them. + """ + files = db.get("files", {}) + bios_dir = Path(str(db.get("bios_dir") or "bios")) + paths: dict[str, set[str]] = {} + for sha1, entry in files.items(): + path = str(entry.get("path", "")) + if path.startswith("bios/") and path.count("/") >= 2: + paths.setdefault(sha1, set()).add(path) + for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items(): + if tail.count("/") < 2 or len(sha1s) != 1 or sha1s[0] not in files: + continue + if (bios_dir / tail).is_file(): + paths.setdefault(sha1s[0], set()).add(f"bios/{tail}") + return paths + + def _group_by_manufacturer(db: dict) -> dict[str, dict[str, list]]: """Group files by manufacturer -> console -> files. diff --git a/tests/test_site_render.py b/tests/test_site_render.py index 904d911c..13630dbe 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -309,5 +309,33 @@ class ForgesUpstreamDoesNotRead(unittest.TestCase): self.assertIn("/browse/revision/f6c00044" + "0" * 32 + "/?path=app/src/os.cpp#L35-L36)", ref) +class TheDatHoldsEveryCopy(unittest.TestCase): + """The DAT held one path per content, the database's: auditing bios/ + with it flagged the 2,502 other copies (RPG Maker, engine trees, a dump + filed under two consoles) as unknown, and a fix moved them away.""" + + def test_a_second_copy_is_a_rom_of_its_own_folder(self): + import generate_site + + sha1 = "9526ef63" + "0" * 32 + with tempfile.TemporaryDirectory() as tmp: + for rel in ("Atari/ST/tos104.img", "Atari/ST/tos104uk.img"): + (Path(tmp) / rel).parent.mkdir(parents=True, exist_ok=True) + (Path(tmp) / rel).write_bytes(b"x") + db = { + "bios_dir": tmp, + "files": {sha1: {"path": "bios/Atari/ST/tos104.img", "sha1": sha1, + "size": 1, "name": "tos104.img"}}, + "indexes": {"by_path_suffix": { + "Atari/ST/tos104.img": [sha1], "ST/tos104.img": [sha1], + "Atari/ST/tos104uk.img": [sha1], "ST/tos104uk.img": [sha1], + }}, + } + dat = generate_site.collection_dat(db, "2026-10-21T00:00:00Z") + self.assertIn('