diff --git a/bios/_mame_clones.json b/_mame_clones.json similarity index 100% rename from bios/_mame_clones.json rename to _mame_clones.json diff --git a/database.json b/database.json index 78d71c6d..81d75718 100644 --- a/database.json +++ b/database.json @@ -1,7 +1,7 @@ { - "generated_at": "2026-03-24T20:29:18Z", - "total_files": 6734, - "total_size": 5288666957, + "generated_at": "2026-03-24T20:51:56Z", + "total_files": 6733, + "total_size": 5288644732, "files": { "520d3d1b5897800af47f92efd2444a26b7a7dead": { "path": "bios/3DO Company/3DO/3do_arcade_saot.bin", @@ -67323,16 +67323,6 @@ "crc32": "0a4e2e07", "adler32": "4d032232" }, - "065c00ac8244d46e75448f667bbdfcf64dfc37c4": { - "path": "bios/_mame_clones.json", - "name": "_mame_clones.json", - "size": 22225, - "sha1": "065c00ac8244d46e75448f667bbdfcf64dfc37c4", - "md5": "178e3d546f24924ccf71160799a09bc4", - "sha256": "098e8ee3af19d9c290a082a03b923f25b0c333b52c689a99ac75cee69f569af3", - "crc32": "834c7566", - "adler32": "12de26b9" - }, "ecfc092fe6371dbf38e238a8ba5f90785b5db52d": { "path": "bios/xrick/data.zip", "name": "data.zip", @@ -74078,7 +74068,6 @@ "6f68e4baf89c8ee4623c19617319184b": "cee76080884af97c20059da0eb1ca956a835f3d0", "2010e5b85f9e1d60685ccb3d84a17115": "c7cc306fb921754ba00794153292d533cf0765ef", "39e5bc84ce9aac3a2d297d8aeb2a0d05": "22bcfeb5b6c6481569b90db96aa3f4b5f06c8848", - "178e3d546f24924ccf71160799a09bc4": "065c00ac8244d46e75448f667bbdfcf64dfc37c4", "a471e64e9f69afbe59c10cc94ed1b184": "ecfc092fe6371dbf38e238a8ba5f90785b5db52d" }, "by_name": { @@ -92631,9 +92620,6 @@ "default.sf2": [ "22bcfeb5b6c6481569b90db96aa3f4b5f06c8848" ], - "_mame_clones.json": [ - "065c00ac8244d46e75448f667bbdfcf64dfc37c4" - ], "data.zip": [ "ecfc092fe6371dbf38e238a8ba5f90785b5db52d" ], @@ -102100,7 +102086,6 @@ "8419990c": "cee76080884af97c20059da0eb1ca956a835f3d0", "3cacb086": "c7cc306fb921754ba00794153292d533cf0765ef", "0a4e2e07": "22bcfeb5b6c6481569b90db96aa3f4b5f06c8848", - "834c7566": "065c00ac8244d46e75448f667bbdfcf64dfc37c4", "74b76447": "ecfc092fe6371dbf38e238a8ba5f90785b5db52d" }, "by_path_suffix": { diff --git a/scripts/common.py b/scripts/common.py index 0a096859..e5f6810c 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -292,9 +292,38 @@ def resolve_local_file( if os.path.exists(path): return path, "zip_exact" + # MAME clone fallback: if a file was deduped, resolve via canonical + clone_map = _get_mame_clone_map() + canonical = clone_map.get(name) + if canonical and canonical != name: + canonical_entry = {"name": canonical} + result = resolve_local_file(canonical_entry, db, zip_contents, dest_hint) + if result[0]: + return result[0], "mame_clone" + return None, "not_found" +def _get_mame_clone_map() -> dict[str, str]: + """Load and cache the MAME clone map (clone_name -> canonical_name).""" + if not hasattr(_get_mame_clone_map, "_cache"): + clone_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "_mame_clones.json", + ) + if os.path.exists(clone_path): + import json as _json + with open(clone_path) as f: + data = _json.load(f) + _get_mame_clone_map._cache = {} + for canonical, info in data.items(): + for clone in info.get("clones", []): + _get_mame_clone_map._cache[clone] = canonical + else: + _get_mame_clone_map._cache = {} + return _get_mame_clone_map._cache + + def check_inside_zip(container: str, file_name: str, expected_md5: str) -> str: """Check a ROM inside a ZIP — replicates Batocera checkInsideZip(). diff --git a/scripts/dedup.py b/scripts/dedup.py index 1e5261a9..245b6f8c 100644 --- a/scripts/dedup.py +++ b/scripts/dedup.py @@ -170,7 +170,11 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict: if dry_run: print(f" WOULD REMOVE: {dup}") else: - os.remove(dup) + try: + os.remove(dup) + except OSError as e: + print(f" WARNING: cannot remove {dup}: {e}") + continue # Clean up empty .variants/ directories parent = os.path.dirname(dup) if os.path.basename(parent) == ".variants" and not os.listdir(parent): @@ -204,7 +208,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict: # Write MAME clone mapping if mame_clones: - clone_path = os.path.join(bios_dir, "_mame_clones.json") + clone_path = "_mame_clones.json" if dry_run: print(f"\nWould write MAME clone map: {clone_path}") print(f" {len(mame_clones)} canonical ZIPs with " diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index f42b9afe..52a2c34a 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -100,31 +100,6 @@ def _sanitize_path(raw: str) -> str: return "/".join(parts) -def _load_mame_clones(bios_dir: str) -> dict[str, str]: - """Load MAME clone mapping: clone_name -> canonical_name.""" - clone_path = os.path.join(bios_dir, "_mame_clones.json") - if not os.path.exists(clone_path): - return {} - with open(clone_path) as f: - data = json.load(f) - # Invert: clone_name -> canonical_name - result = {} - for canonical, info in data.items(): - for clone in info.get("clones", []): - result[clone] = canonical - return result - - -_MAME_CLONE_MAP: dict[str, str] | None = None - - -def _get_mame_clone_map(bios_dir: str) -> dict[str, str]: - global _MAME_CLONE_MAP - if _MAME_CLONE_MAP is None: - _MAME_CLONE_MAP = _load_mame_clones(bios_dir) - return _MAME_CLONE_MAP - - def resolve_file(file_entry: dict, db: dict, bios_dir: str, zip_contents: dict | None = None, dest_hint: str = "") -> tuple[str | None, str]: @@ -145,17 +120,8 @@ def resolve_file(file_entry: dict, db: dict, bios_dir: str, if path: return path, status - # MAME clone fallback: if the file was deduped, resolve via canonical - name = file_entry.get("name", "") - clone_map = _get_mame_clone_map(bios_dir) - canonical = clone_map.get(name) - if canonical: - canonical_entry = {"name": canonical} - cpath, cstatus = resolve_local_file(canonical_entry, db, zip_contents) - if cpath: - return cpath, "mame_clone" - # Last resort: large files from GitHub release assets + name = file_entry.get("name", "") sha1 = file_entry.get("sha1") md5_raw = file_entry.get("md5", "") md5_list = [m.strip().lower() for m in md5_raw.split(",") if m.strip()] if md5_raw else []