diff --git a/database.json b/database.json index f88854c9..fa07c5cb 100644 --- a/database.json +++ b/database.json @@ -1,8 +1,9 @@ { "schema_version": 1, - "generated_at": "2026-10-09T20:11:24Z", + "generated_at": "2026-10-09T20:41:14Z", "total_files": 19002, "total_size": 17497698358, + "bios_dir": "bios", "files": { "738c7f5fd0c8b57ee2e87ae7a97bf8e21a821d07": { "path": "bios/3D Realms/Duke Nukem 3D/DUKE.RTS", diff --git a/schemas/database.schema.json b/schemas/database.schema.json index 75befdae..32a0c3a9 100644 --- a/schemas/database.schema.json +++ b/schemas/database.schema.json @@ -4,55 +4,136 @@ "title": "RetroBIOS content database", "type": "object", "additionalProperties": false, - "required": ["schema_version", "generated_at", "total_files", "total_size", "files", "indexes"], + "required": [ + "schema_version", + "generated_at", + "total_files", + "total_size", + "files", + "indexes" + ], "properties": { - "schema_version": { "const": 1 }, - "generated_at": { "type": "string", "format": "date-time" }, - "total_files": { "type": "integer", "minimum": 0 }, - "total_size": { "type": "integer", "minimum": 0 }, + "schema_version": { + "const": 1 + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "total_files": { + "type": "integer", + "minimum": 0 + }, + "total_size": { + "type": "integer", + "minimum": 0 + }, + "bios_dir": { + "type": "string", + "description": "Directory the files were scanned from; _mame_clones.json sits beside it." + }, "files": { "type": "object", - "propertyNames": { "pattern": "^[a-f0-9]{40}$" }, + "propertyNames": { + "pattern": "^[a-f0-9]{40}$" + }, "additionalProperties": { "type": "object", "additionalProperties": false, - "required": ["path", "name", "size", "sha1", "md5", "sha256", "crc32", "adler32"], + "required": [ + "path", + "name", + "size", + "sha1", + "md5", + "sha256", + "crc32", + "adler32" + ], "properties": { - "path": { "type": "string", "minLength": 1 }, - "name": { "type": "string", "minLength": 1 }, - "size": { "type": "integer", "minimum": 0 }, - "sha1": { "type": "string", "pattern": "^[a-f0-9]{40}$" }, - "md5": { "type": "string", "pattern": "^[a-f0-9]{32}$" }, - "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, - "crc32": { "type": "string", "pattern": "^[a-f0-9]{8}$" }, - "adler32": { "type": "string", "pattern": "^[a-f0-9]{8}$" }, - "provenance": { "type": "object" } + "path": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "sha1": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "md5": { + "type": "string", + "pattern": "^[a-f0-9]{32}$" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "crc32": { + "type": "string", + "pattern": "^[a-f0-9]{8}$" + }, + "adler32": { + "type": "string", + "pattern": "^[a-f0-9]{8}$" + }, + "provenance": { + "type": "object" + } } } }, "indexes": { "type": "object", "additionalProperties": false, - "required": ["by_md5", "by_name", "by_crc32", "by_sha256", "by_path_suffix"], + "required": [ + "by_md5", + "by_name", + "by_crc32", + "by_sha256", + "by_path_suffix" + ], "properties": { - "by_md5": { "$ref": "#/$defs/hashIndex" }, - "by_crc32": { "$ref": "#/$defs/hashIndex" }, - "by_sha256": { "$ref": "#/$defs/hashIndex" }, - "by_name": { "$ref": "#/$defs/listIndex" }, - "by_path_suffix": { "$ref": "#/$defs/listIndex" } + "by_md5": { + "$ref": "#/$defs/hashIndex" + }, + "by_crc32": { + "$ref": "#/$defs/hashIndex" + }, + "by_sha256": { + "$ref": "#/$defs/hashIndex" + }, + "by_name": { + "$ref": "#/$defs/listIndex" + }, + "by_path_suffix": { + "$ref": "#/$defs/listIndex" + } } } }, "$defs": { "hashIndex": { "type": "object", - "additionalProperties": { "type": "string", "pattern": "^[a-f0-9]{40}$" } + "additionalProperties": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } }, "listIndex": { "type": "object", "additionalProperties": { "type": "array", - "items": { "type": "string", "pattern": "^[a-f0-9]{40}$" }, + "items": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, "minItems": 1, "uniqueItems": true } diff --git a/scripts/common.py b/scripts/common.py index f26ed540..1eab0cdd 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -912,7 +912,7 @@ def resolve_local_file( # MAME clone fallback: if a file was deduped, resolve via canonical if _depth < 3 and not has_strong_hash: - clone_map = get_mame_clone_map() + clone_map = get_mame_clone_map(db.get("bios_dir")) canonical = clone_map.get(name) if canonical and canonical != name: canonical_entry = {"name": canonical} @@ -960,28 +960,34 @@ def resolve_local_file( return None, "not_found" -_mame_clone_map_cache: dict[str, str] | None = None +_mame_clone_map_cache: dict[str, dict[str, str]] = {} -def get_mame_clone_map() -> dict[str, str]: - """Load and cache the MAME clone map (clone_name -> canonical_name).""" - global _mame_clone_map_cache - if _mame_clone_map_cache is not None: - return _mame_clone_map_cache - clone_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "_mame_clones.json", - ) +def get_mame_clone_map(bios_dir: str | None = None) -> dict[str, str]: + """The MAME clone map (clone_name -> canonical_name) of a scanned tree. + + dedup.py writes _mame_clones.json beside the tree it scanned, and the + map of one tree says nothing about another: read from the repository + root whatever ``--bios-dir`` said, an alternate tree's renamed clones + fell out of its packs. Without a tree the repository's own is read. + """ + if bios_dir is None: + bios_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "bios" + ) + clone_path = os.path.join(os.path.dirname(os.path.abspath(bios_dir)), "_mame_clones.json") + cached = _mame_clone_map_cache.get(clone_path) + if cached is not None: + return cached + clone_map: dict[str, str] = {} if os.path.exists(clone_path): with open(clone_path) as f: data = json.load(f) - _mame_clone_map_cache = {} for canonical, info in data.items(): for clone in info.get("clones", []): - _mame_clone_map_cache[clone] = canonical - else: - _mame_clone_map_cache = {} - return _mame_clone_map_cache + clone_map[clone] = canonical + _mame_clone_map_cache[clone_path] = clone_map + return clone_map diff --git a/scripts/generate_db.py b/scripts/generate_db.py index a8b05bc2..d41f3561 100644 --- a/scripts/generate_db.py +++ b/scripts/generate_db.py @@ -386,6 +386,9 @@ def main(): "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "total_files": len(files), "total_size": total_size, + # The tree these paths belong to: dedup writes _mame_clones.json + # beside it, and the resolver reads the map there, not at the repo root. + "bios_dir": os.path.normpath(args.bios_dir), "files": files, "indexes": indexes, } diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 05222791..c83743d1 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -6356,7 +6356,7 @@ struct BurnDriver BurnDrvneogeo = { db = self._gap_db("v4bios.zip", "a" * 40, "465c4e1c") original = cross_reference.get_mame_clone_map - cross_reference.get_mame_clone_map = lambda: {"bctvidbs.zip": "v4bios.zip"} + cross_reference.get_mame_clone_map = lambda *_a: {"bctvidbs.zip": "v4bios.zip"} try: entry = {"name": "bctvidbs.zip", "required": True} self.assertEqual(self._gap_source(entry, db), "bios")