diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 7cf425616..9ffff4127 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -49,6 +49,7 @@ from hashing import parse_hash_list from validation import ( case_folded_slots, declaration_settles, + inner_rom_check, installed_with_emulator, outside_gap_scope, read_from_save_dir, @@ -476,7 +477,14 @@ class FileConsumers: entry, self._db, self._zip_contents, dest_hint=dest, data_dir_registry=self._data_dir_registry, ) - if not path or status in ("not_found", "hash_mismatch"): + if not path or status == "not_found": + return None + # Batocera pins naomi2.zip by the md5 of a ROM inside it: the resolver + # hands back the archive as a mismatch, and the member settles it as + # the builder does. + if status == "hash_mismatch" and not ( + entry.get("zipped_file") and inner_rom_check(entry, path) == "ok" + ): return None return self._sha1_by_path.get(path) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 701f97c82..ee0115e46 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4746,12 +4746,17 @@ def file_lookup_records( url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}") system_label = _lookup_system_name(system_id) for entry in system.get("files") or []: + # A hash of a ROM inside the archive (zipped_file) designates + # no collection file by itself: the archive holding it is the one. + archive = (consumers.sha1_of(entry) + if consumers is not None and entry.get("zipped_file") else None) for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): records.append({ "value": value, "algorithm": algorithm, "layer": "platform", "owner": key, "name": str(entry.get("name", "")), "url": url, **({"system_name": system_label} if system_label else {}), + **({"sha1": archive} if archive else {}), }) for key, profile in sorted(profiles.items()): if profile.get("type") in ("alias", "test"): @@ -4843,8 +4848,9 @@ def _held_under_every_hash(records: list[dict], files: dict, indexes: dict) -> l for record in records: if record["layer"] not in ("platform", "emulator"): continue - sha1 = _collection_sha1(files, indexes, record["algorithm"], record["value"]) - if sha1 is None: + sha1 = record.get("sha1") or _collection_sha1( + files, indexes, record["algorithm"], record["value"]) + if sha1 not in files: continue record["sha1"] = sha1 for algorithm in HASH_ALGORITHMS: diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index eeb910aa0..151e078c2 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib import json +import os import sqlite3 import sys import tempfile @@ -670,6 +671,38 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase): downloaded. files/?hash= reads one shard instead, and it must answer at least what the search answered: the card, the platforms, the profiles.""" + def test_an_archive_answers_for_the_rom_a_list_pins_inside_it(self): + """Batocera checks naomi2.zip by the md5 of 315-6146.bin inside it: + dropped on the lookup, the archive listed every setup but Batocera, + and 335 of its 337 zipped_file entries were unreachable that way.""" + import json as _json + + import generate_site # noqa: PLC0415 + from common import load_platform_config # noqa: PLC0415 + from cross_reference import FileConsumers # noqa: PLC0415 + from ziptools import build_zip_contents_index # noqa: PLC0415 + + root = Path(__file__).resolve().parents[1] + db = _json.loads((root / "database.json").read_text(encoding="utf-8")) + config = load_platform_config("batocera", str(root / "platforms")) + entry = next(e for system in config["systems"].values() for e in system.get("files") or [] + if e.get("name") == "naomi2.zip" and e.get("zipped_file")) + cwd = Path.cwd() + try: + os.chdir(root) + consumers = FileConsumers(db, build_zip_contents_index(db)) + archive = consumers.sha1_of(entry) + finally: + os.chdir(cwd) + if archive is None: + self.skipTest("no copy of naomi2.zip holding the pinned ROM") + coverages = {"batocera": {"config": {"systems": {"sega-naomi2": {"files": [entry]}}}}} + records = generate_site.file_lookup_records(db, coverages, {}, [], None, consumers) + found = {(r["owner"], r["value"]) for r in records if r["layer"] == "platform"} + self.assertIn(("batocera", archive), found) + pinned = next(r for r in records if r["layer"] == "platform" and r["algorithm"] == "md5") + self.assertEqual(pinned["sha1"], archive) + def test_a_data_directory_member_answers_every_hash(self): """Dolphin's ApprovedInis.json was found by its SHA1 only: its MD5 and CRC32 answered no match although the pack ships it.""" diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index 3f4470325..aa1252de9 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -502,7 +502,13 @@ const distinct = (items: Item[], key: (item: Item) => string) => { // read under every hash of that file, so the MD5, the SHA1, the CRC32 and // the SHA256 of one dump give one answer. Setups and emulators are counted // per file, by its SHA1. -const renderMatches = async (status: HTMLElement, result: HTMLElement, found: Item[]) => { +const renderMatches = async (status: HTMLElement, result: HTMLElement, matched: Item[]) => { + // A declaration that names the file holding the value: Batocera pins + // naomi2.zip by the MD5 of a ROM inside it, which no file of the + // collection has. + const searched = new Set(matched.map((item) => item.value)); + const carried = [...new Set(matched.flatMap((item) => (item.sha1 && !searched.has(item.sha1) ? [item.sha1] : [])))]; + const found = carried.length ? [...matched, ...(await lookupValues(carried).catch(() => [] as Item[]))] : matched; const held = distinct(found.filter((item) => item.layer === 'collection'), (item) => item.sha1 ?? item.url); const files = await Promise.all(held.map((item) => heldFile(item).catch(() => undefined))); const looked = new Set(found.map((item) => item.value));