diff --git a/scripts/generate_site.py b/scripts/generate_site.py index de984fe3d..701f97c82 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4669,6 +4669,34 @@ def _console_system_names(profiles: dict, coverages: dict) -> dict[tuple[str, st return {key: next(iter(names)) for key, names in found.items() if len(names) == 1} +def _data_member_digests( + members: dict, data_registry: dict | None = None, root: Path = _REPO_ROOT, +) -> dict[str, dict[str, str]]: + """sha1 -> every hash the lookup takes, for each data directory member + whose copy in the tree's local cache has the sha1 the manifest records. + The manifests carry sha1 alone: the MD5 of Dolphin's ApprovedInis.json + found nothing although the pack ships it.""" + from common import load_data_dir_registry # noqa: PLC0415 + from hashing import compute_hashes # noqa: PLC0415 + + if data_registry is None: + data_registry = load_data_dir_registry(str(root / "platforms")) + found: dict[str, dict[str, str]] = {} + for ref, tree in sorted(members.items()): + cache = (data_registry.get(ref) or {}).get("local_cache") + if not cache: + continue + for member, info in sorted(tree.items()): + sha1 = str(info.get("sha1", "")) + path = root / cache / member + if not sha1 or sha1 in found or not path.is_file(): + continue + digests = compute_hashes(path, frozenset(HASH_ALGORITHMS)) + if digests.get("sha1") == sha1: + found[sha1] = {algo: digests[algo].lower() for algo in HASH_ALGORITHMS} + return found + + def file_lookup_records( db: dict, coverages: dict, @@ -4690,15 +4718,19 @@ def file_lookup_records( str(info.get("sha1", "")): member.rsplit("/", 1)[-1] for tree in (members or {}).values() for member, info in sorted(tree.items()) } + member_digests = _data_member_digests(members or {}) records: list[dict] = [] for item in locations: sha1 = item["sha1"] entry = files.get(sha1) if entry is None: - # A data directory member: the manifests record its sha1 only. - records.append({"value": sha1, "algorithm": "sha1", "layer": "data-directory", - "name": member_names.get(sha1, ""), "sha1": sha1, - "url": item["url"]}) + # A data directory member: the manifests record its sha1, the + # tree on disk the other three. + digests = member_digests.get(sha1) or {"sha1": sha1} + records.extend({"value": digests[algorithm], "algorithm": algorithm, + "layer": "data-directory", "name": member_names.get(sha1, ""), + "sha1": sha1, "url": item["url"]} + for algorithm in HASH_ALGORITHMS if algorithm in digests) continue parts = str(entry.get("path", "")).split("/") held_by = named.get((_slug(parts[1]), parts[2])) if len(parts) > 3 else None diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 1a3dea8fe..eeb910aa0 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -670,6 +670,37 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase): downloaded. files/?hash= reads one shard instead, and it must answer at least what the search answered: the card, the platforms, the profiles.""" + def test_a_data_directory_member_answers_every_hash(self): + """Dolphin's ApprovedInis.json was found by its SHA1 only: its MD5 + and CRC32 answered no match although the pack ships it.""" + import hashlib + import tempfile + import zlib + + import generate_site # noqa: PLC0415 + + body = b"approved inis\n" + sha1 = hashlib.sha1(body).hexdigest() + with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp: + member = Path(tmp, "data", "dolphin-sys", "ApprovedInis.json") + member.parent.mkdir(parents=True) + member.write_bytes(body) + stale = Path(tmp, "data", "dolphin-sys", "Stale.json") + stale.write_bytes(b"other bytes") + members = {"dolphin-sys": {"ApprovedInis.json": {"sha1": sha1}, + "Stale.json": {"sha1": "ef" * 20}}} + registry = {"dolphin-sys": {"local_cache": "data/dolphin-sys"}} + digests = generate_site._data_member_digests(members, registry, Path(tmp)) + self.assertEqual(digests, {sha1: { + "sha1": sha1, "md5": hashlib.md5(body).hexdigest(), + "sha256": hashlib.sha256(body).hexdigest(), + "crc32": f"{zlib.crc32(body):08x}"}}) + with mock.patch.object(generate_site, "_data_member_digests", return_value=digests): + records = generate_site.file_lookup_records( + {"files": {}}, {}, {}, [{"sha1": sha1, "url": "u"}], + {"dolphin-sys": {"ApprovedInis.json": {"sha1": sha1}}}) + self.assertEqual({r["algorithm"]: r["value"] for r in records}, digests[sha1]) + def test_records_cover_every_layer_and_shard_by_prefix(self): import json as _json import tempfile