diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index f82966b5..5a7c755d 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -361,6 +361,52 @@ def _content_matches(location: tuple[str, str], declared: dict[str, set[str]]) - return any(actual[field] in values for field, values in declared.items()) +_tree_cache: dict[tuple[str, str], list[str]] = {} + + +def _trees(data_root: str, bios_root: str) -> list[str]: + """Every directory under data/ and bios/, as a path with a trailing slash.""" + key = (data_root, bios_root) + if key not in _tree_cache: + _tree_cache[key] = sorted( + f"{path.as_posix()}/" + for root in (data_root, bios_root) if Path(root).is_dir() + for path in Path(root).rglob("*") + if path.is_dir() + and not any(p.startswith(".") for p in path.relative_to(root).parts) + ) + return _tree_cache[key] + + +def held_copy( + entry: dict, data_root: str = "data", bios_root: str = "bios" +) -> tuple[str, str] | None: + """The data/ file, directory or bios/ ZIP member holding an entry. + + The database indexes the files under bios/, and an entry held otherwise + (specmate.rom as a member of spectrum_opus.zip, Dolphin's fonts in its + data directory) read as held with no word of where. Returns (path, + member), the member empty for a file or a directory. A file entry that + declares a hash is held only by a copy carrying it, as _data_hit judges. + """ + names = [entry.get("name", ""), entry.get("path") or "", *(entry.get("aliases") or [])] + names = [str(n) for n in names if n] + if entry.get("type") == "directory" or any(n.endswith("/") for n in names[:1]): + for name in names: + tail = "/" + name.strip("/") + "/" + for tree in _trees(data_root, bios_root): + if tree.endswith(tail): + return tree, "" + return None + declared = _declared_content(entry) + locations = _supplemental_locations(data_root, bios_root) + for name in names: + for location in locations.get(name.rsplit("/", 1)[-1].lower(), ()): + if not declared or _content_matches(location, declared): + return location + return None + + class FileConsumers: """Who reads each collected file, joined through the resolver. @@ -459,6 +505,10 @@ class FileConsumers: self.installed.setdefault(sha1, {}).setdefault( platform, set()).add(str(entry.get("dest", ""))) + def sha1_at(self, path: str) -> str | None: + """The file the database records at a collection path.""" + return self._sha1_by_path.get(path) + def path_of(self, sha1: str) -> str: """The collection path the database records for a file.""" return str(self._db.get("files", {}).get(sha1, {}).get("path", "")) @@ -742,17 +792,18 @@ def _cross_reference_profile( continue # --- resolve source provenance --- - storage = f.get("storage", "") - if storage in ("release", "large_file"): + # A release asset is held when the database records it, as any other + # file: storage says where the bytes are served from, not that the + # collection has them (kenji-nx's user-picked firmware package). + source = next( + ( + found for version in versions[slot] + if (found := entry_source(version, index)) is not None + ), + "missing", + ) + if source != "missing" and f.get("storage", "") in ("release", "large_file"): source = "large_file" - else: - source = next( - ( - found for version in versions[slot] - if (found := entry_source(version, index)) is not None - ), - "missing", - ) in_repo = source != "missing" diff --git a/scripts/generate_site.py b/scripts/generate_site.py index efcf28d7..b6c99522 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -15,6 +15,7 @@ from siterender import WIKI_SRC_DIR import argparse import csv import hashlib +import html import io import json import os @@ -48,7 +49,7 @@ from common import ( yaml_load, ) from artifacts import ArtifactLockBusy, artifact_lock -from cross_reference import FileConsumers +from cross_reference import FileConsumers, held_copy from nativemode import reads_file_contents @@ -3121,18 +3122,21 @@ def _render_emulator_file( details.extend(_directory_copies(f, consumers)) if consumers: sha1 = consumers.sha1_of(f, f.get("path") or "", owner=owner) - if sha1 and in_repo: + held = consumers.path_of(sha1) if sha1 and in_repo else "" + parts = held.split("/") + if len(parts) >= 3 and parts[0] == "bios": # Which copy answers this entry, linked to its collection card # (provenance, every hash, download): an entry declaring no hash # said "in repo" without naming the file. - held = consumers.path_of(sha1) - parts = held.split("/") - if len(parts) >= 3 and parts[0] == "bios": - page = parts[1].lower().replace(" ", "-") - details.append( - f"Collection copy: [{_code(held)}]" - f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" - ) + page = parts[1].lower().replace(" ", "-") + details.append( + f"Collection copy: [{_code(held)}]" + f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" + ) + elif in_repo and not any(d.startswith("Collection copies:") for d in details): + copy = held_copy(f) + if copy: + details.append(_held_copy_line(copy, consumers)) plat_links = _platform_links(sha1, consumers) if sha1 else [] if plat_links: details.append(f"Platforms: {', '.join(plat_links)}") @@ -3189,6 +3193,33 @@ def _render_emulator_file( return lines +def _held_copy_line(copy: tuple[str, str], consumers: FileConsumers) -> str: + """Where a copy the database does not index sits: a ZIP member under + bios/, or a file or tree of a data directory the packs fetch.""" + path, member = copy + parts = path.rstrip("/").split("/") + if member: + archive = _code(path) + zsha1 = consumers.sha1_at(path) + if zsha1 and len(parts) >= 3 and parts[0] == "bios": + page = urllib.parse.quote(parts[1].lower().replace(" ", "-")) + archive = f"[{archive}](../systems/{page}.md#sha1-{zsha1[:12]})" + return f"Collection copy: member {_code(member)} of {archive}" + if parts[0] == "data" and len(parts) >= 2: + ref, inside = parts[1], "/".join(parts[2:]) + if path.endswith("/"): + where = _code(inside + "/") if inside else "the tree" + return f"Collection copy: {where} in the {_code(ref)} data directory" + info = (_DATA_MEMBERS.get(ref) or {}).get(inside) or {} + shown = _code(inside) + if info.get("sha1"): + shown = (f'' + f"{html.escape(inside)}") + return f"Collection copy: {shown} in the {_code(ref)} data directory" + tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(path.rstrip('/'))}" + return f"Collection copy: [{_code(path)}]({tree})" + + def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]: """What the collection holds under a directory entry, and where. @@ -3425,8 +3456,6 @@ def _availability_check( # under the name belongs to another game or release. if not f.get("name") or f.get("unsourceable"): return False - if f.get("storage", "") in ("release", "large_file"): - return True return entry_source(f, index) is not None return _file_available diff --git a/scripts/verify.py b/scripts/verify.py index 09947723..17913623 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -644,11 +644,10 @@ def find_undeclared_files( entry["required"] = True continue - # Resolution: storage flag, then name, then path basename - storage = f.get("storage", "") - if storage in ("release", "large_file"): - in_repo = True - elif shipped_on_mismatch and (f.get("md5") or f.get("sha1")): + # Resolution: name, then path basename + # storage: release names where the bytes are served from, not + # that the collection holds them: resolved like any file. + if shipped_on_mismatch and (f.get("md5") or f.get("sha1")): # The entry states what its content should be and the builder # drops a copy that contradicts it, so content decides here # too. A name match would not do: generic names collide across diff --git a/tests/test_site_availability.py b/tests/test_site_availability.py index a1e09fbe..12fa9052 100644 --- a/tests/test_site_availability.py +++ b/tests/test_site_availability.py @@ -33,6 +33,13 @@ class UnsourceableIsNotHeld(unittest.TestCase): self.assertTrue(available({"name": "pak0.pk3"})) self.assertFalse(available({"name": "pak0.pk3", "unsourceable": "retail data"})) + def test_a_release_asset_is_held_only_when_collected(self): + """storage: large_file said "in repo" for kenji-nx's user-picked + firmware package, which the collection does not hold.""" + available = _availability_check(DB, set()) + self.assertFalse(available({"name": ".zip", "storage": "large_file"})) + self.assertTrue(available({"name": "pak0.pk3", "storage": "release"})) + def test_the_badge_says_why(self): entry = {"name": "pak0.pk3", "unsourceable": "retail data"} badges = " ".join(_file_badges(entry, False)) @@ -57,5 +64,50 @@ class UnsourceableIsNotHeld(unittest.TestCase): self.assertEqual(shown_held, []) +class AHeldEntrySaysWhere(unittest.TestCase): + """164 entries wore the "in repo" badge with no copy named: they are held + as a ZIP member (zesarux's specmate.rom in spectrum_opus.zip) or in a + data directory (Dolphin's fonts), which the database does not index.""" + + def test_member_file_and_tree(self): + import hashlib + import tempfile + import zipfile + + import generate_site + from cross_reference import FileConsumers, held_copy + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + data, bios = Path(tmp) / "data", Path(tmp) / "bios" + (data / "dolphin-sys" / "GC").mkdir(parents=True) + (data / "dolphin-sys" / "GC" / "font_western.bin").write_bytes(b"font") + (bios / "Arcade" / "MAME").mkdir(parents=True) + archive = bios / "Arcade" / "MAME" / "spectrum_opus.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("specmate.rom", b"rom") + roots = {"data_root": str(data), "bios_root": str(bios)} + member = held_copy({"name": "specmate.rom", + "sha1": hashlib.sha1(b"rom").hexdigest()}, **roots) + self.assertEqual(member, (str(archive), "specmate.rom")) + # A declared hash the copy does not carry names no copy. + self.assertIsNone(held_copy({"name": "specmate.rom", "sha1": "0" * 40}, **roots)) + self.assertEqual(held_copy({"name": "font_western.bin"}, **roots), + (str(data / "dolphin-sys/GC/font_western.bin"), "")) + self.assertEqual(held_copy({"name": "GC/", "type": "directory"}, **roots), + (f"{(data / 'dolphin-sys/GC').as_posix()}/", "")) + + consumers = FileConsumers({"files": {"z" * 40: { + "path": "bios/Arcade/MAME/spectrum_opus.zip"}}}, data_dir_registry={}) + line = generate_site._held_copy_line( + ("bios/Arcade/MAME/spectrum_opus.zip", "specmate.rom"), consumers) + self.assertEqual(line, "Collection copy: member `specmate.rom` of " + "[`bios/Arcade/MAME/spectrum_opus.zip`]" + "(../systems/arcade.md#sha1-zzzzzzzzzzzz)") + line = generate_site._held_copy_line(("data/dolphin-sys/GC/font_western.bin", ""), + consumers) + self.assertEqual(line, "Collection copy: `GC/font_western.bin` in the " + "`dolphin-sys` data directory") + + if __name__ == "__main__": unittest.main()