fix: name where a held profile entry sits

164 entries wore the in repo badge with no copy named: they are held as
a ZIP member or in a data directory, which the database does not index.
held_copy() finds that copy and the card names it. storage: large_file
no longer counts as held by itself: kenji-nx's user-picked firmware
package read as held although nothing collects it.

scripts/cross_reference.py

@@ -361,6 +361,52 @@ def _content_matches(location: tuple[str, str], declared: dict[str, set[str]]) -
     return any(actual[field] in values for field, values in declared.items())
 
 
+_tree_cache: dict[tuple[str, str], list[str]] = {}
+
+
+def _trees(data_root: str, bios_root: str) -> list[str]:
+    """Every directory under data/ and bios/, as a path with a trailing slash."""
+    key = (data_root, bios_root)
+    if key not in _tree_cache:
+        _tree_cache[key] = sorted(
+            f"{path.as_posix()}/"
+            for root in (data_root, bios_root) if Path(root).is_dir()
+            for path in Path(root).rglob("*")
+            if path.is_dir()
+            and not any(p.startswith(".") for p in path.relative_to(root).parts)
+        )
+    return _tree_cache[key]
+
+
+def held_copy(
+    entry: dict, data_root: str = "data", bios_root: str = "bios"
+) -> tuple[str, str] | None:
+    """The data/ file, directory or bios/ ZIP member holding an entry.
+
+    The database indexes the files under bios/, and an entry held otherwise
+    (specmate.rom as a member of spectrum_opus.zip, Dolphin's fonts in its
+    data directory) read as held with no word of where. Returns (path,
+    member), the member empty for a file or a directory. A file entry that
+    declares a hash is held only by a copy carrying it, as _data_hit judges.
+    """
+    names = [entry.get("name", ""), entry.get("path") or "", *(entry.get("aliases") or [])]
+    names = [str(n) for n in names if n]
+    if entry.get("type") == "directory" or any(n.endswith("/") for n in names[:1]):
+        for name in names:
+            tail = "/" + name.strip("/") + "/"
+            for tree in _trees(data_root, bios_root):
+                if tree.endswith(tail):
+                    return tree, ""
+        return None
+    declared = _declared_content(entry)
+    locations = _supplemental_locations(data_root, bios_root)
+    for name in names:
+        for location in locations.get(name.rsplit("/", 1)[-1].lower(), ()):
+            if not declared or _content_matches(location, declared):
+                return location
+    return None
+
+
 class FileConsumers:
     """Who reads each collected file, joined through the resolver.
 
@@ -459,6 +505,10 @@ class FileConsumers:
                     self.installed.setdefault(sha1, {}).setdefault(
                         platform, set()).add(str(entry.get("dest", "")))
 
+    def sha1_at(self, path: str) -> str | None:
+        """The file the database records at a collection path."""
+        return self._sha1_by_path.get(path)
+
     def path_of(self, sha1: str) -> str:
         """The collection path the database records for a file."""
         return str(self._db.get("files", {}).get(sha1, {}).get("path", ""))
@@ -742,17 +792,18 @@ def _cross_reference_profile(
             continue
 
         # --- resolve source provenance ---
-        storage = f.get("storage", "")
+        # A release asset is held when the database records it, as any other
-        if storage in ("release", "large_file"):
+        # file: storage says where the bytes are served from, not that the
+        # collection has them (kenji-nx's user-picked firmware package).
+        source = next(
+            (
+                found for version in versions[slot]
+                if (found := entry_source(version, index)) is not None
+            ),
+            "missing",
+        )
+        if source != "missing" and f.get("storage", "") in ("release", "large_file"):
             source = "large_file"
-        else:
-            source = next(
-                (
-                    found for version in versions[slot]
-                    if (found := entry_source(version, index)) is not None
-                ),
-                "missing",
-            )
 
         in_repo = source != "missing"
 

scripts/generate_site.py

@@ -15,6 +15,7 @@ from siterender import WIKI_SRC_DIR
 import argparse
 import csv
 import hashlib
+import html
 import io
 import json
 import os
@@ -48,7 +49,7 @@ from common import (
     yaml_load,
 )
 from artifacts import ArtifactLockBusy, artifact_lock
-from cross_reference import FileConsumers
+from cross_reference import FileConsumers, held_copy
 from nativemode import reads_file_contents
 
 
@@ -3121,18 +3122,21 @@ def _render_emulator_file(
         details.extend(_directory_copies(f, consumers))
     if consumers:
         sha1 = consumers.sha1_of(f, f.get("path") or "", owner=owner)
-        if sha1 and in_repo:
+        held = consumers.path_of(sha1) if sha1 and in_repo else ""
+        parts = held.split("/")
+        if len(parts) >= 3 and parts[0] == "bios":
             # Which copy answers this entry, linked to its collection card
             # (provenance, every hash, download): an entry declaring no hash
             # said "in repo" without naming the file.
-            held = consumers.path_of(sha1)
+            page = parts[1].lower().replace(" ", "-")
-            parts = held.split("/")
+            details.append(
-            if len(parts) >= 3 and parts[0] == "bios":
+                f"Collection copy: [{_code(held)}]"
-                page = parts[1].lower().replace(" ", "-")
+                f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})"
-                details.append(
+            )
-                    f"Collection copy: [{_code(held)}]"
+        elif in_repo and not any(d.startswith("Collection copies:") for d in details):
-                    f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})"
+            copy = held_copy(f)
-                )
+            if copy:
+                details.append(_held_copy_line(copy, consumers))
         plat_links = _platform_links(sha1, consumers) if sha1 else []
         if plat_links:
             details.append(f"Platforms: {', '.join(plat_links)}")
@@ -3189,6 +3193,33 @@ def _render_emulator_file(
     return lines
 
 
+def _held_copy_line(copy: tuple[str, str], consumers: FileConsumers) -> str:
+    """Where a copy the database does not index sits: a ZIP member under
+    bios/, or a file or tree of a data directory the packs fetch."""
+    path, member = copy
+    parts = path.rstrip("/").split("/")
+    if member:
+        archive = _code(path)
+        zsha1 = consumers.sha1_at(path)
+        if zsha1 and len(parts) >= 3 and parts[0] == "bios":
+            page = urllib.parse.quote(parts[1].lower().replace(" ", "-"))
+            archive = f"[{archive}](../systems/{page}.md#sha1-{zsha1[:12]})"
+        return f"Collection copy: member {_code(member)} of {archive}"
+    if parts[0] == "data" and len(parts) >= 2:
+        ref, inside = parts[1], "/".join(parts[2:])
+        if path.endswith("/"):
+            where = _code(inside + "/") if inside else "the tree"
+            return f"Collection copy: {where} in the {_code(ref)} data directory"
+        info = (_DATA_MEMBERS.get(ref) or {}).get(inside) or {}
+        shown = _code(inside)
+        if info.get("sha1"):
+            shown = (f'<a href="../../files/?sha1={info["sha1"]}">'
+                     f"<code>{html.escape(inside)}</code></a>")
+        return f"Collection copy: {shown} in the {_code(ref)} data directory"
+    tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(path.rstrip('/'))}"
+    return f"Collection copy: [{_code(path)}]({tree})"
+
+
 def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]:
     """What the collection holds under a directory entry, and where.
 
@@ -3425,8 +3456,6 @@ def _availability_check(
         # under the name belongs to another game or release.
         if not f.get("name") or f.get("unsourceable"):
             return False
-        if f.get("storage", "") in ("release", "large_file"):
-            return True
         return entry_source(f, index) is not None
 
     return _file_available

scripts/verify.py

@@ -644,11 +644,10 @@ def find_undeclared_files(
                         entry["required"] = True
                     continue
 
-                # Resolution: storage flag, then name, then path basename
+                # Resolution: name, then path basename
-                storage = f.get("storage", "")
+                # storage: release names where the bytes are served from, not
-                if storage in ("release", "large_file"):
+                # that the collection holds them: resolved like any file.
-                    in_repo = True
+                if shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
-                elif shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
                     # The entry states what its content should be and the builder
                     # drops a copy that contradicts it, so content decides here
                     # too. A name match would not do: generic names collide across

tests/test_site_availability.py

@@ -33,6 +33,13 @@ class UnsourceableIsNotHeld(unittest.TestCase):
         self.assertTrue(available({"name": "pak0.pk3"}))
         self.assertFalse(available({"name": "pak0.pk3", "unsourceable": "retail data"}))
 
+    def test_a_release_asset_is_held_only_when_collected(self):
+        """storage: large_file said "in repo" for kenji-nx's user-picked
+        firmware package, which the collection does not hold."""
+        available = _availability_check(DB, set())
+        self.assertFalse(available({"name": "<firmware>.zip", "storage": "large_file"}))
+        self.assertTrue(available({"name": "pak0.pk3", "storage": "release"}))
+
     def test_the_badge_says_why(self):
         entry = {"name": "pak0.pk3", "unsourceable": "retail data"}
         badges = " ".join(_file_badges(entry, False))
@@ -57,5 +64,50 @@ class UnsourceableIsNotHeld(unittest.TestCase):
         self.assertEqual(shown_held, [])
 
 
+class AHeldEntrySaysWhere(unittest.TestCase):
+    """164 entries wore the "in repo" badge with no copy named: they are held
+    as a ZIP member (zesarux's specmate.rom in spectrum_opus.zip) or in a
+    data directory (Dolphin's fonts), which the database does not index."""
+
+    def test_member_file_and_tree(self):
+        import hashlib
+        import tempfile
+        import zipfile
+
+        import generate_site
+        from cross_reference import FileConsumers, held_copy
+
+        with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
+            data, bios = Path(tmp) / "data", Path(tmp) / "bios"
+            (data / "dolphin-sys" / "GC").mkdir(parents=True)
+            (data / "dolphin-sys" / "GC" / "font_western.bin").write_bytes(b"font")
+            (bios / "Arcade" / "MAME").mkdir(parents=True)
+            archive = bios / "Arcade" / "MAME" / "spectrum_opus.zip"
+            with zipfile.ZipFile(archive, "w") as zf:
+                zf.writestr("specmate.rom", b"rom")
+            roots = {"data_root": str(data), "bios_root": str(bios)}
+            member = held_copy({"name": "specmate.rom",
+                                "sha1": hashlib.sha1(b"rom").hexdigest()}, **roots)
+            self.assertEqual(member, (str(archive), "specmate.rom"))
+            # A declared hash the copy does not carry names no copy.
+            self.assertIsNone(held_copy({"name": "specmate.rom", "sha1": "0" * 40}, **roots))
+            self.assertEqual(held_copy({"name": "font_western.bin"}, **roots),
+                             (str(data / "dolphin-sys/GC/font_western.bin"), ""))
+            self.assertEqual(held_copy({"name": "GC/", "type": "directory"}, **roots),
+                             (f"{(data / 'dolphin-sys/GC').as_posix()}/", ""))
+
+        consumers = FileConsumers({"files": {"z" * 40: {
+            "path": "bios/Arcade/MAME/spectrum_opus.zip"}}}, data_dir_registry={})
+        line = generate_site._held_copy_line(
+            ("bios/Arcade/MAME/spectrum_opus.zip", "specmate.rom"), consumers)
+        self.assertEqual(line, "Collection copy: member `specmate.rom` of "
+                         "[`bios/Arcade/MAME/spectrum_opus.zip`]"
+                         "(../systems/arcade.md#sha1-zzzzzzzzzzzz)")
+        line = generate_site._held_copy_line(("data/dolphin-sys/GC/font_western.bin", ""),
+                                             consumers)
+        self.assertEqual(line, "Collection copy: `GC/font_western.bin` in the "
+                         "`dolphin-sys` data directory")
+
+
 if __name__ == "__main__":
     unittest.main()