diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py
index f82966b5..5a7c755d 100644
--- a/scripts/cross_reference.py
+++ b/scripts/cross_reference.py
@@ -361,6 +361,52 @@ def _content_matches(location: tuple[str, str], declared: dict[str, set[str]]) -
return any(actual[field] in values for field, values in declared.items())
+_tree_cache: dict[tuple[str, str], list[str]] = {}
+
+
+def _trees(data_root: str, bios_root: str) -> list[str]:
+ """Every directory under data/ and bios/, as a path with a trailing slash."""
+ key = (data_root, bios_root)
+ if key not in _tree_cache:
+ _tree_cache[key] = sorted(
+ f"{path.as_posix()}/"
+ for root in (data_root, bios_root) if Path(root).is_dir()
+ for path in Path(root).rglob("*")
+ if path.is_dir()
+ and not any(p.startswith(".") for p in path.relative_to(root).parts)
+ )
+ return _tree_cache[key]
+
+
+def held_copy(
+ entry: dict, data_root: str = "data", bios_root: str = "bios"
+) -> tuple[str, str] | None:
+ """The data/ file, directory or bios/ ZIP member holding an entry.
+
+ The database indexes the files under bios/, and an entry held otherwise
+ (specmate.rom as a member of spectrum_opus.zip, Dolphin's fonts in its
+ data directory) read as held with no word of where. Returns (path,
+ member), the member empty for a file or a directory. A file entry that
+ declares a hash is held only by a copy carrying it, as _data_hit judges.
+ """
+ names = [entry.get("name", ""), entry.get("path") or "", *(entry.get("aliases") or [])]
+ names = [str(n) for n in names if n]
+ if entry.get("type") == "directory" or any(n.endswith("/") for n in names[:1]):
+ for name in names:
+ tail = "/" + name.strip("/") + "/"
+ for tree in _trees(data_root, bios_root):
+ if tree.endswith(tail):
+ return tree, ""
+ return None
+ declared = _declared_content(entry)
+ locations = _supplemental_locations(data_root, bios_root)
+ for name in names:
+ for location in locations.get(name.rsplit("/", 1)[-1].lower(), ()):
+ if not declared or _content_matches(location, declared):
+ return location
+ return None
+
+
class FileConsumers:
"""Who reads each collected file, joined through the resolver.
@@ -459,6 +505,10 @@ class FileConsumers:
self.installed.setdefault(sha1, {}).setdefault(
platform, set()).add(str(entry.get("dest", "")))
+ def sha1_at(self, path: str) -> str | None:
+ """The file the database records at a collection path."""
+ return self._sha1_by_path.get(path)
+
def path_of(self, sha1: str) -> str:
"""The collection path the database records for a file."""
return str(self._db.get("files", {}).get(sha1, {}).get("path", ""))
@@ -742,17 +792,18 @@ def _cross_reference_profile(
continue
# --- resolve source provenance ---
- storage = f.get("storage", "")
- if storage in ("release", "large_file"):
+ # A release asset is held when the database records it, as any other
+ # file: storage says where the bytes are served from, not that the
+ # collection has them (kenji-nx's user-picked firmware package).
+ source = next(
+ (
+ found for version in versions[slot]
+ if (found := entry_source(version, index)) is not None
+ ),
+ "missing",
+ )
+ if source != "missing" and f.get("storage", "") in ("release", "large_file"):
source = "large_file"
- else:
- source = next(
- (
- found for version in versions[slot]
- if (found := entry_source(version, index)) is not None
- ),
- "missing",
- )
in_repo = source != "missing"
diff --git a/scripts/generate_site.py b/scripts/generate_site.py
index efcf28d7..b6c99522 100644
--- a/scripts/generate_site.py
+++ b/scripts/generate_site.py
@@ -15,6 +15,7 @@ from siterender import WIKI_SRC_DIR
import argparse
import csv
import hashlib
+import html
import io
import json
import os
@@ -48,7 +49,7 @@ from common import (
yaml_load,
)
from artifacts import ArtifactLockBusy, artifact_lock
-from cross_reference import FileConsumers
+from cross_reference import FileConsumers, held_copy
from nativemode import reads_file_contents
@@ -3121,18 +3122,21 @@ def _render_emulator_file(
details.extend(_directory_copies(f, consumers))
if consumers:
sha1 = consumers.sha1_of(f, f.get("path") or "", owner=owner)
- if sha1 and in_repo:
+ held = consumers.path_of(sha1) if sha1 and in_repo else ""
+ parts = held.split("/")
+ if len(parts) >= 3 and parts[0] == "bios":
# Which copy answers this entry, linked to its collection card
# (provenance, every hash, download): an entry declaring no hash
# said "in repo" without naming the file.
- held = consumers.path_of(sha1)
- parts = held.split("/")
- if len(parts) >= 3 and parts[0] == "bios":
- page = parts[1].lower().replace(" ", "-")
- details.append(
- f"Collection copy: [{_code(held)}]"
- f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})"
- )
+ page = parts[1].lower().replace(" ", "-")
+ details.append(
+ f"Collection copy: [{_code(held)}]"
+ f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})"
+ )
+ elif in_repo and not any(d.startswith("Collection copies:") for d in details):
+ copy = held_copy(f)
+ if copy:
+ details.append(_held_copy_line(copy, consumers))
plat_links = _platform_links(sha1, consumers) if sha1 else []
if plat_links:
details.append(f"Platforms: {', '.join(plat_links)}")
@@ -3189,6 +3193,33 @@ def _render_emulator_file(
return lines
+def _held_copy_line(copy: tuple[str, str], consumers: FileConsumers) -> str:
+ """Where a copy the database does not index sits: a ZIP member under
+ bios/, or a file or tree of a data directory the packs fetch."""
+ path, member = copy
+ parts = path.rstrip("/").split("/")
+ if member:
+ archive = _code(path)
+ zsha1 = consumers.sha1_at(path)
+ if zsha1 and len(parts) >= 3 and parts[0] == "bios":
+ page = urllib.parse.quote(parts[1].lower().replace(" ", "-"))
+ archive = f"[{archive}](../systems/{page}.md#sha1-{zsha1[:12]})"
+ return f"Collection copy: member {_code(member)} of {archive}"
+ if parts[0] == "data" and len(parts) >= 2:
+ ref, inside = parts[1], "/".join(parts[2:])
+ if path.endswith("/"):
+ where = _code(inside + "/") if inside else "the tree"
+ return f"Collection copy: {where} in the {_code(ref)} data directory"
+ info = (_DATA_MEMBERS.get(ref) or {}).get(inside) or {}
+ shown = _code(inside)
+ if info.get("sha1"):
+ shown = (f''
+ f"{html.escape(inside)}")
+ return f"Collection copy: {shown} in the {_code(ref)} data directory"
+ tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(path.rstrip('/'))}"
+ return f"Collection copy: [{_code(path)}]({tree})"
+
+
def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]:
"""What the collection holds under a directory entry, and where.
@@ -3425,8 +3456,6 @@ def _availability_check(
# under the name belongs to another game or release.
if not f.get("name") or f.get("unsourceable"):
return False
- if f.get("storage", "") in ("release", "large_file"):
- return True
return entry_source(f, index) is not None
return _file_available
diff --git a/scripts/verify.py b/scripts/verify.py
index 09947723..17913623 100644
--- a/scripts/verify.py
+++ b/scripts/verify.py
@@ -644,11 +644,10 @@ def find_undeclared_files(
entry["required"] = True
continue
- # Resolution: storage flag, then name, then path basename
- storage = f.get("storage", "")
- if storage in ("release", "large_file"):
- in_repo = True
- elif shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
+ # Resolution: name, then path basename
+ # storage: release names where the bytes are served from, not
+ # that the collection holds them: resolved like any file.
+ if shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
# The entry states what its content should be and the builder
# drops a copy that contradicts it, so content decides here
# too. A name match would not do: generic names collide across
diff --git a/tests/test_site_availability.py b/tests/test_site_availability.py
index a1e09fbe..12fa9052 100644
--- a/tests/test_site_availability.py
+++ b/tests/test_site_availability.py
@@ -33,6 +33,13 @@ class UnsourceableIsNotHeld(unittest.TestCase):
self.assertTrue(available({"name": "pak0.pk3"}))
self.assertFalse(available({"name": "pak0.pk3", "unsourceable": "retail data"}))
+ def test_a_release_asset_is_held_only_when_collected(self):
+ """storage: large_file said "in repo" for kenji-nx's user-picked
+ firmware package, which the collection does not hold."""
+ available = _availability_check(DB, set())
+ self.assertFalse(available({"name": ".zip", "storage": "large_file"}))
+ self.assertTrue(available({"name": "pak0.pk3", "storage": "release"}))
+
def test_the_badge_says_why(self):
entry = {"name": "pak0.pk3", "unsourceable": "retail data"}
badges = " ".join(_file_badges(entry, False))
@@ -57,5 +64,50 @@ class UnsourceableIsNotHeld(unittest.TestCase):
self.assertEqual(shown_held, [])
+class AHeldEntrySaysWhere(unittest.TestCase):
+ """164 entries wore the "in repo" badge with no copy named: they are held
+ as a ZIP member (zesarux's specmate.rom in spectrum_opus.zip) or in a
+ data directory (Dolphin's fonts), which the database does not index."""
+
+ def test_member_file_and_tree(self):
+ import hashlib
+ import tempfile
+ import zipfile
+
+ import generate_site
+ from cross_reference import FileConsumers, held_copy
+
+ with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
+ data, bios = Path(tmp) / "data", Path(tmp) / "bios"
+ (data / "dolphin-sys" / "GC").mkdir(parents=True)
+ (data / "dolphin-sys" / "GC" / "font_western.bin").write_bytes(b"font")
+ (bios / "Arcade" / "MAME").mkdir(parents=True)
+ archive = bios / "Arcade" / "MAME" / "spectrum_opus.zip"
+ with zipfile.ZipFile(archive, "w") as zf:
+ zf.writestr("specmate.rom", b"rom")
+ roots = {"data_root": str(data), "bios_root": str(bios)}
+ member = held_copy({"name": "specmate.rom",
+ "sha1": hashlib.sha1(b"rom").hexdigest()}, **roots)
+ self.assertEqual(member, (str(archive), "specmate.rom"))
+ # A declared hash the copy does not carry names no copy.
+ self.assertIsNone(held_copy({"name": "specmate.rom", "sha1": "0" * 40}, **roots))
+ self.assertEqual(held_copy({"name": "font_western.bin"}, **roots),
+ (str(data / "dolphin-sys/GC/font_western.bin"), ""))
+ self.assertEqual(held_copy({"name": "GC/", "type": "directory"}, **roots),
+ (f"{(data / 'dolphin-sys/GC').as_posix()}/", ""))
+
+ consumers = FileConsumers({"files": {"z" * 40: {
+ "path": "bios/Arcade/MAME/spectrum_opus.zip"}}}, data_dir_registry={})
+ line = generate_site._held_copy_line(
+ ("bios/Arcade/MAME/spectrum_opus.zip", "specmate.rom"), consumers)
+ self.assertEqual(line, "Collection copy: member `specmate.rom` of "
+ "[`bios/Arcade/MAME/spectrum_opus.zip`]"
+ "(../systems/arcade.md#sha1-zzzzzzzzzzzz)")
+ line = generate_site._held_copy_line(("data/dolphin-sys/GC/font_western.bin", ""),
+ consumers)
+ self.assertEqual(line, "Collection copy: `GC/font_western.bin` in the "
+ "`dolphin-sys` data directory")
+
+
if __name__ == "__main__":
unittest.main()