feat: find archive members by their CRC32

The lookup held a member only where a profile's contents: declares it,
so 26,014 of the 33,403 members of collection archives answered nothing.
Each member's CRC32 is now an archive-member record leading to its
archive's card, read from the central directory, and files/ names the
archive holding it.

schemas/site-api-envelope.schema.json

@@ -389,11 +389,16 @@
                 "layer": {
                   "enum": [
                     "collection",
+                    "archive-member",
                     "data-directory",
                     "platform",
                     "emulator"
                   ]
                 },
+                "archive": {
+                  "type": "string",
+                  "description": "On an archive-member record: the name of the collection archive holding the member, whose SHA1 is `sha1`. Its value is the member's CRC32, read from the archive's central directory."
+                },
                 "name": {
                   "type": "string"
                 },

scripts/generate_site.py

@@ -2201,8 +2201,8 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
         "One hash without the whole list: `api/v1/lookup/<xx>.json`, where `<xx>` "
         "is the first two characters of the hash in lowercase, holds every SHA1, "
         "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` "
-        "(the collection, a data directory, a platform's list or an emulator "
-        "profile), the file `name`, the `url` of the page that shows it and, "
+        "(the collection, a member of a collection archive, a data directory, a "
+        "platform's list or an emulator profile), the file `name`, the `url` of the page that shows it and, "
         "when one is known, the `system_name` the site gives its system. "
         "[Find or check a file](files/index.md) reads it.",
         "",
@@ -2226,7 +2226,7 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
         "  const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);",
         "  if (!response.ok) return [];",
         "  const shard = await response.json();",
-        "  // layer: collection, data-directory, platform or emulator. owner: which one.",
+        "  // layer: collection, archive-member, data-directory, platform or emulator.",
         "  return shard.items.filter((item) => item.value === sha1);",
         "}",
         "```",
@@ -4924,6 +4924,31 @@ def _data_member_digests(
     return found
 
 
+def _archive_member_records(entry: dict, sha1: str, url: str,
+                            root: Path = _REPO_ROOT) -> list[dict]:
+    """The CRC32 of each member of a collection archive, read from its
+    central directory.
+
+    A pasted CRC of a ROM the collection holds only inside a set (logica2.u2
+    in a500.zip, 8484f426) found nothing: 26,014 of 33,403 members were in
+    no record. The record leads to the archive's card and names it.
+    """
+    path = root / str(entry.get("path", ""))
+    if path.suffix.lower() != ".zip" or not path.is_file():
+        return []
+    try:
+        with zipfile.ZipFile(path) as archive:
+            members = [(info.filename, info.CRC) for info in archive.infolist()
+                       if not info.is_dir()]
+    except (OSError, zipfile.BadZipFile) as exc:
+        print(f"  WARNING: {path}: members unreadable ({exc})", file=sys.stderr)
+        return []
+    return [{"value": f"{crc:08x}", "algorithm": "crc32", "layer": "archive-member",
+             "name": member.rsplit("/", 1)[-1], "archive": str(entry.get("name", "")),
+             "sha1": sha1, "url": url}
+            for member, crc in members]
+
+
 def file_lookup_records(
     db: dict,
     coverages: dict,
@@ -4968,6 +4993,7 @@ def file_lookup_records(
                     "name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"],
                     **({"system_name": held_by} if held_by else {}),
                 })
+        records += _archive_member_records(entry, sha1, item["url"])
     for key, coverage in sorted(coverages.items()):
         for system_id, system in sorted((coverage["config"].get("systems") or {}).items()):
             url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}")

tests/test_site_exports.py

@@ -785,6 +785,53 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase):
             self.assertNotIn("no.json", names)
 
 
+class AnArchiveMemberIsFoundByItsCrc(unittest.TestCase):
+    """A pasted CRC of logica2.u2 (8484f426), which the collection holds
+    inside a500.zip, answered that no file the site knows has it: the
+    lookup held the members a profile's `contents:` declares, 7,389 of
+    33,403."""
+
+    def test_each_member_is_a_record_leading_to_its_archive(self):
+        import zipfile  # noqa: PLC0415
+
+        import generate_site  # noqa: PLC0415
+
+        with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp:
+            root = Path(tmp)
+            archive = root / "bios" / "Arcade" / "Arcade" / "a500.zip"
+            archive.parent.mkdir(parents=True)
+            with zipfile.ZipFile(archive, "w") as handle:
+                handle.writestr("logica2.u2", b"rom")
+                handle.writestr("sub/kick.rom", b"kick")
+            entry = {"name": "a500.zip", "path": "bios/Arcade/Arcade/a500.zip"}
+            records = generate_site._archive_member_records(entry, "ab" * 20, "/retrobios/x/", root)
+            plain = generate_site._archive_member_records(
+                {"name": "a.bin", "path": "bios/a.bin"}, "cd" * 20, "/retrobios/y/", root)
+        self.assertEqual(plain, [])
+        import zlib  # noqa: PLC0415
+
+        self.assertEqual(records, [
+            {"value": f"{zlib.crc32(b'rom'):08x}", "algorithm": "crc32", "layer": "archive-member",
+             "name": "logica2.u2", "archive": "a500.zip", "sha1": "ab" * 20, "url": "/retrobios/x/"},
+            {"value": f"{zlib.crc32(b'kick'):08x}", "algorithm": "crc32", "layer": "archive-member",
+             "name": "kick.rom", "archive": "a500.zip", "sha1": "ab" * 20, "url": "/retrobios/x/"},
+        ])
+
+    def test_the_lookup_and_the_schema_carry_the_layer(self):
+        import json as _json  # noqa: PLC0415
+
+        import generate_site  # noqa: PLC0415
+
+        source = Path(generate_site.__file__).read_text(encoding="utf-8")
+        self.assertIn("records += _archive_member_records(entry, sha1, item[\"url\"])", source)
+        schema = _json.loads((Path(__file__).resolve().parents[1] / "schemas"
+                              / "site-api-envelope.schema.json").read_text())
+        self.assertIn('"archive-member"', _json.dumps(schema))
+        script = (Path(__file__).resolve().parents[1] / "website" / "src" / "scripts"
+                  / "site.ts").read_text(encoding="utf-8")
+        self.assertIn("byLayer('archive-member')", script)
+
+
 class AFileHasAnAddressByContent(unittest.TestCase):
     """mt32_control.rom moved from the Commodore page to the Roland page and
     every link to its card broke: the anchor was content-based, the page

website/src/scripts/site.ts

@@ -21,6 +21,8 @@ type Item = {
 	owner?: string;
 	sha1?: string;
 	system_name?: string;
+	// On an archive-member record: the collection archive holding the member.
+	archive?: string;
 };
 
 const siteRoot = () => new URL(import.meta.env.BASE_URL.replace(/\/?$/, '/'), location.origin);
@@ -521,7 +523,9 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched:
 	// naomi2.zip by the MD5 of a ROM inside it, which no file of the
 	// collection has.
 	const searched = new Set(matched.map((item) => item.value));
-	const carried = [...new Set(matched.flatMap((item) => (item.sha1 && !searched.has(item.sha1) ? [item.sha1] : [])))];
+	// A member's record names its archive, which is not the file searched for.
+	const carried = [...new Set(matched.flatMap((item) =>
+		(item.sha1 && item.layer !== 'archive-member' && !searched.has(item.sha1) ? [item.sha1] : [])))];
 	const found = carried.length ? [...matched, ...(await lookupValues(carried).catch(() => [] as Item[]))] : matched;
 	const held = distinct(found.filter((item) => item.layer === 'collection'), (item) => item.sha1 ?? item.url);
 	const files = await Promise.all(held.map((item) => heldFile(item).catch(() => undefined)));
@@ -550,8 +554,11 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched:
 	const readEntries = emulators.map((item) => readerEntry(reader.get(item.owner ?? ''), item, valuesOf(item)));
 	const setups = new Set(platforms.map((item) => item.owner));
 
+	const members = distinct(byLayer('archive-member'), (item) => `${item.sha1}|${item.name}`);
 	const counted = [
-		held.length ? count(held.length, 'file in the collection', 'files in the collection') : 'not in the collection',
+		held.length ? count(held.length, 'file in the collection', 'files in the collection')
+			: members.length ? count(members.length, 'member of a collection archive', 'members of collection archives')
+			: 'not in the collection',
 		...(platforms.length ? [`${count(platforms.length, 'declaration', 'declarations')} by ${count(setups.size, 'setup', 'setups')}`] : []),
 		...(owners.length ? [count(owners.length, 'emulator that reads it', 'emulators that read it')] : []),
 	];
@@ -606,7 +613,17 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched:
 			),
 		);
 	});
-	if (!held.length) blocks.push(paragraph('No file in the collection has this hash.'));
+	if (!held.length && !members.length) blocks.push(paragraph('No file in the collection has this hash.'));
+	if (members.length) {
+		// A ROM the collection holds inside a set: the set's card, by its name.
+		const list = document.createElement('ul');
+		for (const item of members) {
+			const row = document.createElement('li');
+			row.append(code(item.name ?? item.value), ' in ', link(item.url, code(item.archive ?? '')));
+			list.append(row);
+		}
+		blocks.push(paragraph(strong('Inside an archive of the collection')), list);
+	}
 	const shipped = byLayer('data-directory');
 	if (shipped.length) {
 		const list = document.createElement('ul');