From b23931e14d95c60418660e4434cc86bb6ddc06dd Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:12:20 +0100 Subject: [PATCH] feat: find archive members by their CRC32 The lookup held a member only where a profile's contents: declares it, so 26,014 of the 33,403 members of collection archives answered nothing. Each member's CRC32 is now an archive-member record leading to its archive's card, read from the central directory, and files/ names the archive holding it. --- schemas/site-api-envelope.schema.json | 5 +++ scripts/generate_site.py | 32 ++++++++++++++++-- tests/test_site_exports.py | 47 +++++++++++++++++++++++++++ website/src/scripts/site.ts | 23 +++++++++++-- 4 files changed, 101 insertions(+), 6 deletions(-) diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index af4962366..4f45a180f 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -389,11 +389,16 @@ "layer": { "enum": [ "collection", + "archive-member", "data-directory", "platform", "emulator" ] }, + "archive": { + "type": "string", + "description": "On an archive-member record: the name of the collection archive holding the member, whose SHA1 is `sha1`. Its value is the member's CRC32, read from the archive's central directory." + }, "name": { "type": "string" }, diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 1cfdfa100..f9e9df03f 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2201,8 +2201,8 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N "One hash without the whole list: `api/v1/lookup/.json`, where `` " "is the first two characters of the hash in lowercase, holds every SHA1, " "MD5, CRC32 and SHA256 starting with them. Each record names its `layer` " - "(the collection, a data directory, a platform's list or an emulator " - "profile), the file `name`, the `url` of the page that shows it and, " + "(the collection, a member of a collection archive, a data directory, a " + "platform's list or an emulator profile), the file `name`, the `url` of the page that shows it and, " "when one is known, the `system_name` the site gives its system. " "[Find or check a file](files/index.md) reads it.", "", @@ -2226,7 +2226,7 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N " const response = await fetch(`${base}${sha1.slice(0, 2)}.json`);", " if (!response.ok) return [];", " const shard = await response.json();", - " // layer: collection, data-directory, platform or emulator. owner: which one.", + " // layer: collection, archive-member, data-directory, platform or emulator.", " return shard.items.filter((item) => item.value === sha1);", "}", "```", @@ -4924,6 +4924,31 @@ def _data_member_digests( return found +def _archive_member_records(entry: dict, sha1: str, url: str, + root: Path = _REPO_ROOT) -> list[dict]: + """The CRC32 of each member of a collection archive, read from its + central directory. + + A pasted CRC of a ROM the collection holds only inside a set (logica2.u2 + in a500.zip, 8484f426) found nothing: 26,014 of 33,403 members were in + no record. The record leads to the archive's card and names it. + """ + path = root / str(entry.get("path", "")) + if path.suffix.lower() != ".zip" or not path.is_file(): + return [] + try: + with zipfile.ZipFile(path) as archive: + members = [(info.filename, info.CRC) for info in archive.infolist() + if not info.is_dir()] + except (OSError, zipfile.BadZipFile) as exc: + print(f" WARNING: {path}: members unreadable ({exc})", file=sys.stderr) + return [] + return [{"value": f"{crc:08x}", "algorithm": "crc32", "layer": "archive-member", + "name": member.rsplit("/", 1)[-1], "archive": str(entry.get("name", "")), + "sha1": sha1, "url": url} + for member, crc in members] + + def file_lookup_records( db: dict, coverages: dict, @@ -4968,6 +4993,7 @@ def file_lookup_records( "name": str(entry.get("name", "")), "sha1": sha1, "url": item["url"], **({"system_name": held_by} if held_by else {}), }) + records += _archive_member_records(entry, sha1, item["url"]) for key, coverage in sorted(coverages.items()): for system_id, system in sorted((coverage["config"].get("systems") or {}).items()): url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}") diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 828e47792..1d96d3f0a 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -785,6 +785,53 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase): self.assertNotIn("no.json", names) +class AnArchiveMemberIsFoundByItsCrc(unittest.TestCase): + """A pasted CRC of logica2.u2 (8484f426), which the collection holds + inside a500.zip, answered that no file the site knows has it: the + lookup held the members a profile's `contents:` declares, 7,389 of + 33,403.""" + + def test_each_member_is_a_record_leading_to_its_archive(self): + import zipfile # noqa: PLC0415 + + import generate_site # noqa: PLC0415 + + with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp: + root = Path(tmp) + archive = root / "bios" / "Arcade" / "Arcade" / "a500.zip" + archive.parent.mkdir(parents=True) + with zipfile.ZipFile(archive, "w") as handle: + handle.writestr("logica2.u2", b"rom") + handle.writestr("sub/kick.rom", b"kick") + entry = {"name": "a500.zip", "path": "bios/Arcade/Arcade/a500.zip"} + records = generate_site._archive_member_records(entry, "ab" * 20, "/retrobios/x/", root) + plain = generate_site._archive_member_records( + {"name": "a.bin", "path": "bios/a.bin"}, "cd" * 20, "/retrobios/y/", root) + self.assertEqual(plain, []) + import zlib # noqa: PLC0415 + + self.assertEqual(records, [ + {"value": f"{zlib.crc32(b'rom'):08x}", "algorithm": "crc32", "layer": "archive-member", + "name": "logica2.u2", "archive": "a500.zip", "sha1": "ab" * 20, "url": "/retrobios/x/"}, + {"value": f"{zlib.crc32(b'kick'):08x}", "algorithm": "crc32", "layer": "archive-member", + "name": "kick.rom", "archive": "a500.zip", "sha1": "ab" * 20, "url": "/retrobios/x/"}, + ]) + + def test_the_lookup_and_the_schema_carry_the_layer(self): + import json as _json # noqa: PLC0415 + + import generate_site # noqa: PLC0415 + + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn("records += _archive_member_records(entry, sha1, item[\"url\"])", source) + schema = _json.loads((Path(__file__).resolve().parents[1] / "schemas" + / "site-api-envelope.schema.json").read_text()) + self.assertIn('"archive-member"', _json.dumps(schema)) + script = (Path(__file__).resolve().parents[1] / "website" / "src" / "scripts" + / "site.ts").read_text(encoding="utf-8") + self.assertIn("byLayer('archive-member')", script) + + class AFileHasAnAddressByContent(unittest.TestCase): """mt32_control.rom moved from the Commodore page to the Roland page and every link to its card broke: the anchor was content-based, the page diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index 30a5741d2..899a7cebf 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -21,6 +21,8 @@ type Item = { owner?: string; sha1?: string; system_name?: string; + // On an archive-member record: the collection archive holding the member. + archive?: string; }; const siteRoot = () => new URL(import.meta.env.BASE_URL.replace(/\/?$/, '/'), location.origin); @@ -521,7 +523,9 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched: // naomi2.zip by the MD5 of a ROM inside it, which no file of the // collection has. const searched = new Set(matched.map((item) => item.value)); - const carried = [...new Set(matched.flatMap((item) => (item.sha1 && !searched.has(item.sha1) ? [item.sha1] : [])))]; + // A member's record names its archive, which is not the file searched for. + const carried = [...new Set(matched.flatMap((item) => + (item.sha1 && item.layer !== 'archive-member' && !searched.has(item.sha1) ? [item.sha1] : [])))]; const found = carried.length ? [...matched, ...(await lookupValues(carried).catch(() => [] as Item[]))] : matched; const held = distinct(found.filter((item) => item.layer === 'collection'), (item) => item.sha1 ?? item.url); const files = await Promise.all(held.map((item) => heldFile(item).catch(() => undefined))); @@ -550,8 +554,11 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched: const readEntries = emulators.map((item) => readerEntry(reader.get(item.owner ?? ''), item, valuesOf(item))); const setups = new Set(platforms.map((item) => item.owner)); + const members = distinct(byLayer('archive-member'), (item) => `${item.sha1}|${item.name}`); const counted = [ - held.length ? count(held.length, 'file in the collection', 'files in the collection') : 'not in the collection', + held.length ? count(held.length, 'file in the collection', 'files in the collection') + : members.length ? count(members.length, 'member of a collection archive', 'members of collection archives') + : 'not in the collection', ...(platforms.length ? [`${count(platforms.length, 'declaration', 'declarations')} by ${count(setups.size, 'setup', 'setups')}`] : []), ...(owners.length ? [count(owners.length, 'emulator that reads it', 'emulators that read it')] : []), ]; @@ -606,7 +613,17 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched: ), ); }); - if (!held.length) blocks.push(paragraph('No file in the collection has this hash.')); + if (!held.length && !members.length) blocks.push(paragraph('No file in the collection has this hash.')); + if (members.length) { + // A ROM the collection holds inside a set: the set's card, by its name. + const list = document.createElement('ul'); + for (const item of members) { + const row = document.createElement('li'); + row.append(code(item.name ?? item.value), ' in ', link(item.url, code(item.archive ?? ''))); + list.append(row); + } + blocks.push(paragraph(strong('Inside an archive of the collection')), list); + } const shipped = byLayer('data-directory'); if (shipped.length) { const list = document.createElement('ul');