From 4f930b5573c905880910bc360337460fee20f3b0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:19:08 +0100 Subject: [PATCH] feat: find wanted dumps by hash on the files page files/?hash= read the collection, platforms, emulators and retired files, never the Wanted list: the SHA1 of Redump's ps2-dvd-212k.bin answered that nothing the site knows has it. The lookup now carries a wanted layer from the rows of provenance-wanted.csv, computed once for both, and the page names the catalog that lists the dump. --- schemas/site-api-envelope.schema.json | 6 +++-- scripts/generate_site.py | 27 ++++++++++++++++++--- tests/test_site_exports.py | 34 +++++++++++++++++++++++++++ website/src/scripts/site.ts | 17 +++++++++++++- 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index adbda67c5..4b9e765fd 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -393,7 +393,8 @@ "retired", "data-directory", "platform", - "emulator" + "emulator", + "wanted" ] }, "left": { @@ -433,7 +434,8 @@ "description": "On a platform or emulator record: the algorithm of the value its owner wrote, filed under that value and under the file's other hashes. name: the emulator reads the file by its name and declares no hash that matches it." }, "owner": { - "type": "string" + "type": "string", + "description": "On a platform or emulator record: the platform or profile key. On a wanted record: the dump catalog that lists the value (Redump, No-Intro, TOSEC), the collection holding no file with it." }, "system_name": { "type": "string", diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 0eece3cfc..53e2c875f 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1620,6 +1620,7 @@ def generate_data_exports( members = data_directory_members() locations = file_location_items(db, profiles, members) + wanted = _provenance_wanted_rows(db) envelopes = { "platforms.json": ("platforms", platform_items), "emulators.json": ("emulators", emulator_items), @@ -1628,7 +1629,7 @@ def generate_data_exports( } write_lookup_shards( api, file_lookup_records(db, coverages, profiles, locations, members, consumers, - retired_files.load(_REPO_ROOT / "retired.json")), + retired_files.load(_REPO_ROOT / "retired.json"), wanted), generated_at, commit, ) for filename, (kind, items) in envelopes.items(): @@ -1747,7 +1748,7 @@ def generate_data_exports( _record(downloads / "retrobios.sqlite") write_if_changed( str(downloads / "provenance-wanted.csv"), - _csv_document(list(WANTED_FIELDS), _provenance_wanted_rows(db)), + _csv_document(list(WANTED_FIELDS), wanted), ) assets = [ @@ -5010,12 +5011,14 @@ def file_lookup_records( members: dict | None = None, consumers: FileConsumers | None = None, retired: dict | None = None, + wanted: list[dict] | None = None, ) -> list[dict]: """Every hash the site knows, one record per value, for files/?hash=. The collection's own files lead to their card, a platform's declared value to the system section of its page, a profile's to the entry's - card. The site search indexed these values as text, 4 MB of the 21 MB + card, a wanted dump (``wanted``, the provenance-wanted.csv rows) to + its catalog on the provenance page. The site search indexed these values as text, 4 MB of the 21 MB it made every visitor download: the lookup reads one shard instead. """ files = db.get("files", {}) @@ -5088,6 +5091,7 @@ def file_lookup_records( **({"system_name": entry_label} if entry_label else {}), }) records += retired_records(retired or {}, files, locations) + records += wanted_records(wanted or []) records += _held_under_every_hash(records, files, db.get("indexes") or {}) for record in records: if record["layer"] in ("platform", "emulator"): @@ -5099,6 +5103,23 @@ def file_lookup_records( r["value"], r["layer"], r.get("owner", ""), r["name"], r["url"])) +def wanted_records(rows: list[dict]) -> list[dict]: + """A record under each hash of a dump a catalog lists and the collection + lacks, from the rows of provenance-wanted.csv. A pasted hash of a + wanted dump (Redump's ps2-dvd-212k.bin) answered that nothing the site + knows has it, and never said the dump is wanted.""" + records = [] + for row in rows: + label = _PROVENANCE_LABELS.get(row["catalog"], row["catalog"]) + url = _site_path(f"provenance.md#{_slugify_anchor(label)}") + for algorithm in ("sha1", "md5", "crc32"): + value = str(row.get(algorithm) or "").lower() + if value: + records.append({"value": value, "algorithm": algorithm, "layer": "wanted", + "owner": label, "name": row["name"], "url": url}) + return records + + def _read_by_name(records: list[dict], files: dict, consumers: FileConsumers) -> list[dict]: """The emulators the file's card names that no declared hash reached. diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 1d96d3f0a..5455a422b 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -832,6 +832,40 @@ class AnArchiveMemberIsFoundByItsCrc(unittest.TestCase): self.assertIn("byLayer('archive-member')", script) +class AWantedDumpIsFoundByItsHash(unittest.TestCase): + """The SHA1 of a PS2 SCPH-30005R BIOS, which Redump lists as + ps2-dvd-212k.bin and the Wanted page asks for, answered that no file + the site knows has it.""" + + def test_each_hash_of_a_wanted_row_is_a_record(self): + import generate_site # noqa: PLC0415 + + row = {"catalog": "redump", "dat": "Sony - PlayStation 2 - BIOS Images", + "name": "ps2-dvd-212k.bin", "description": "", "size": "4194304", + "crc32": "1A2B3C4D", "md5": "", "sha1": "5b346eb9258d710609a9855befd35cab8f3a334a"} + records = generate_site.file_lookup_records( + {"files": {}, "indexes": {}}, {}, {}, [], wanted=[row, {**row, "dat": "other"}]) + self.assertEqual(records, [ + {"value": "1a2b3c4d", "algorithm": "crc32", "layer": "wanted", "owner": "Redump", + "name": "ps2-dvd-212k.bin", "url": "/retrobios/provenance/#redump"}, + {"value": "5b346eb9258d710609a9855befd35cab8f3a334a", "algorithm": "sha1", + "layer": "wanted", "owner": "Redump", "name": "ps2-dvd-212k.bin", + "url": "/retrobios/provenance/#redump"}, + ]) + + def test_the_schema_and_the_page_carry_the_layer(self): + import json as _json # noqa: PLC0415 + + import generate_site # noqa: PLC0415 + + root = Path(__file__).resolve().parents[1] + schema = _json.loads((root / "schemas" / "site-api-envelope.schema.json").read_text()) + self.assertIn('"wanted"', _json.dumps(schema)) + self.assertIn("byLayer('wanted')", (root / "website" / "src" / "scripts" / "site.ts").read_text()) + source = Path(generate_site.__file__).read_text(encoding="utf-8") + self.assertIn('retired_files.load(_REPO_ROOT / "retired.json"), wanted)', source) + + class AFileHasAnAddressByContent(unittest.TestCase): """mt32_control.rom moved from the Commodore page to the Roland page and every link to its card broke: the anchor was content-based, the page diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index 09d20902f..263309d34 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -558,9 +558,12 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched: const setups = new Set(platforms.map((item) => item.owner)); const members = distinct(byLayer('archive-member'), (item) => `${item.sha1}|${item.name}`); + // A dump a catalog lists and the collection lacks: the Wanted list. + const wanted = held.length || members.length ? [] : distinct(byLayer('wanted'), (item) => `${item.owner}|${item.name}`); const counted = [ held.length ? count(held.length, 'file in the collection', 'files in the collection') : members.length ? count(members.length, 'member of a collection archive', 'members of collection archives') + : wanted.length ? 'a wanted dump, not in the collection yet' : 'not in the collection', ...(platforms.length ? [`${count(platforms.length, 'declaration', 'declarations')} by ${count(setups.size, 'setup', 'setups')}`] : []), ...(owners.length ? [count(owners.length, 'emulator that reads it', 'emulators that read it')] : []), @@ -617,8 +620,20 @@ const renderMatches = async (status: HTMLElement, result: HTMLElement, matched: ); }); const retired = held.length ? [] : distinct(byLayer('retired'), (item) => item.sha1 ?? item.value); - if (!held.length && !members.length && !retired.length) blocks.push(paragraph('No file in the collection has this hash.')); + if (!held.length && !members.length && !retired.length && !wanted.length) blocks.push(paragraph('No file in the collection has this hash.')); blocks.push(...retired.map(retiredParagraph)); + if (wanted.length) { + const list = document.createElement('ul'); + for (const item of wanted) { + const row = document.createElement('li'); + row.append(link(item.url, item.owner ?? ''), ' lists it as ', code(item.name ?? item.value)); + list.append(row); + } + blocks.push( + paragraph(strong('Wanted.'), ' The collection does not hold this dump yet. If you dumped it yourself, ', link('contributing/', 'add it'), '.'), + list, + ); + } if (members.length) { // A ROM the collection holds inside a set: the set's card, by its name. const list = document.createElement('ul');