diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 2efe25a0..ae190302 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1216,6 +1216,48 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: PAGE_DATA: dict[str, dict] = {} +SYSTEM_PARTS_IN_FULL = 1000 + + +def system_structured_data(slug: str, consoles: dict[str, list]) -> dict: + """A system page's files for a machine: each card as a CreativeWork. + + The pages that hold a card per file published a bare TechArticle with + no file, hash or provenance, while emulator pages listed theirs. + """ + parts = [] + for _console, entries in sorted(consoles.items()): + for entry in entries: + sha1 = str(entry.get("sha1") or "") + part: dict = {"@type": "CreativeWork", "name": str(entry.get("name", ""))} + if sha1: + part["url"] = urllib.parse.urljoin(SITE_URL, f"systems/{slug}/#sha1-{sha1[:12]}") + identifiers = [ + f"{field}:{str(entry[field]).lower()}" + for field in ("sha1", "md5", "crc32", "sha256") if entry.get(field) + ] + if identifiers: + part["identifier"] = identifiers + if entry.get("size"): + part["contentSize"] = f"{entry['size']} B" + citations = [ + f"{catalog}: {record.get('dat', '')} ({record.get('name', '')})" + for catalog, record in sorted((entry.get("provenance") or {}).items()) + if isinstance(record, dict) + ] + if citations: + part["citation"] = citations + parts.append(part) + if len(parts) > SYSTEM_PARTS_IN_FULL: + # game-engines would carry 2.6 MB of JSON-LD: past the cap a page + # lists its catalogue-verified files and points at the full table. + return { + "hasPart": [part for part in parts if part.get("citation")], + "isBasedOn": urllib.parse.urljoin(SITE_URL, "downloads/files.csv"), + } + return {"hasPart": parts} if parts else {} + + def dataset_structured_data(exports: list[dict], db: dict) -> dict: """The data page as a schema.org Dataset a harvester can index. @@ -4129,6 +4171,7 @@ def main() -> None: for mfr, consoles in manufacturers.items(): slug = mfr.lower().replace(" ", "-") page = generate_system_page(mfr, consoles, consumers) + PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles) write_if_changed(str(docs / "systems" / f"{slug}.md"), page) # Generate emulator pages diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index c8f0673f..82edb659 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -4690,5 +4690,33 @@ class TheDatasetDescribesItsDownloads(unittest.TestCase): self.assertNotEqual(data["name"], "Data & API") + +class SystemPagesDescribeTheirFiles(unittest.TestCase): + """systems/* published a generic TechArticle: no file, hash or + provenance for the pages that carry a card per file.""" + + def test_parts_carry_hashes_anchor_and_provenance(self): + from scripts.generate_site import system_structured_data + + entry = {"name": "scph5501.bin", "sha1": "0555c6fae8906f3f09baf5988f00e55f88e9f30b", + "md5": "490f666e1afb15b7362b406ed1cea246", "crc32": "8d8cb7e4", "size": 524288, + "provenance": {"redump": {"dat": "Sony - PlayStation - BIOS Images", + "name": "ps-30a.bin"}}} + (part,) = system_structured_data("sony", {"PlayStation": [entry]})["hasPart"] + self.assertTrue(part["url"].endswith("systems/sony/#sha1-0555c6fae890")) + self.assertIn("md5:490f666e1afb15b7362b406ed1cea246", part["identifier"]) + self.assertIn("redump: Sony - PlayStation - BIOS Images (ps-30a.bin)", part["citation"]) + + def test_a_huge_page_lists_its_verified_files_and_points_at_the_table(self): + from scripts.generate_site import SYSTEM_PARTS_IN_FULL, system_structured_data + + plain = [{"name": f"f{i}", "sha1": f"{i:040x}"} for i in range(SYSTEM_PARTS_IN_FULL)] + cited = {"name": "v", "sha1": "f" * 40, + "provenance": {"tosec": {"dat": "d", "name": "n"}}} + data = system_structured_data("engines", {"x": [*plain, cited]}) + self.assertEqual([p["name"] for p in data["hasPart"]], ["v"]) + self.assertTrue(data["isBasedOn"].endswith("downloads/files.csv")) + + if __name__ == "__main__": unittest.main()