feat: describe system page files for machines

scripts/generate_site.py

@@ -1216,6 +1216,48 @@ def _page_description(markdown: str, relative: Path, title: str) -> str:
 PAGE_DATA: dict[str, dict] = {}
 
 
+SYSTEM_PARTS_IN_FULL = 1000
+
+
+def system_structured_data(slug: str, consoles: dict[str, list]) -> dict:
+    """A system page's files for a machine: each card as a CreativeWork.
+
+    The pages that hold a card per file published a bare TechArticle with
+    no file, hash or provenance, while emulator pages listed theirs.
+    """
+    parts = []
+    for _console, entries in sorted(consoles.items()):
+        for entry in entries:
+            sha1 = str(entry.get("sha1") or "")
+            part: dict = {"@type": "CreativeWork", "name": str(entry.get("name", ""))}
+            if sha1:
+                part["url"] = urllib.parse.urljoin(SITE_URL, f"systems/{slug}/#sha1-{sha1[:12]}")
+            identifiers = [
+                f"{field}:{str(entry[field]).lower()}"
+                for field in ("sha1", "md5", "crc32", "sha256") if entry.get(field)
+            ]
+            if identifiers:
+                part["identifier"] = identifiers
+            if entry.get("size"):
+                part["contentSize"] = f"{entry['size']} B"
+            citations = [
+                f"{catalog}: {record.get('dat', '')} ({record.get('name', '')})"
+                for catalog, record in sorted((entry.get("provenance") or {}).items())
+                if isinstance(record, dict)
+            ]
+            if citations:
+                part["citation"] = citations
+            parts.append(part)
+    if len(parts) > SYSTEM_PARTS_IN_FULL:
+        # game-engines would carry 2.6 MB of JSON-LD: past the cap a page
+        # lists its catalogue-verified files and points at the full table.
+        return {
+            "hasPart": [part for part in parts if part.get("citation")],
+            "isBasedOn": urllib.parse.urljoin(SITE_URL, "downloads/files.csv"),
+        }
+    return {"hasPart": parts} if parts else {}
+
+
 def dataset_structured_data(exports: list[dict], db: dict) -> dict:
     """The data page as a schema.org Dataset a harvester can index.
 
@@ -4129,6 +4171,7 @@ def main() -> None:
     for mfr, consoles in manufacturers.items():
         slug = mfr.lower().replace(" ", "-")
         page = generate_system_page(mfr, consoles, consumers)
+        PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles)
         write_if_changed(str(docs / "systems" / f"{slug}.md"), page)
 
     # Generate emulator pages

tests/test_audit_regressions.py

@@ -4690,5 +4690,33 @@ class TheDatasetDescribesItsDownloads(unittest.TestCase):
         self.assertNotEqual(data["name"], "Data & API")
 
 
+
+class SystemPagesDescribeTheirFiles(unittest.TestCase):
+    """systems/* published a generic TechArticle: no file, hash or
+    provenance for the pages that carry a card per file."""
+
+    def test_parts_carry_hashes_anchor_and_provenance(self):
+        from scripts.generate_site import system_structured_data
+
+        entry = {"name": "scph5501.bin", "sha1": "0555c6fae8906f3f09baf5988f00e55f88e9f30b",
+                 "md5": "490f666e1afb15b7362b406ed1cea246", "crc32": "8d8cb7e4", "size": 524288,
+                 "provenance": {"redump": {"dat": "Sony - PlayStation - BIOS Images",
+                                           "name": "ps-30a.bin"}}}
+        (part,) = system_structured_data("sony", {"PlayStation": [entry]})["hasPart"]
+        self.assertTrue(part["url"].endswith("systems/sony/#sha1-0555c6fae890"))
+        self.assertIn("md5:490f666e1afb15b7362b406ed1cea246", part["identifier"])
+        self.assertIn("redump: Sony - PlayStation - BIOS Images (ps-30a.bin)", part["citation"])
+
+    def test_a_huge_page_lists_its_verified_files_and_points_at_the_table(self):
+        from scripts.generate_site import SYSTEM_PARTS_IN_FULL, system_structured_data
+
+        plain = [{"name": f"f{i}", "sha1": f"{i:040x}"} for i in range(SYSTEM_PARTS_IN_FULL)]
+        cited = {"name": "v", "sha1": "f" * 40,
+                 "provenance": {"tosec": {"dat": "d", "name": "n"}}}
+        data = system_structured_data("engines", {"x": [*plain, cited]})
+        self.assertEqual([p["name"] for p in data["hasPart"]], ["v"])
+        self.assertTrue(data["isBasedOn"].endswith("downloads/files.csv"))
+
+
 if __name__ == "__main__":
     unittest.main()