From 69965ebb2512c4670671075a564dbd06b45a4c28 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:15:11 +0200 Subject: [PATCH] feat: link each platform and system to its data An emulator page linked its YAML and api/v1 JSON, platform and system pages nothing. Each platform page now links its YAML, its own api/v1/platforms/.json and its install manifest, and each system page its api/v1/systems/.json, both validated by the envelope schema. --- schemas/site-api-envelope.schema.json | 71 +++++++++++++++++++++++++++ scripts/generate_site.py | 69 ++++++++++++++++++++++++-- scripts/validate_schemas.py | 4 +- tests/test_site_exports.py | 37 ++++++++++++++ 4 files changed, 176 insertions(+), 5 deletions(-) diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index df64e518..18bf24ff 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -25,6 +25,7 @@ "catalog", "platforms", "emulators", + "systems", "verification-and-coverage-gaps", "file-locations" ] @@ -261,6 +262,76 @@ } } } + }, + { + "if": { + "properties": { + "kind": { + "const": "systems" + } + } + }, + "then": { + "properties": { + "items": { + "items": { + "type": "object", + "required": [ + "consoles", + "id", + "name" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "consoles": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "path", + "sha1" + ], + "properties": { + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "sha1": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "md5": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "crc32": { + "type": "string" + } + } + } + } + } + } + } + } + } + } } ] } diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 785c16fb..13cefaea 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1060,6 +1060,35 @@ def _write_sqlite_export( os.replace(temp_path, destination) +def write_single_item(path: Path, kind: str, item: dict, generated_at: str) -> None: + """One item in the envelope of its collection, for a page to link.""" + path.parent.mkdir(parents=True, exist_ok=True) + write_if_changed( + str(path), + json.dumps( + {"schema_version": 1, "generated_at": generated_at, "kind": kind, + "count": 1, "items": [item]}, + indent=2, sort_keys=True, ensure_ascii=False, default=str, + ) + "\n", + ) + + +def system_export_item(slug: str, manufacturer: str, consoles: dict[str, list]) -> dict: + """A system page's files by console, with the hashes its cards show.""" + fields = ("name", "path", "size", "sha1", "md5", "sha256", "crc32") + return { + "id": slug, + "name": manufacturer, + "consoles": { + console: [ + {field: entry[field] for field in fields if entry.get(field) is not None} + for entry in sorted(entries, key=lambda e: (e.get("name", ""), e.get("sha1", ""))) + ] + for console, entries in sorted(consoles.items()) + }, + } + + def generate_data_exports( docs: Path, db: dict, @@ -1100,6 +1129,11 @@ def generate_data_exports( str(api / filename), json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True) + "\n", ) + # One platform without the others, as api/v1/emulators/.json is one + # profile: platforms.json is 2.8 MB for twelve platforms. + for item in platform_items: + write_single_item(api / "platforms" / f"{item['id']}.json", + "platforms", item, generated_at) write_if_changed(str(api / "stats.json"), generate_stats(stats)) write_if_changed( str(api / "database.json"), @@ -1280,7 +1314,9 @@ def generate_data_page(exports: list[dict]) -> str: "`api/v1/emulators/.json`, the envelope of `emulators.json` holding " "that one item, its `profile` valid against " "[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each " - "emulator page links its own.", + "emulator page links its own. `api/v1/platforms/.json` holds one " + "platform of `platforms.json`, and `api/v1/systems/.json` the files " + "of one system page by console. Each platform and system page links its own.", "", "## Bulk downloads", "", @@ -2000,6 +2036,22 @@ def _render_platform_systems( _EXPORTED: set[str] = set() +def _platform_data_links(name: str) -> str: + """The platform's own data, machine-readable: the YAML transcribed from + its upstream list, this page's JSON and the manifest the installer reads. + + An emulator page linked its YAML and JSON, a platform page nothing: a + tool had to download all twelve platforms in platforms.json and filter. + """ + links = [ + f"[YAML]({REPO_URL}/blob/main/platforms/{name}.yml)", + f"[JSON](../api/v1/platforms/{name}.json)", + ] + if (_REPO_ROOT / "install" / f"{name}.json").is_file(): + links.append(f"[install manifest]({REPO_URL}/blob/main/install/{name}.json)") + return ", ".join(links) + + def generate_platform_page( name: str, cov: dict, @@ -2093,6 +2145,7 @@ def generate_platform_page( lines.append(f"| Transcribed from | <{transcribed}>{note} |") if name in _EXPORTED: lines.append(f"| Updated file | [Download it and see the changes](../exports.md#{name}) |") + lines.append(f"| Data | {_platform_data_links(name)} |") contrib_list = (registry or {}).get(name, {}).get("contributed_by", []) if contrib_list: for cb in contrib_list: @@ -2101,8 +2154,8 @@ def generate_platform_page( pr = cb.get("pr") pr_link = f" ([#{pr}]({REPO_URL}/pull/{pr}))" if pr else "" lines.append( - f"| Contributed by | [@{username}](https://github.com/{username})" - f" - {contribution}{pr_link} |" + f"| Contributed by | [@{username}](https://github.com/{username}):" + f" {contribution}{pr_link} |" ) lines.extend( [ @@ -2794,10 +2847,13 @@ def generate_system_page( consoles: dict[str, list], consumers: FileConsumers, ) -> str: - manufacturer.lower().replace(" ", "-") + slug = manufacturer.lower().replace(" ", "-") lines = [ f"# {manufacturer}", "", + f"The files below by console, with every hash, are also published as " + f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json).", + "", ] anchored: set[str] = set() @@ -5439,6 +5495,11 @@ def _generate(args: argparse.Namespace) -> None: slug = mfr.lower().replace(" ", "-") page = generate_system_page(mfr, consoles, consumers) PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles) + write_single_item( + docs / "api" / "v1" / "systems" / f"{slug}.json", "systems", + system_export_item(slug, mfr, consoles), + str(db.get("generated_at") or _timestamp()), + ) write_if_changed(str(docs / "systems" / f"{slug}.md"), page) print("Generating emulator pages...") diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 2f9c3d52..9186078a 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -419,7 +419,9 @@ def main() -> int: api_dir / name for name in ("catalog.json", "platforms.json", "emulators.json", "gaps.json") if (api_dir / name).exists() - ] + ] + sorted((api_dir / "platforms").glob("*.json")) + sorted( + (api_dir / "systems").glob("*.json") + ) errors.extend( _validate_json_files(envelope_paths, "site-api-envelope.schema.json") ) diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 43693d08..91643f13 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -882,5 +882,42 @@ class NestedProfileBlocksRenderAsLists(unittest.TestCase): self.assertIn("name: a.bin, size: 2", html) +class EveryPageLinksItsOwnData(unittest.TestCase): + """An emulator page linked its YAML and JSON. A platform page linked + nothing, and a tool had to download platforms.json (2.8 MB) and filter + it. A system page linked nothing either.""" + + def test_platform_and_system_documents(self): + import generate_site # noqa: PLC0415 + from cross_reference import FileConsumers # noqa: PLC0415 + + cov = {"platform": "Batocera", "mode": "md5", "present": 0, "total": 0, + "verified": 0, "missing": 0, "details": [], "config": {"systems": {}}} + page = generate_site.generate_platform_page("batocera", cov) + self.assertIn("[JSON](../api/v1/platforms/batocera.json)", page) + self.assertIn("/blob/main/platforms/batocera.yml)", page) + + entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3, + "path": "bios/Sony/PlayStation/a.bin"} + page = generate_site.generate_system_page( + "Sony", {"PlayStation": [entry]}, FileConsumers({"files": {}})) + self.assertIn("[JSON](../api/v1/systems/sony.json)", page) + + item = generate_site.system_export_item("sony", "Sony", {"PlayStation": [entry]}) + self.assertEqual(item["consoles"]["PlayStation"][0]["sha1"], "a" * 40) + with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp: + path = Path(tmp) / "sony.json" + generate_site.write_single_item(path, "systems", item, "2026-10-21T00:00:00Z") + document = json.loads(path.read_text()) + self.assertEqual(document["count"], 1) + try: + import jsonschema # noqa: PLC0415 + except ImportError: + return + root = Path(__file__).resolve().parents[1] + envelope = json.loads((root / "schemas" / "site-api-envelope.schema.json").read_text()) + jsonschema.validate(document, envelope) + + if __name__ == "__main__": unittest.main()