mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
feat: link each platform and system to its data
An emulator page linked its YAML and api/v1 JSON, platform and system pages nothing. Each platform page now links its YAML, its own api/v1/platforms/<id>.json and its install manifest, and each system page its api/v1/systems/<slug>.json, both validated by the envelope schema.
This commit is contained in:
1 parent
2239df669f
commit
69965ebb25
4 files changed
+176
-5
No files matched your search
@@ -25,6 +25,7 @@
|
||||
"catalog",
|
||||
"platforms",
|
||||
"emulators",
|
||||
"systems",
|
||||
"verification-and-coverage-gaps",
|
||||
"file-locations"
|
||||
]
|
||||
@@ -261,6 +262,76 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "systems"
|
||||
}
|
||||
}
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"items": {
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"consoles",
|
||||
"id",
|
||||
"name"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"consoles": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"name",
|
||||
"path",
|
||||
"sha1"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"path": {
|
||||
"type": "string"
|
||||
},
|
||||
"size": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"sha1": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-f]{40}$"
|
||||
},
|
||||
"md5": {
|
||||
"type": "string"
|
||||
},
|
||||
"sha256": {
|
||||
"type": "string"
|
||||
},
|
||||
"crc32": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1060,6 +1060,35 @@ def _write_sqlite_export(
|
||||
os.replace(temp_path, destination)
|
||||
|
||||
|
||||
def write_single_item(path: Path, kind: str, item: dict, generated_at: str) -> None:
|
||||
"""One item in the envelope of its collection, for a page to link."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_if_changed(
|
||||
str(path),
|
||||
json.dumps(
|
||||
{"schema_version": 1, "generated_at": generated_at, "kind": kind,
|
||||
"count": 1, "items": [item]},
|
||||
indent=2, sort_keys=True, ensure_ascii=False, default=str,
|
||||
) + "\n",
|
||||
)
|
||||
|
||||
|
||||
def system_export_item(slug: str, manufacturer: str, consoles: dict[str, list]) -> dict:
|
||||
"""A system page's files by console, with the hashes its cards show."""
|
||||
fields = ("name", "path", "size", "sha1", "md5", "sha256", "crc32")
|
||||
return {
|
||||
"id": slug,
|
||||
"name": manufacturer,
|
||||
"consoles": {
|
||||
console: [
|
||||
{field: entry[field] for field in fields if entry.get(field) is not None}
|
||||
for entry in sorted(entries, key=lambda e: (e.get("name", ""), e.get("sha1", "")))
|
||||
]
|
||||
for console, entries in sorted(consoles.items())
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def generate_data_exports(
|
||||
docs: Path,
|
||||
db: dict,
|
||||
@@ -1100,6 +1129,11 @@ def generate_data_exports(
|
||||
str(api / filename),
|
||||
json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
)
|
||||
# One platform without the others, as api/v1/emulators/<id>.json is one
|
||||
# profile: platforms.json is 2.8 MB for twelve platforms.
|
||||
for item in platform_items:
|
||||
write_single_item(api / "platforms" / f"{item['id']}.json",
|
||||
"platforms", item, generated_at)
|
||||
write_if_changed(str(api / "stats.json"), generate_stats(stats))
|
||||
write_if_changed(
|
||||
str(api / "database.json"),
|
||||
@@ -1280,7 +1314,9 @@ def generate_data_page(exports: list[dict]) -> str:
|
||||
"`api/v1/emulators/<id>.json`, the envelope of `emulators.json` holding "
|
||||
"that one item, its `profile` valid against "
|
||||
"[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each "
|
||||
"emulator page links its own.",
|
||||
"emulator page links its own. `api/v1/platforms/<id>.json` holds one "
|
||||
"platform of `platforms.json`, and `api/v1/systems/<slug>.json` the files "
|
||||
"of one system page by console. Each platform and system page links its own.",
|
||||
"",
|
||||
"## Bulk downloads",
|
||||
"",
|
||||
@@ -2000,6 +2036,22 @@ def _render_platform_systems(
|
||||
_EXPORTED: set[str] = set()
|
||||
|
||||
|
||||
def _platform_data_links(name: str) -> str:
|
||||
"""The platform's own data, machine-readable: the YAML transcribed from
|
||||
its upstream list, this page's JSON and the manifest the installer reads.
|
||||
|
||||
An emulator page linked its YAML and JSON, a platform page nothing: a
|
||||
tool had to download all twelve platforms in platforms.json and filter.
|
||||
"""
|
||||
links = [
|
||||
f"[YAML]({REPO_URL}/blob/main/platforms/{name}.yml)",
|
||||
f"[JSON](../api/v1/platforms/{name}.json)",
|
||||
]
|
||||
if (_REPO_ROOT / "install" / f"{name}.json").is_file():
|
||||
links.append(f"[install manifest]({REPO_URL}/blob/main/install/{name}.json)")
|
||||
return ", ".join(links)
|
||||
|
||||
|
||||
def generate_platform_page(
|
||||
name: str,
|
||||
cov: dict,
|
||||
@@ -2093,6 +2145,7 @@ def generate_platform_page(
|
||||
lines.append(f"| Transcribed from | <{transcribed}>{note} |")
|
||||
if name in _EXPORTED:
|
||||
lines.append(f"| Updated file | [Download it and see the changes](../exports.md#{name}) |")
|
||||
lines.append(f"| Data | {_platform_data_links(name)} |")
|
||||
contrib_list = (registry or {}).get(name, {}).get("contributed_by", [])
|
||||
if contrib_list:
|
||||
for cb in contrib_list:
|
||||
@@ -2101,8 +2154,8 @@ def generate_platform_page(
|
||||
pr = cb.get("pr")
|
||||
pr_link = f" ([#{pr}]({REPO_URL}/pull/{pr}))" if pr else ""
|
||||
lines.append(
|
||||
f"| Contributed by | [@{username}](https://github.com/{username})"
|
||||
f" - {contribution}{pr_link} |"
|
||||
f"| Contributed by | [@{username}](https://github.com/{username}):"
|
||||
f" {contribution}{pr_link} |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
@@ -2794,10 +2847,13 @@ def generate_system_page(
|
||||
consoles: dict[str, list],
|
||||
consumers: FileConsumers,
|
||||
) -> str:
|
||||
manufacturer.lower().replace(" ", "-")
|
||||
slug = manufacturer.lower().replace(" ", "-")
|
||||
lines = [
|
||||
f"# {manufacturer}",
|
||||
"",
|
||||
f"The files below by console, with every hash, are also published as "
|
||||
f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json).",
|
||||
"",
|
||||
]
|
||||
anchored: set[str] = set()
|
||||
|
||||
@@ -5439,6 +5495,11 @@ def _generate(args: argparse.Namespace) -> None:
|
||||
slug = mfr.lower().replace(" ", "-")
|
||||
page = generate_system_page(mfr, consoles, consumers)
|
||||
PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles)
|
||||
write_single_item(
|
||||
docs / "api" / "v1" / "systems" / f"{slug}.json", "systems",
|
||||
system_export_item(slug, mfr, consoles),
|
||||
str(db.get("generated_at") or _timestamp()),
|
||||
)
|
||||
write_if_changed(str(docs / "systems" / f"{slug}.md"), page)
|
||||
|
||||
print("Generating emulator pages...")
|
||||
|
||||
@@ -419,7 +419,9 @@ def main() -> int:
|
||||
api_dir / name
|
||||
for name in ("catalog.json", "platforms.json", "emulators.json", "gaps.json")
|
||||
if (api_dir / name).exists()
|
||||
]
|
||||
] + sorted((api_dir / "platforms").glob("*.json")) + sorted(
|
||||
(api_dir / "systems").glob("*.json")
|
||||
)
|
||||
errors.extend(
|
||||
_validate_json_files(envelope_paths, "site-api-envelope.schema.json")
|
||||
)
|
||||
|
||||
@@ -882,5 +882,42 @@ class NestedProfileBlocksRenderAsLists(unittest.TestCase):
|
||||
self.assertIn("name: a.bin, size: 2", html)
|
||||
|
||||
|
||||
class EveryPageLinksItsOwnData(unittest.TestCase):
|
||||
"""An emulator page linked its YAML and JSON. A platform page linked
|
||||
nothing, and a tool had to download platforms.json (2.8 MB) and filter
|
||||
it. A system page linked nothing either."""
|
||||
|
||||
def test_platform_and_system_documents(self):
|
||||
import generate_site # noqa: PLC0415
|
||||
from cross_reference import FileConsumers # noqa: PLC0415
|
||||
|
||||
cov = {"platform": "Batocera", "mode": "md5", "present": 0, "total": 0,
|
||||
"verified": 0, "missing": 0, "details": [], "config": {"systems": {}}}
|
||||
page = generate_site.generate_platform_page("batocera", cov)
|
||||
self.assertIn("[JSON](../api/v1/platforms/batocera.json)", page)
|
||||
self.assertIn("/blob/main/platforms/batocera.yml)", page)
|
||||
|
||||
entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3,
|
||||
"path": "bios/Sony/PlayStation/a.bin"}
|
||||
page = generate_site.generate_system_page(
|
||||
"Sony", {"PlayStation": [entry]}, FileConsumers({"files": {}}))
|
||||
self.assertIn("[JSON](../api/v1/systems/sony.json)", page)
|
||||
|
||||
item = generate_site.system_export_item("sony", "Sony", {"PlayStation": [entry]})
|
||||
self.assertEqual(item["consoles"]["PlayStation"][0]["sha1"], "a" * 40)
|
||||
with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp:
|
||||
path = Path(tmp) / "sony.json"
|
||||
generate_site.write_single_item(path, "systems", item, "2026-10-21T00:00:00Z")
|
||||
document = json.loads(path.read_text())
|
||||
self.assertEqual(document["count"], 1)
|
||||
try:
|
||||
import jsonschema # noqa: PLC0415
|
||||
except ImportError:
|
||||
return
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
envelope = json.loads((root / "schemas" / "site-api-envelope.schema.json").read_text())
|
||||
jsonschema.validate(document, envelope)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user